drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 - 2016 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38 #include <sys/types.h>
  39 #include <dirent.h>
  40 #include <sys/stat.h>
  41 #include <fcntl.h>
  42 #include <sys/mman.h>
  43
  44 #include <libsze2.h>
  45
  46 #include <rte_mbuf.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_malloc.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_kvargs.h>
  51 #include <rte_dev.h>
  52 #include <rte_atomic.h>
  53
  54 #include "rte_eth_szedata2.h"
  55
  56 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  57 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  58 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  59
  60 /**
  61  * size of szedata2_packet header with alignment
  62  */
  63 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  64
  65 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2
  66 #define RTE_SZEDATA2_PCI_DRIVER_NAME "rte_szedata2_pmd"
  67
  68 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
  69
  70 struct szedata2_rx_queue {
  71         struct szedata *sze;
  72         uint8_t rx_channel;
  73         uint8_t in_port;
  74         struct rte_mempool *mb_pool;
  75         volatile uint64_t rx_pkts;
  76         volatile uint64_t rx_bytes;
  77         volatile uint64_t err_pkts;
  78 };
  79
  80 struct szedata2_tx_queue {
  81         struct szedata *sze;
  82         uint8_t tx_channel;
  83         volatile uint64_t tx_pkts;
  84         volatile uint64_t tx_bytes;
  85         volatile uint64_t err_pkts;
  86 };
  87
  88 struct pmd_internals {
  89         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  90         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  91         uint16_t max_rx_queues;
  92         uint16_t max_tx_queues;
  93         char sze_dev[PATH_MAX];
  94         struct rte_mem_resource *pci_rsc;
  95 };
  96
  97 static struct ether_addr eth_addr = {
  98         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
  99 };
 100
 101 static uint16_t
 102 eth_szedata2_rx(void *queue,
 103                 struct rte_mbuf **bufs,
 104                 uint16_t nb_pkts)
 105 {
 106         unsigned int i;
 107         struct rte_mbuf *mbuf;
 108         struct szedata2_rx_queue *sze_q = queue;
 109         struct rte_pktmbuf_pool_private *mbp_priv;
 110         uint16_t num_rx = 0;
 111         uint16_t buf_size;
 112         uint16_t sg_size;
 113         uint16_t hw_size;
 114         uint16_t packet_size;
 115         uint64_t num_bytes = 0;
 116         struct szedata *sze = sze_q->sze;
 117         uint8_t *header_ptr = NULL; /* header of packet */
 118         uint8_t *packet_ptr1 = NULL;
 119         uint8_t *packet_ptr2 = NULL;
 120         uint16_t packet_len1 = 0;
 121         uint16_t packet_len2 = 0;
 122         uint16_t hw_data_align;
 123
 124         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 125                 return 0;
 126
 127         /*
 128          * Reads the given number of packets from szedata2 channel given
 129          * by queue and copies the packet data into a newly allocated mbuf
 130          * to return.
 131          */
 132         for (i = 0; i < nb_pkts; i++) {
 133                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 134
 135                 if (unlikely(mbuf == NULL))
 136                         break;
 137
 138                 /* get the next sze packet */
 139                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 140                                 sze->ct_rx_lck->next == NULL) {
 141                         /* unlock old data */
 142                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 143                         sze->ct_rx_lck_orig = NULL;
 144                         sze->ct_rx_lck = NULL;
 145                 }
 146
 147                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 148                         /* nothing to read, lock new data */
 149                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 150                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 151
 152                         if (sze->ct_rx_lck == NULL) {
 153                                 /* nothing to lock */
 154                                 rte_pktmbuf_free(mbuf);
 155                                 break;
 156                         }
 157
 158                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 159                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 160
 161                         if (!sze->ct_rx_rem_bytes) {
 162                                 rte_pktmbuf_free(mbuf);
 163                                 break;
 164                         }
 165                 }
 166
 167                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 168                         /*
 169                          * cut in header
 170                          * copy parts of header to merge buffer
 171                          */
 172                         if (sze->ct_rx_lck->next == NULL) {
 173                                 rte_pktmbuf_free(mbuf);
 174                                 break;
 175                         }
 176
 177                         /* copy first part of header */
 178                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 179                                         sze->ct_rx_rem_bytes);
 180
 181                         /* copy second part of header */
 182                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 183                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 184                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 185                                 sze->ct_rx_cur_ptr,
 186                                 RTE_SZE2_PACKET_HEADER_SIZE -
 187                                 sze->ct_rx_rem_bytes);
 188
 189                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 190                                 sze->ct_rx_rem_bytes;
 191                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 192                                 RTE_SZE2_PACKET_HEADER_SIZE +
 193                                 sze->ct_rx_rem_bytes;
 194
 195                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 196                 } else {
 197                         /* not cut */
 198                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 199                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 200                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 201                 }
 202
 203                 sg_size = le16toh(*((uint16_t *)header_ptr));
 204                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 205                 packet_size = sg_size -
 206                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 207
 208
 209                 /* checks if packet all right */
 210                 if (!sg_size)
 211                         errx(5, "Zero segsize");
 212
 213                 /* check sg_size and hwsize */
 214                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 215                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 216                                 "hwsize: %d", sg_size, hw_size);
 217                 }
 218
 219                 hw_data_align =
 220                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 221                         RTE_SZE2_PACKET_HEADER_SIZE;
 222
 223                 if (sze->ct_rx_rem_bytes >=
 224                                 (uint16_t)(sg_size -
 225                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 226                         /* no cut */
 227                         /* one packet ready - go to another */
 228                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 229                         packet_len1 = packet_size;
 230                         packet_ptr2 = NULL;
 231                         packet_len2 = 0;
 232
 233                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 234                                 RTE_SZE2_PACKET_HEADER_SIZE;
 235                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 236                                 RTE_SZE2_PACKET_HEADER_SIZE;
 237                 } else {
 238                         /* cut in data */
 239                         if (sze->ct_rx_lck->next == NULL) {
 240                                 errx(6, "Need \"next\" lock, "
 241                                         "but it is missing: %u",
 242                                         sze->ct_rx_rem_bytes);
 243                         }
 244
 245                         /* skip hw data */
 246                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 247                                 uint16_t rem_size = hw_data_align -
 248                                         sze->ct_rx_rem_bytes;
 249
 250                                 /* MOVE to next lock */
 251                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 252                                 sze->ct_rx_cur_ptr =
 253                                         (void *)(((uint8_t *)
 254                                         (sze->ct_rx_lck->start)) + rem_size);
 255
 256                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 257                                 packet_len1 = packet_size;
 258                                 packet_ptr2 = NULL;
 259                                 packet_len2 = 0;
 260
 261                                 sze->ct_rx_cur_ptr +=
 262                                         RTE_SZE2_ALIGN8(packet_size);
 263                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 264                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 265                         } else {
 266                                 /* get pointer and length from first part */
 267                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 268                                         hw_data_align;
 269                                 packet_len1 = sze->ct_rx_rem_bytes -
 270                                         hw_data_align;
 271
 272                                 /* MOVE to next lock */
 273                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 274                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 275
 276                                 /* get pointer and length from second part */
 277                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 278                                 packet_len2 = packet_size - packet_len1;
 279
 280                                 sze->ct_rx_cur_ptr +=
 281                                         RTE_SZE2_ALIGN8(packet_size) -
 282                                         packet_len1;
 283                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 284                                         (RTE_SZE2_ALIGN8(packet_size) -
 285                                          packet_len1);
 286                         }
 287                 }
 288
 289                 if (unlikely(packet_ptr1 == NULL)) {
 290                         rte_pktmbuf_free(mbuf);
 291                         break;
 292                 }
 293
 294                 /* get the space available for data in the mbuf */
 295                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 296                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 297                                 RTE_PKTMBUF_HEADROOM);
 298
 299                 if (packet_size <= buf_size) {
 300                         /* sze packet will fit in one mbuf, go ahead and copy */
 301                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 302                                         packet_ptr1, packet_len1);
 303                         if (packet_ptr2 != NULL) {
 304                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 305                                         uint8_t *) + packet_len1),
 306                                         packet_ptr2, packet_len2);
 307                         }
 308                         mbuf->data_len = (uint16_t)packet_size;
 309
 310                         mbuf->pkt_len = packet_size;
 311                         mbuf->port = sze_q->in_port;
 312                         bufs[num_rx] = mbuf;
 313                         num_rx++;
 314                         num_bytes += packet_size;
 315                 } else {
 316                         /*
 317                          * sze packet will not fit in one mbuf,
 318                          * scattered mode is not enabled, drop packet
 319                          */
 320                         RTE_LOG(ERR, PMD,
 321                                 "SZE segment %d bytes will not fit in one mbuf "
 322                                 "(%d bytes), scattered mode is not enabled, "
 323                                 "drop packet!!\n",
 324                                 packet_size, buf_size);
 325                         rte_pktmbuf_free(mbuf);
 326                 }
 327         }
 328
 329         sze_q->rx_pkts += num_rx;
 330         sze_q->rx_bytes += num_bytes;
 331         return num_rx;
 332 }
 333
 334 static uint16_t
 335 eth_szedata2_rx_scattered(void *queue,
 336                 struct rte_mbuf **bufs,
 337                 uint16_t nb_pkts)
 338 {
 339         unsigned int i;
 340         struct rte_mbuf *mbuf;
 341         struct szedata2_rx_queue *sze_q = queue;
 342         struct rte_pktmbuf_pool_private *mbp_priv;
 343         uint16_t num_rx = 0;
 344         uint16_t buf_size;
 345         uint16_t sg_size;
 346         uint16_t hw_size;
 347         uint16_t packet_size;
 348         uint64_t num_bytes = 0;
 349         struct szedata *sze = sze_q->sze;
 350         uint8_t *header_ptr = NULL; /* header of packet */
 351         uint8_t *packet_ptr1 = NULL;
 352         uint8_t *packet_ptr2 = NULL;
 353         uint16_t packet_len1 = 0;
 354         uint16_t packet_len2 = 0;
 355         uint16_t hw_data_align;
 356
 357         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 358                 return 0;
 359
 360         /*
 361          * Reads the given number of packets from szedata2 channel given
 362          * by queue and copies the packet data into a newly allocated mbuf
 363          * to return.
 364          */
 365         for (i = 0; i < nb_pkts; i++) {
 366                 const struct szedata_lock *ct_rx_lck_backup;
 367                 unsigned int ct_rx_rem_bytes_backup;
 368                 unsigned char *ct_rx_cur_ptr_backup;
 369
 370                 /* get the next sze packet */
 371                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 372                                 sze->ct_rx_lck->next == NULL) {
 373                         /* unlock old data */
 374                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 375                         sze->ct_rx_lck_orig = NULL;
 376                         sze->ct_rx_lck = NULL;
 377                 }
 378
 379                 /*
 380                  * Store items from sze structure which can be changed
 381                  * before mbuf allocating. Use these items in case of mbuf
 382                  * allocating failure.
 383                  */
 384                 ct_rx_lck_backup = sze->ct_rx_lck;
 385                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 386                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 387
 388                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 389                         /* nothing to read, lock new data */
 390                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 391                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 392
 393                         /*
 394                          * Backup items from sze structure must be updated
 395                          * after locking to contain pointers to new locks.
 396                          */
 397                         ct_rx_lck_backup = sze->ct_rx_lck;
 398                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 399                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 400
 401                         if (sze->ct_rx_lck == NULL)
 402                                 /* nothing to lock */
 403                                 break;
 404
 405                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 406                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 407
 408                         if (!sze->ct_rx_rem_bytes)
 409                                 break;
 410                 }
 411
 412                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 413                         /*
 414                          * cut in header - copy parts of header to merge buffer
 415                          */
 416                         if (sze->ct_rx_lck->next == NULL)
 417                                 break;
 418
 419                         /* copy first part of header */
 420                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 421                                         sze->ct_rx_rem_bytes);
 422
 423                         /* copy second part of header */
 424                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 425                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 426                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 427                                 sze->ct_rx_cur_ptr,
 428                                 RTE_SZE2_PACKET_HEADER_SIZE -
 429                                 sze->ct_rx_rem_bytes);
 430
 431                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 432                                 sze->ct_rx_rem_bytes;
 433                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 434                                 RTE_SZE2_PACKET_HEADER_SIZE +
 435                                 sze->ct_rx_rem_bytes;
 436
 437                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 438                 } else {
 439                         /* not cut */
 440                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 441                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 442                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 443                 }
 444
 445                 sg_size = le16toh(*((uint16_t *)header_ptr));
 446                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 447                 packet_size = sg_size -
 448                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 449
 450
 451                 /* checks if packet all right */
 452                 if (!sg_size)
 453                         errx(5, "Zero segsize");
 454
 455                 /* check sg_size and hwsize */
 456                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 457                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 458                                         "hwsize: %d", sg_size, hw_size);
 459                 }
 460
 461                 hw_data_align =
 462                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 463                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 464
 465                 if (sze->ct_rx_rem_bytes >=
 466                                 (uint16_t)(sg_size -
 467                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 468                         /* no cut */
 469                         /* one packet ready - go to another */
 470                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 471                         packet_len1 = packet_size;
 472                         packet_ptr2 = NULL;
 473                         packet_len2 = 0;
 474
 475                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 476                                 RTE_SZE2_PACKET_HEADER_SIZE;
 477                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 478                                 RTE_SZE2_PACKET_HEADER_SIZE;
 479                 } else {
 480                         /* cut in data */
 481                         if (sze->ct_rx_lck->next == NULL) {
 482                                 errx(6, "Need \"next\" lock, but it is "
 483                                         "missing: %u", sze->ct_rx_rem_bytes);
 484                         }
 485
 486                         /* skip hw data */
 487                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 488                                 uint16_t rem_size = hw_data_align -
 489                                         sze->ct_rx_rem_bytes;
 490
 491                                 /* MOVE to next lock */
 492                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 493                                 sze->ct_rx_cur_ptr =
 494                                         (void *)(((uint8_t *)
 495                                         (sze->ct_rx_lck->start)) + rem_size);
 496
 497                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 498                                 packet_len1 = packet_size;
 499                                 packet_ptr2 = NULL;
 500                                 packet_len2 = 0;
 501
 502                                 sze->ct_rx_cur_ptr +=
 503                                         RTE_SZE2_ALIGN8(packet_size);
 504                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 505                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 506                         } else {
 507                                 /* get pointer and length from first part */
 508                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 509                                         hw_data_align;
 510                                 packet_len1 = sze->ct_rx_rem_bytes -
 511                                         hw_data_align;
 512
 513                                 /* MOVE to next lock */
 514                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 515                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 516
 517                                 /* get pointer and length from second part */
 518                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 519                                 packet_len2 = packet_size - packet_len1;
 520
 521                                 sze->ct_rx_cur_ptr +=
 522                                         RTE_SZE2_ALIGN8(packet_size) -
 523                                         packet_len1;
 524                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 525                                         (RTE_SZE2_ALIGN8(packet_size) -
 526                                          packet_len1);
 527                         }
 528                 }
 529
 530                 if (unlikely(packet_ptr1 == NULL))
 531                         break;
 532
 533                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 534
 535                 if (unlikely(mbuf == NULL)) {
 536                         /*
 537                          * Restore items from sze structure to state after
 538                          * unlocking (eventually locking).
 539                          */
 540                         sze->ct_rx_lck = ct_rx_lck_backup;
 541                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 542                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 543                         break;
 544                 }
 545
 546                 /* get the space available for data in the mbuf */
 547                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 548                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 549                                 RTE_PKTMBUF_HEADROOM);
 550
 551                 if (packet_size <= buf_size) {
 552                         /* sze packet will fit in one mbuf, go ahead and copy */
 553                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 554                                         packet_ptr1, packet_len1);
 555                         if (packet_ptr2 != NULL) {
 556                                 rte_memcpy((void *)
 557                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 558                                         packet_len1), packet_ptr2, packet_len2);
 559                         }
 560                         mbuf->data_len = (uint16_t)packet_size;
 561                 } else {
 562                         /*
 563                          * sze packet will not fit in one mbuf,
 564                          * scatter packet into more mbufs
 565                          */
 566                         struct rte_mbuf *m = mbuf;
 567                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 568
 569                         /* copy first part of packet */
 570                         /* fill first mbuf */
 571                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 572                                 len);
 573                         packet_len1 -= len;
 574                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 575
 576                         while (packet_len1 > 0) {
 577                                 /* fill new mbufs */
 578                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 579
 580                                 if (unlikely(m->next == NULL)) {
 581                                         rte_pktmbuf_free(mbuf);
 582                                         /*
 583                                          * Restore items from sze structure
 584                                          * to state after unlocking (eventually
 585                                          * locking).
 586                                          */
 587                                         sze->ct_rx_lck = ct_rx_lck_backup;
 588                                         sze->ct_rx_rem_bytes =
 589                                                 ct_rx_rem_bytes_backup;
 590                                         sze->ct_rx_cur_ptr =
 591                                                 ct_rx_cur_ptr_backup;
 592                                         goto finish;
 593                                 }
 594
 595                                 m = m->next;
 596
 597                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 598                                         packet_len1);
 599                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 600                                         packet_ptr1, len);
 601
 602                                 (mbuf->nb_segs)++;
 603                                 packet_len1 -= len;
 604                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 605                         }
 606
 607                         if (packet_ptr2 != NULL) {
 608                                 /* copy second part of packet, if exists */
 609                                 /* fill the rest of currently last mbuf */
 610                                 len = rte_pktmbuf_tailroom(m);
 611                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 612                                         packet_ptr2, len);
 613                                 packet_len2 -= len;
 614                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 615
 616                                 while (packet_len2 > 0) {
 617                                         /* fill new mbufs */
 618                                         m->next = rte_pktmbuf_alloc(
 619                                                         sze_q->mb_pool);
 620
 621                                         if (unlikely(m->next == NULL)) {
 622                                                 rte_pktmbuf_free(mbuf);
 623                                                 /*
 624                                                  * Restore items from sze
 625                                                  * structure to state after
 626                                                  * unlocking (eventually
 627                                                  * locking).
 628                                                  */
 629                                                 sze->ct_rx_lck =
 630                                                         ct_rx_lck_backup;
 631                                                 sze->ct_rx_rem_bytes =
 632                                                         ct_rx_rem_bytes_backup;
 633                                                 sze->ct_rx_cur_ptr =
 634                                                         ct_rx_cur_ptr_backup;
 635                                                 goto finish;
 636                                         }
 637
 638                                         m = m->next;
 639
 640                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 641                                                 packet_len2);
 642                                         rte_memcpy(
 643                                                 rte_pktmbuf_append(mbuf, len),
 644                                                 packet_ptr2, len);
 645
 646                                         (mbuf->nb_segs)++;
 647                                         packet_len2 -= len;
 648                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 649                                                 len;
 650                                 }
 651                         }
 652                 }
 653                 mbuf->pkt_len = packet_size;
 654                 mbuf->port = sze_q->in_port;
 655                 bufs[num_rx] = mbuf;
 656                 num_rx++;
 657                 num_bytes += packet_size;
 658         }
 659
 660 finish:
 661         sze_q->rx_pkts += num_rx;
 662         sze_q->rx_bytes += num_bytes;
 663         return num_rx;
 664 }
 665
 666 static uint16_t
 667 eth_szedata2_tx(void *queue,
 668                 struct rte_mbuf **bufs,
 669                 uint16_t nb_pkts)
 670 {
 671         struct rte_mbuf *mbuf;
 672         struct szedata2_tx_queue *sze_q = queue;
 673         uint16_t num_tx = 0;
 674         uint64_t num_bytes = 0;
 675
 676         const struct szedata_lock *lck;
 677         uint32_t lock_size;
 678         uint32_t lock_size2;
 679         void *dst;
 680         uint32_t pkt_len;
 681         uint32_t hwpkt_len;
 682         uint32_t unlock_size;
 683         uint32_t rem_len;
 684         uint8_t mbuf_segs;
 685         uint16_t pkt_left = nb_pkts;
 686
 687         if (sze_q->sze == NULL || nb_pkts == 0)
 688                 return 0;
 689
 690         while (pkt_left > 0) {
 691                 unlock_size = 0;
 692                 lck = szedata_tx_lock_data(sze_q->sze,
 693                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 694                         sze_q->tx_channel);
 695                 if (lck == NULL)
 696                         continue;
 697
 698                 dst = lck->start;
 699                 lock_size = lck->len;
 700                 lock_size2 = lck->next ? lck->next->len : 0;
 701
 702 next_packet:
 703                 mbuf = bufs[nb_pkts - pkt_left];
 704
 705                 pkt_len = mbuf->pkt_len;
 706                 mbuf_segs = mbuf->nb_segs;
 707
 708                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 709                         RTE_SZE2_ALIGN8(pkt_len);
 710
 711                 if (lock_size + lock_size2 < hwpkt_len) {
 712                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 713                         continue;
 714                 }
 715
 716                 num_bytes += pkt_len;
 717
 718                 if (lock_size > hwpkt_len) {
 719                         void *tmp_dst;
 720
 721                         rem_len = 0;
 722
 723                         /* write packet length at first 2 bytes in 8B header */
 724                         *((uint16_t *)dst) = htole16(
 725                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 726                                         pkt_len);
 727                         *(((uint16_t *)dst) + 1) = htole16(0);
 728
 729                         /* copy packet from mbuf */
 730                         tmp_dst = ((uint8_t *)(dst)) +
 731                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 732                         if (mbuf_segs == 1) {
 733                                 /*
 734                                  * non-scattered packet,
 735                                  * transmit from one mbuf
 736                                  */
 737                                 rte_memcpy(tmp_dst,
 738                                         rte_pktmbuf_mtod(mbuf, const void *),
 739                                         pkt_len);
 740                         } else {
 741                                 /* scattered packet, transmit from more mbufs */
 742                                 struct rte_mbuf *m = mbuf;
 743                                 while (m) {
 744                                         rte_memcpy(tmp_dst,
 745                                                 rte_pktmbuf_mtod(m,
 746                                                 const void *),
 747                                                 m->data_len);
 748                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 749                                                 m->data_len;
 750                                         m = m->next;
 751                                 }
 752                         }
 753
 754
 755                         dst = ((uint8_t *)dst) + hwpkt_len;
 756                         unlock_size += hwpkt_len;
 757                         lock_size -= hwpkt_len;
 758
 759                         rte_pktmbuf_free(mbuf);
 760                         num_tx++;
 761                         pkt_left--;
 762                         if (pkt_left == 0) {
 763                                 szedata_tx_unlock_data(sze_q->sze, lck,
 764                                         unlock_size);
 765                                 break;
 766                         }
 767                         goto next_packet;
 768                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 769                         void *tmp_dst;
 770                         uint16_t write_len;
 771
 772                         /* write packet length at first 2 bytes in 8B header */
 773                         *((uint16_t *)dst) =
 774                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 775                                         pkt_len);
 776                         *(((uint16_t *)dst) + 1) = htole16(0);
 777
 778                         /*
 779                          * If the raw packet (pkt_len) is smaller than lock_size
 780                          * get the correct length for memcpy
 781                          */
 782                         write_len =
 783                                 pkt_len < lock_size -
 784                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 785                                 pkt_len :
 786                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 787
 788                         rem_len = hwpkt_len - lock_size;
 789
 790                         tmp_dst = ((uint8_t *)(dst)) +
 791                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 792                         if (mbuf_segs == 1) {
 793                                 /*
 794                                  * non-scattered packet,
 795                                  * transmit from one mbuf
 796                                  */
 797                                 /* copy part of packet to first area */
 798                                 rte_memcpy(tmp_dst,
 799                                         rte_pktmbuf_mtod(mbuf, const void *),
 800                                         write_len);
 801
 802                                 if (lck->next)
 803                                         dst = lck->next->start;
 804
 805                                 /* copy part of packet to second area */
 806                                 rte_memcpy(dst,
 807                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 808                                                         const uint8_t *) +
 809                                         write_len), pkt_len - write_len);
 810                         } else {
 811                                 /* scattered packet, transmit from more mbufs */
 812                                 struct rte_mbuf *m = mbuf;
 813                                 uint16_t written = 0;
 814                                 uint16_t to_write = 0;
 815                                 bool new_mbuf = true;
 816                                 uint16_t write_off = 0;
 817
 818                                 /* copy part of packet to first area */
 819                                 while (m && written < write_len) {
 820                                         to_write = RTE_MIN(m->data_len,
 821                                                         write_len - written);
 822                                         rte_memcpy(tmp_dst,
 823                                                 rte_pktmbuf_mtod(m,
 824                                                         const void *),
 825                                                 to_write);
 826
 827                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 828                                                 to_write;
 829                                         if (m->data_len <= write_len -
 830                                                         written) {
 831                                                 m = m->next;
 832                                                 new_mbuf = true;
 833                                         } else {
 834                                                 new_mbuf = false;
 835                                         }
 836                                         written += to_write;
 837                                 }
 838
 839                                 if (lck->next)
 840                                         dst = lck->next->start;
 841
 842                                 tmp_dst = dst;
 843                                 written = 0;
 844                                 write_off = new_mbuf ? 0 : to_write;
 845
 846                                 /* copy part of packet to second area */
 847                                 while (m && written < pkt_len - write_len) {
 848                                         rte_memcpy(tmp_dst, (const void *)
 849                                                 (rte_pktmbuf_mtod(m,
 850                                                 uint8_t *) + write_off),
 851                                                 m->data_len - write_off);
 852
 853                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 854                                                 (m->data_len - write_off);
 855                                         written += m->data_len - write_off;
 856                                         m = m->next;
 857                                         write_off = 0;
 858                                 }
 859                         }
 860
 861                         dst = ((uint8_t *)dst) + rem_len;
 862                         unlock_size += hwpkt_len;
 863                         lock_size = lock_size2 - rem_len;
 864                         lock_size2 = 0;
 865
 866                         rte_pktmbuf_free(mbuf);
 867                         num_tx++;
 868                 }
 869
 870                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 871                 pkt_left--;
 872         }
 873
 874         sze_q->tx_pkts += num_tx;
 875         sze_q->err_pkts += nb_pkts - num_tx;
 876         sze_q->tx_bytes += num_bytes;
 877         return num_tx;
 878 }
 879
 880 static int
 881 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
 882 {
 883         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 884         int ret;
 885         struct pmd_internals *internals = (struct pmd_internals *)
 886                 dev->data->dev_private;
 887
 888         if (rxq->sze == NULL) {
 889                 uint32_t rx = 1 << rxq->rx_channel;
 890                 uint32_t tx = 0;
 891                 rxq->sze = szedata_open(internals->sze_dev);
 892                 if (rxq->sze == NULL)
 893                         return -EINVAL;
 894                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
 895                 if (ret != 0 || rx == 0)
 896                         goto err;
 897         }
 898
 899         ret = szedata_start(rxq->sze);
 900         if (ret != 0)
 901                 goto err;
 902         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 903         return 0;
 904
 905 err:
 906         szedata_close(rxq->sze);
 907         rxq->sze = NULL;
 908         return -EINVAL;
 909 }
 910
 911 static int
 912 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
 913 {
 914         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 915
 916         if (rxq->sze != NULL) {
 917                 szedata_close(rxq->sze);
 918                 rxq->sze = NULL;
 919         }
 920
 921         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 922         return 0;
 923 }
 924
 925 static int
 926 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
 927 {
 928         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 929         int ret;
 930         struct pmd_internals *internals = (struct pmd_internals *)
 931                 dev->data->dev_private;
 932
 933         if (txq->sze == NULL) {
 934                 uint32_t rx = 0;
 935                 uint32_t tx = 1 << txq->tx_channel;
 936                 txq->sze = szedata_open(internals->sze_dev);
 937                 if (txq->sze == NULL)
 938                         return -EINVAL;
 939                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
 940                 if (ret != 0 || tx == 0)
 941                         goto err;
 942         }
 943
 944         ret = szedata_start(txq->sze);
 945         if (ret != 0)
 946                 goto err;
 947         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 948         return 0;
 949
 950 err:
 951         szedata_close(txq->sze);
 952         txq->sze = NULL;
 953         return -EINVAL;
 954 }
 955
 956 static int
 957 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
 958 {
 959         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 960
 961         if (txq->sze != NULL) {
 962                 szedata_close(txq->sze);
 963                 txq->sze = NULL;
 964         }
 965
 966         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 967         return 0;
 968 }
 969
 970 static int
 971 eth_dev_start(struct rte_eth_dev *dev)
 972 {
 973         int ret;
 974         uint16_t i;
 975         uint16_t nb_rx = dev->data->nb_rx_queues;
 976         uint16_t nb_tx = dev->data->nb_tx_queues;
 977
 978         for (i = 0; i < nb_rx; i++) {
 979                 ret = eth_rx_queue_start(dev, i);
 980                 if (ret != 0)
 981                         goto err_rx;
 982         }
 983
 984         for (i = 0; i < nb_tx; i++) {
 985                 ret = eth_tx_queue_start(dev, i);
 986                 if (ret != 0)
 987                         goto err_tx;
 988         }
 989
 990         return 0;
 991
 992 err_tx:
 993         for (i = 0; i < nb_tx; i++)
 994                 eth_tx_queue_stop(dev, i);
 995 err_rx:
 996         for (i = 0; i < nb_rx; i++)
 997                 eth_rx_queue_stop(dev, i);
 998         return ret;
 999 }
1000
1001 static void
1002 eth_dev_stop(struct rte_eth_dev *dev)
1003 {
1004         uint16_t i;
1005         uint16_t nb_rx = dev->data->nb_rx_queues;
1006         uint16_t nb_tx = dev->data->nb_tx_queues;
1007
1008         for (i = 0; i < nb_tx; i++)
1009                 eth_tx_queue_stop(dev, i);
1010
1011         for (i = 0; i < nb_rx; i++)
1012                 eth_rx_queue_stop(dev, i);
1013 }
1014
1015 static int
1016 eth_dev_configure(struct rte_eth_dev *dev)
1017 {
1018         struct rte_eth_dev_data *data = dev->data;
1019         if (data->dev_conf.rxmode.enable_scatter == 1) {
1020                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1021                 data->scattered_rx = 1;
1022         } else {
1023                 dev->rx_pkt_burst = eth_szedata2_rx;
1024                 data->scattered_rx = 0;
1025         }
1026         return 0;
1027 }
1028
1029 static void
1030 eth_dev_info(struct rte_eth_dev *dev,
1031                 struct rte_eth_dev_info *dev_info)
1032 {
1033         struct pmd_internals *internals = dev->data->dev_private;
1034         dev_info->if_index = 0;
1035         dev_info->max_mac_addrs = 1;
1036         dev_info->max_rx_pktlen = (uint32_t)-1;
1037         dev_info->max_rx_queues = internals->max_rx_queues;
1038         dev_info->max_tx_queues = internals->max_tx_queues;
1039         dev_info->min_rx_bufsize = 0;
1040         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1041 }
1042
1043 static void
1044 eth_stats_get(struct rte_eth_dev *dev,
1045                 struct rte_eth_stats *stats)
1046 {
1047         uint16_t i;
1048         uint16_t nb_rx = dev->data->nb_rx_queues;
1049         uint16_t nb_tx = dev->data->nb_tx_queues;
1050         uint64_t rx_total = 0;
1051         uint64_t tx_total = 0;
1052         uint64_t tx_err_total = 0;
1053         uint64_t rx_total_bytes = 0;
1054         uint64_t tx_total_bytes = 0;
1055         const struct pmd_internals *internals = dev->data->dev_private;
1056
1057         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_rx; i++) {
1058                 stats->q_ipackets[i] = internals->rx_queue[i].rx_pkts;
1059                 stats->q_ibytes[i] = internals->rx_queue[i].rx_bytes;
1060                 rx_total += stats->q_ipackets[i];
1061                 rx_total_bytes += stats->q_ibytes[i];
1062         }
1063
1064         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_tx; i++) {
1065                 stats->q_opackets[i] = internals->tx_queue[i].tx_pkts;
1066                 stats->q_obytes[i] = internals->tx_queue[i].tx_bytes;
1067                 stats->q_errors[i] = internals->tx_queue[i].err_pkts;
1068                 tx_total += stats->q_opackets[i];
1069                 tx_total_bytes += stats->q_obytes[i];
1070                 tx_err_total += stats->q_errors[i];
1071         }
1072
1073         stats->ipackets = rx_total;
1074         stats->opackets = tx_total;
1075         stats->ibytes = rx_total_bytes;
1076         stats->obytes = tx_total_bytes;
1077         stats->oerrors = tx_err_total;
1078 }
1079
1080 static void
1081 eth_stats_reset(struct rte_eth_dev *dev)
1082 {
1083         uint16_t i;
1084         uint16_t nb_rx = dev->data->nb_rx_queues;
1085         uint16_t nb_tx = dev->data->nb_tx_queues;
1086         struct pmd_internals *internals = dev->data->dev_private;
1087
1088         for (i = 0; i < nb_rx; i++) {
1089                 internals->rx_queue[i].rx_pkts = 0;
1090                 internals->rx_queue[i].rx_bytes = 0;
1091                 internals->rx_queue[i].err_pkts = 0;
1092         }
1093         for (i = 0; i < nb_tx; i++) {
1094                 internals->tx_queue[i].tx_pkts = 0;
1095                 internals->tx_queue[i].tx_bytes = 0;
1096                 internals->tx_queue[i].err_pkts = 0;
1097         }
1098 }
1099
1100 static void
1101 eth_rx_queue_release(void *q)
1102 {
1103         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1104         if (rxq->sze != NULL) {
1105                 szedata_close(rxq->sze);
1106                 rxq->sze = NULL;
1107         }
1108 }
1109
1110 static void
1111 eth_tx_queue_release(void *q)
1112 {
1113         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1114         if (txq->sze != NULL) {
1115                 szedata_close(txq->sze);
1116                 txq->sze = NULL;
1117         }
1118 }
1119
1120 static void
1121 eth_dev_close(struct rte_eth_dev *dev)
1122 {
1123         uint16_t i;
1124         uint16_t nb_rx = dev->data->nb_rx_queues;
1125         uint16_t nb_tx = dev->data->nb_tx_queues;
1126
1127         eth_dev_stop(dev);
1128
1129         for (i = 0; i < nb_rx; i++) {
1130                 eth_rx_queue_release(dev->data->rx_queues[i]);
1131                 dev->data->rx_queues[i] = NULL;
1132         }
1133         dev->data->nb_rx_queues = 0;
1134         for (i = 0; i < nb_tx; i++) {
1135                 eth_tx_queue_release(dev->data->tx_queues[i]);
1136                 dev->data->tx_queues[i] = NULL;
1137         }
1138         dev->data->nb_tx_queues = 0;
1139 }
1140
1141 static int
1142 eth_link_update(struct rte_eth_dev *dev,
1143                 int wait_to_complete __rte_unused)
1144 {
1145         struct rte_eth_link link;
1146         struct rte_eth_link *link_ptr = &link;
1147         struct rte_eth_link *dev_link = &dev->data->dev_link;
1148         struct pmd_internals *internals = (struct pmd_internals *)
1149                 dev->data->dev_private;
1150         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1151                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1152                         volatile struct szedata2_cgmii_ibuf *);
1153
1154         switch (cgmii_link_speed(ibuf)) {
1155         case SZEDATA2_LINK_SPEED_10G:
1156                 link.link_speed = ETH_SPEED_NUM_10G;
1157                 break;
1158         case SZEDATA2_LINK_SPEED_40G:
1159                 link.link_speed = ETH_SPEED_NUM_40G;
1160                 break;
1161         case SZEDATA2_LINK_SPEED_100G:
1162                 link.link_speed = ETH_SPEED_NUM_100G;
1163                 break;
1164         default:
1165                 link.link_speed = ETH_SPEED_NUM_10G;
1166                 break;
1167         }
1168
1169         /* szedata2 uses only full duplex */
1170         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1171
1172         link.link_status = (cgmii_ibuf_is_enabled(ibuf) &&
1173                         cgmii_ibuf_is_link_up(ibuf)) ? ETH_LINK_UP : ETH_LINK_DOWN;
1174
1175         link.link_autoneg = ETH_LINK_SPEED_FIXED;
1176
1177         rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link,
1178                         *(uint64_t *)link_ptr);
1179
1180         return 0;
1181 }
1182
1183 static int
1184 eth_dev_set_link_up(struct rte_eth_dev *dev)
1185 {
1186         struct pmd_internals *internals = (struct pmd_internals *)
1187                 dev->data->dev_private;
1188         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1189                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1190                         volatile struct szedata2_cgmii_ibuf *);
1191         volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1192                         internals->pci_rsc, SZEDATA2_CGMII_OBUF_BASE_OFF,
1193                         volatile struct szedata2_cgmii_obuf *);
1194
1195         cgmii_ibuf_enable(ibuf);
1196         cgmii_obuf_enable(obuf);
1197         return 0;
1198 }
1199
1200 static int
1201 eth_dev_set_link_down(struct rte_eth_dev *dev)
1202 {
1203         struct pmd_internals *internals = (struct pmd_internals *)
1204                 dev->data->dev_private;
1205         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1206                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1207                         volatile struct szedata2_cgmii_ibuf *);
1208         volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1209                         internals->pci_rsc, SZEDATA2_CGMII_OBUF_BASE_OFF,
1210                         volatile struct szedata2_cgmii_obuf *);
1211
1212         cgmii_ibuf_disable(ibuf);
1213         cgmii_obuf_disable(obuf);
1214         return 0;
1215 }
1216
1217 static int
1218 eth_rx_queue_setup(struct rte_eth_dev *dev,
1219                 uint16_t rx_queue_id,
1220                 uint16_t nb_rx_desc __rte_unused,
1221                 unsigned int socket_id __rte_unused,
1222                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1223                 struct rte_mempool *mb_pool)
1224 {
1225         struct pmd_internals *internals = dev->data->dev_private;
1226         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1227         int ret;
1228         uint32_t rx = 1 << rx_queue_id;
1229         uint32_t tx = 0;
1230
1231         rxq->sze = szedata_open(internals->sze_dev);
1232         if (rxq->sze == NULL)
1233                 return -EINVAL;
1234         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1235         if (ret != 0 || rx == 0) {
1236                 szedata_close(rxq->sze);
1237                 rxq->sze = NULL;
1238                 return -EINVAL;
1239         }
1240         rxq->rx_channel = rx_queue_id;
1241         rxq->in_port = dev->data->port_id;
1242         rxq->mb_pool = mb_pool;
1243         rxq->rx_pkts = 0;
1244         rxq->rx_bytes = 0;
1245         rxq->err_pkts = 0;
1246
1247         dev->data->rx_queues[rx_queue_id] = rxq;
1248         return 0;
1249 }
1250
1251 static int
1252 eth_tx_queue_setup(struct rte_eth_dev *dev,
1253                 uint16_t tx_queue_id,
1254                 uint16_t nb_tx_desc __rte_unused,
1255                 unsigned int socket_id __rte_unused,
1256                 const struct rte_eth_txconf *tx_conf __rte_unused)
1257 {
1258         struct pmd_internals *internals = dev->data->dev_private;
1259         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1260         int ret;
1261         uint32_t rx = 0;
1262         uint32_t tx = 1 << tx_queue_id;
1263
1264         txq->sze = szedata_open(internals->sze_dev);
1265         if (txq->sze == NULL)
1266                 return -EINVAL;
1267         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1268         if (ret != 0 || tx == 0) {
1269                 szedata_close(txq->sze);
1270                 txq->sze = NULL;
1271                 return -EINVAL;
1272         }
1273         txq->tx_channel = tx_queue_id;
1274         txq->tx_pkts = 0;
1275         txq->tx_bytes = 0;
1276         txq->err_pkts = 0;
1277
1278         dev->data->tx_queues[tx_queue_id] = txq;
1279         return 0;
1280 }
1281
1282 static void
1283 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1284                 struct ether_addr *mac_addr __rte_unused)
1285 {
1286 }
1287
1288 static void
1289 eth_promiscuous_enable(struct rte_eth_dev *dev)
1290 {
1291         struct pmd_internals *internals = (struct pmd_internals *)
1292                 dev->data->dev_private;
1293         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1294                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1295                         volatile struct szedata2_cgmii_ibuf *);
1296         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_PROMISC);
1297 }
1298
1299 static void
1300 eth_promiscuous_disable(struct rte_eth_dev *dev)
1301 {
1302         struct pmd_internals *internals = (struct pmd_internals *)
1303                 dev->data->dev_private;
1304         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1305                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1306                         volatile struct szedata2_cgmii_ibuf *);
1307         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1308 }
1309
1310 static void
1311 eth_allmulticast_enable(struct rte_eth_dev *dev)
1312 {
1313         struct pmd_internals *internals = (struct pmd_internals *)
1314                 dev->data->dev_private;
1315         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1316                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1317                         volatile struct szedata2_cgmii_ibuf *);
1318         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1319 }
1320
1321 static void
1322 eth_allmulticast_disable(struct rte_eth_dev *dev)
1323 {
1324         struct pmd_internals *internals = (struct pmd_internals *)
1325                 dev->data->dev_private;
1326         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1327                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1328                         volatile struct szedata2_cgmii_ibuf *);
1329         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1330 }
1331
1332 static const struct eth_dev_ops ops = {
1333         .dev_start          = eth_dev_start,
1334         .dev_stop           = eth_dev_stop,
1335         .dev_set_link_up    = eth_dev_set_link_up,
1336         .dev_set_link_down  = eth_dev_set_link_down,
1337         .dev_close          = eth_dev_close,
1338         .dev_configure      = eth_dev_configure,
1339         .dev_infos_get      = eth_dev_info,
1340         .promiscuous_enable   = eth_promiscuous_enable,
1341         .promiscuous_disable  = eth_promiscuous_disable,
1342         .allmulticast_enable  = eth_allmulticast_enable,
1343         .allmulticast_disable = eth_allmulticast_disable,
1344         .rx_queue_start     = eth_rx_queue_start,
1345         .rx_queue_stop      = eth_rx_queue_stop,
1346         .tx_queue_start     = eth_tx_queue_start,
1347         .tx_queue_stop      = eth_tx_queue_stop,
1348         .rx_queue_setup     = eth_rx_queue_setup,
1349         .tx_queue_setup     = eth_tx_queue_setup,
1350         .rx_queue_release   = eth_rx_queue_release,
1351         .tx_queue_release   = eth_tx_queue_release,
1352         .link_update        = eth_link_update,
1353         .stats_get          = eth_stats_get,
1354         .stats_reset        = eth_stats_reset,
1355         .mac_addr_set       = eth_mac_addr_set,
1356 };
1357
1358 /*
1359  * This function goes through sysfs and looks for an index of szedata2
1360  * device file (/dev/szedataIIX, where X is the index).
1361  *
1362  * @return
1363  *           0 on success
1364  *          -1 on error
1365  */
1366 static int
1367 get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index)
1368 {
1369         DIR *dir;
1370         struct dirent *entry;
1371         int ret;
1372         uint32_t tmp_index;
1373         FILE *fd;
1374         char pcislot_path[PATH_MAX];
1375         uint32_t domain;
1376         uint32_t bus;
1377         uint32_t devid;
1378         uint32_t function;
1379
1380         dir = opendir("/sys/class/combo");
1381         if (dir == NULL)
1382                 return -1;
1383
1384         /*
1385          * Iterate through all combosixX directories.
1386          * When the value in /sys/class/combo/combosixX/device/pcislot
1387          * file is the location of the ethernet device dev, "X" is the
1388          * index of the device.
1389          */
1390         while ((entry = readdir(dir)) != NULL) {
1391                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1392                 if (ret != 1)
1393                         continue;
1394
1395                 snprintf(pcislot_path, PATH_MAX,
1396                         "/sys/class/combo/combosix%u/device/pcislot",
1397                         tmp_index);
1398
1399                 fd = fopen(pcislot_path, "r");
1400                 if (fd == NULL)
1401                         continue;
1402
1403                 ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8,
1404                                 &domain, &bus, &devid, &function);
1405                 fclose(fd);
1406                 if (ret != 4)
1407                         continue;
1408
1409                 if (pcislot_addr->domain == domain &&
1410                                 pcislot_addr->bus == bus &&
1411                                 pcislot_addr->devid == devid &&
1412                                 pcislot_addr->function == function) {
1413                         *index = tmp_index;
1414                         closedir(dir);
1415                         return 0;
1416                 }
1417         }
1418
1419         closedir(dir);
1420         return -1;
1421 }
1422
1423 static int
1424 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1425 {
1426         struct rte_eth_dev_data *data = dev->data;
1427         struct pmd_internals *internals = (struct pmd_internals *)
1428                 data->dev_private;
1429         struct szedata *szedata_temp;
1430         int ret;
1431         uint32_t szedata2_index;
1432         struct rte_pci_device *pci_dev = dev->pci_dev;
1433         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1434         struct rte_mem_resource *pci_rsc =
1435                 &pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1436         char rsc_filename[PATH_MAX];
1437         void *pci_resource_ptr = NULL;
1438         int fd;
1439
1440         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1441                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1442                         pci_addr->function);
1443
1444         /* Get index of szedata2 device file and create path to device file */
1445         ret = get_szedata2_index(pci_addr, &szedata2_index);
1446         if (ret != 0) {
1447                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1448                 return -ENODEV;
1449         }
1450         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1451                         szedata2_index);
1452
1453         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1454
1455         /*
1456          * Get number of available DMA RX and TX channels, which is maximum
1457          * number of queues that can be created and store it in private device
1458          * data structure.
1459          */
1460         szedata_temp = szedata_open(internals->sze_dev);
1461         if (szedata_temp == NULL) {
1462                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1463                                 internals->sze_dev);
1464                 return -EINVAL;
1465         }
1466         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1467                         SZE2_DIR_RX);
1468         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1469                         SZE2_DIR_TX);
1470         szedata_close(szedata_temp);
1471
1472         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1473                         internals->max_rx_queues, internals->max_tx_queues);
1474
1475         /* Set rx, tx burst functions */
1476         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1477                 data->scattered_rx == 1) {
1478                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1479                 data->scattered_rx = 1;
1480         } else {
1481                 dev->rx_pkt_burst = eth_szedata2_rx;
1482                 data->scattered_rx = 0;
1483         }
1484         dev->tx_pkt_burst = eth_szedata2_tx;
1485
1486         /* Set function callbacks for Ethernet API */
1487         dev->dev_ops = &ops;
1488
1489         rte_eth_copy_pci_info(dev, pci_dev);
1490
1491         /* mmap pci resource0 file to rte_mem_resource structure */
1492         if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1493                         0) {
1494                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1495                                 PCI_RESOURCE_NUMBER);
1496                 return -EINVAL;
1497         }
1498         snprintf(rsc_filename, PATH_MAX,
1499                 "%s/" PCI_PRI_FMT "/resource%u", pci_get_sysfs_path(),
1500                 pci_addr->domain, pci_addr->bus,
1501                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1502         fd = open(rsc_filename, O_RDWR);
1503         if (fd < 0) {
1504                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1505                 return -EINVAL;
1506         }
1507
1508         pci_resource_ptr = mmap(0,
1509                         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1510                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1511         close(fd);
1512         if (pci_resource_ptr == NULL) {
1513                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1514                                 rsc_filename, fd);
1515                 return -EINVAL;
1516         }
1517         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr;
1518         internals->pci_rsc = pci_rsc;
1519
1520         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1521                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1522                         (unsigned long long)pci_rsc->phys_addr,
1523                         (unsigned long long)pci_rsc->len,
1524                         (unsigned long long)pci_rsc->addr);
1525
1526         /* Get link state */
1527         eth_link_update(dev, 0);
1528
1529         /* Allocate space for one mac address */
1530         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1531                         RTE_CACHE_LINE_SIZE);
1532         if (data->mac_addrs == NULL) {
1533                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1534                 munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1535                        pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1536                 return -EINVAL;
1537         }
1538
1539         ether_addr_copy(&eth_addr, data->mac_addrs);
1540
1541         /* At initial state COMBO card is in promiscuous mode so disable it */
1542         eth_promiscuous_disable(dev);
1543
1544         RTE_LOG(INFO, PMD, "szedata2 device ("
1545                         PCI_PRI_FMT ") successfully initialized\n",
1546                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1547                         pci_addr->function);
1548
1549         return 0;
1550 }
1551
1552 static int
1553 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1554 {
1555         struct rte_pci_device *pci_dev = dev->pci_dev;
1556         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1557
1558         rte_free(dev->data->mac_addrs);
1559         dev->data->mac_addrs = NULL;
1560         munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1561                pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1562
1563         RTE_LOG(INFO, PMD, "szedata2 device ("
1564                         PCI_PRI_FMT ") successfully uninitialized\n",
1565                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1566                         pci_addr->function);
1567
1568         return 0;
1569 }
1570
1571 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1572         {
1573                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1574                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1575         },
1576         {
1577                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1578                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1579         },
1580         {
1581                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1582                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1583         },
1584         {
1585                 .vendor_id = 0,
1586         }
1587 };
1588
1589 static struct eth_driver szedata2_eth_driver = {
1590         .pci_drv = {
1591                 .id_table = rte_szedata2_pci_id_table,
1592                 .probe = rte_eth_dev_pci_probe,
1593                 .remove = rte_eth_dev_pci_remove,
1594         },
1595         .eth_dev_init     = rte_szedata2_eth_dev_init,
1596         .eth_dev_uninit   = rte_szedata2_eth_dev_uninit,
1597         .dev_private_size = sizeof(struct pmd_internals),
1598 };
1599
1600 RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver.pci_drv);
1601 RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1602 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
1603         "* combo6core & combov3 & szedata2 & szedata2_cv3");