drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38
  39 #include <libsze2.h>
  40
  41 #include <rte_mbuf.h>
  42 #include <rte_ethdev.h>
  43 #include <rte_malloc.h>
  44 #include <rte_memcpy.h>
  45 #include <rte_kvargs.h>
  46 #include <rte_dev.h>
  47
  48 #include "rte_eth_szedata2.h"
  49
  50 #define RTE_ETH_SZEDATA2_DEV_PATH_ARG "dev_path"
  51 #define RTE_ETH_SZEDATA2_RX_IFACES_ARG "rx_ifaces"
  52 #define RTE_ETH_SZEDATA2_TX_IFACES_ARG "tx_ifaces"
  53
  54 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  55 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  56 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  57
  58 /**
  59  * size of szedata2_packet header with alignment
  60  */
  61 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  62
  63 struct szedata2_rx_queue {
  64         struct szedata *sze;
  65         uint8_t rx_channel;
  66         uint8_t in_port;
  67         struct rte_mempool *mb_pool;
  68         volatile uint64_t rx_pkts;
  69         volatile uint64_t rx_bytes;
  70         volatile uint64_t err_pkts;
  71 };
  72
  73 struct szedata2_tx_queue {
  74         struct szedata *sze;
  75         uint8_t tx_channel;
  76         volatile uint64_t tx_pkts;
  77         volatile uint64_t err_pkts;
  78         volatile uint64_t tx_bytes;
  79 };
  80
  81 struct rxtx_szedata2 {
  82         uint32_t num_of_rx;
  83         uint32_t num_of_tx;
  84         uint32_t sze_rx_mask_req;
  85         uint32_t sze_tx_mask_req;
  86         char *sze_dev;
  87 };
  88
  89 struct pmd_internals {
  90         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  91         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  92         unsigned nb_rx_queues;
  93         unsigned nb_tx_queues;
  94         uint32_t num_of_rx;
  95         uint32_t num_of_tx;
  96         uint32_t sze_rx_req;
  97         uint32_t sze_tx_req;
  98         int if_index;
  99         char *sze_dev;
 100 };
 101
 102 static const char *valid_arguments[] = {
 103         RTE_ETH_SZEDATA2_DEV_PATH_ARG,
 104         RTE_ETH_SZEDATA2_RX_IFACES_ARG,
 105         RTE_ETH_SZEDATA2_TX_IFACES_ARG,
 106         NULL
 107 };
 108
 109 static struct ether_addr eth_addr = {
 110         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
 111 };
 112 static const char *drivername = "SZEdata2 PMD";
 113 static struct rte_eth_link pmd_link = {
 114                 .link_speed = ETH_LINK_SPEED_10G,
 115                 .link_duplex = ETH_LINK_FULL_DUPLEX,
 116                 .link_status = 0
 117 };
 118
 119
 120 static uint32_t
 121 count_ones(uint32_t num)
 122 {
 123         num = num - ((num >> 1) & 0x55555555); /* reuse input as temporary */
 124         num = (num & 0x33333333) + ((num >> 2) & 0x33333333);        /* temp */
 125         return (((num + (num >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24; /* count */
 126 }
 127
 128 static uint16_t
 129 eth_szedata2_rx(void *queue,
 130                 struct rte_mbuf **bufs,
 131                 uint16_t nb_pkts)
 132 {
 133         unsigned int i;
 134         struct rte_mbuf *mbuf;
 135         struct szedata2_rx_queue *sze_q = queue;
 136         struct rte_pktmbuf_pool_private *mbp_priv;
 137         uint16_t num_rx = 0;
 138         uint16_t buf_size;
 139         uint16_t sg_size;
 140         uint16_t hw_size;
 141         uint16_t packet_size;
 142         uint64_t num_bytes = 0;
 143         struct szedata *sze = sze_q->sze;
 144         uint8_t *header_ptr = NULL; /* header of packet */
 145         uint8_t *packet_ptr1 = NULL;
 146         uint8_t *packet_ptr2 = NULL;
 147         uint16_t packet_len1 = 0;
 148         uint16_t packet_len2 = 0;
 149         uint16_t hw_data_align;
 150
 151         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 152                 return 0;
 153
 154         /*
 155          * Reads the given number of packets from szedata2 channel given
 156          * by queue and copies the packet data into a newly allocated mbuf
 157          * to return.
 158          */
 159         for (i = 0; i < nb_pkts; i++) {
 160                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 161
 162                 if (unlikely(mbuf == NULL))
 163                         break;
 164
 165                 /* get the next sze packet */
 166                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 167                                 sze->ct_rx_lck->next == NULL) {
 168                         /* unlock old data */
 169                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 170                         sze->ct_rx_lck_orig = NULL;
 171                         sze->ct_rx_lck = NULL;
 172                 }
 173
 174                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 175                         /* nothing to read, lock new data */
 176                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 177                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 178
 179                         if (sze->ct_rx_lck == NULL) {
 180                                 /* nothing to lock */
 181                                 rte_pktmbuf_free(mbuf);
 182                                 break;
 183                         }
 184
 185                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 186                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 187
 188                         if (!sze->ct_rx_rem_bytes) {
 189                                 rte_pktmbuf_free(mbuf);
 190                                 break;
 191                         }
 192                 }
 193
 194                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 195                         /*
 196                          * cut in header
 197                          * copy parts of header to merge buffer
 198                          */
 199                         if (sze->ct_rx_lck->next == NULL) {
 200                                 rte_pktmbuf_free(mbuf);
 201                                 break;
 202                         }
 203
 204                         /* copy first part of header */
 205                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 206                                         sze->ct_rx_rem_bytes);
 207
 208                         /* copy second part of header */
 209                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 210                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 211                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 212                                 sze->ct_rx_cur_ptr,
 213                                 RTE_SZE2_PACKET_HEADER_SIZE -
 214                                 sze->ct_rx_rem_bytes);
 215
 216                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 217                                 sze->ct_rx_rem_bytes;
 218                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 219                                 RTE_SZE2_PACKET_HEADER_SIZE +
 220                                 sze->ct_rx_rem_bytes;
 221
 222                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 223                 } else {
 224                         /* not cut */
 225                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 226                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 227                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 228                 }
 229
 230                 sg_size = le16toh(*((uint16_t *)header_ptr));
 231                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 232                 packet_size = sg_size -
 233                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 234
 235
 236                 /* checks if packet all right */
 237                 if (!sg_size)
 238                         errx(5, "Zero segsize");
 239
 240                 /* check sg_size and hwsize */
 241                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 242                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 243                                 "hwsize: %d", sg_size, hw_size);
 244                 }
 245
 246                 hw_data_align =
 247                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 248                         RTE_SZE2_PACKET_HEADER_SIZE;
 249
 250                 if (sze->ct_rx_rem_bytes >=
 251                                 (uint16_t)(sg_size -
 252                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 253                         /* no cut */
 254                         /* one packet ready - go to another */
 255                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 256                         packet_len1 = packet_size;
 257                         packet_ptr2 = NULL;
 258                         packet_len2 = 0;
 259
 260                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 261                                 RTE_SZE2_PACKET_HEADER_SIZE;
 262                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 263                                 RTE_SZE2_PACKET_HEADER_SIZE;
 264                 } else {
 265                         /* cut in data */
 266                         if (sze->ct_rx_lck->next == NULL) {
 267                                 errx(6, "Need \"next\" lock, "
 268                                         "but it is missing: %u",
 269                                         sze->ct_rx_rem_bytes);
 270                         }
 271
 272                         /* skip hw data */
 273                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 274                                 uint16_t rem_size = hw_data_align -
 275                                         sze->ct_rx_rem_bytes;
 276
 277                                 /* MOVE to next lock */
 278                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 279                                 sze->ct_rx_cur_ptr =
 280                                         (void *)(((uint8_t *)
 281                                         (sze->ct_rx_lck->start)) + rem_size);
 282
 283                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 284                                 packet_len1 = packet_size;
 285                                 packet_ptr2 = NULL;
 286                                 packet_len2 = 0;
 287
 288                                 sze->ct_rx_cur_ptr +=
 289                                         RTE_SZE2_ALIGN8(packet_size);
 290                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 291                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 292                         } else {
 293                                 /* get pointer and length from first part */
 294                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 295                                         hw_data_align;
 296                                 packet_len1 = sze->ct_rx_rem_bytes -
 297                                         hw_data_align;
 298
 299                                 /* MOVE to next lock */
 300                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 301                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 302
 303                                 /* get pointer and length from second part */
 304                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 305                                 packet_len2 = packet_size - packet_len1;
 306
 307                                 sze->ct_rx_cur_ptr +=
 308                                         RTE_SZE2_ALIGN8(packet_size) -
 309                                         packet_len1;
 310                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 311                                         (RTE_SZE2_ALIGN8(packet_size) -
 312                                          packet_len1);
 313                         }
 314                 }
 315
 316                 if (unlikely(packet_ptr1 == NULL)) {
 317                         rte_pktmbuf_free(mbuf);
 318                         break;
 319                 }
 320
 321                 /* get the space available for data in the mbuf */
 322                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 323                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 324                                 RTE_PKTMBUF_HEADROOM);
 325
 326                 if (packet_size <= buf_size) {
 327                         /* sze packet will fit in one mbuf, go ahead and copy */
 328                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 329                                         packet_ptr1, packet_len1);
 330                         if (packet_ptr2 != NULL) {
 331                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 332                                         uint8_t *) + packet_len1),
 333                                         packet_ptr2, packet_len2);
 334                         }
 335                         mbuf->data_len = (uint16_t)packet_size;
 336
 337                         mbuf->pkt_len = packet_size;
 338                         mbuf->port = sze_q->in_port;
 339                         bufs[num_rx] = mbuf;
 340                         num_rx++;
 341                         num_bytes += packet_size;
 342                 } else {
 343                         /*
 344                          * sze packet will not fit in one mbuf,
 345                          * scattered mode is not enabled, drop packet
 346                          */
 347                         RTE_LOG(ERR, PMD,
 348                                 "SZE segment %d bytes will not fit in one mbuf "
 349                                 "(%d bytes), scattered mode is not enabled, "
 350                                 "drop packet!!\n",
 351                                 packet_size, buf_size);
 352                         rte_pktmbuf_free(mbuf);
 353                 }
 354         }
 355
 356         sze_q->rx_pkts += num_rx;
 357         sze_q->rx_bytes += num_bytes;
 358         return num_rx;
 359 }
 360
 361 static uint16_t
 362 eth_szedata2_rx_scattered(void *queue,
 363                 struct rte_mbuf **bufs,
 364                 uint16_t nb_pkts)
 365 {
 366         unsigned int i;
 367         struct rte_mbuf *mbuf;
 368         struct szedata2_rx_queue *sze_q = queue;
 369         struct rte_pktmbuf_pool_private *mbp_priv;
 370         uint16_t num_rx = 0;
 371         uint16_t buf_size;
 372         uint16_t sg_size;
 373         uint16_t hw_size;
 374         uint16_t packet_size;
 375         uint64_t num_bytes = 0;
 376         struct szedata *sze = sze_q->sze;
 377         uint8_t *header_ptr = NULL; /* header of packet */
 378         uint8_t *packet_ptr1 = NULL;
 379         uint8_t *packet_ptr2 = NULL;
 380         uint16_t packet_len1 = 0;
 381         uint16_t packet_len2 = 0;
 382         uint16_t hw_data_align;
 383
 384         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 385                 return 0;
 386
 387         /*
 388          * Reads the given number of packets from szedata2 channel given
 389          * by queue and copies the packet data into a newly allocated mbuf
 390          * to return.
 391          */
 392         for (i = 0; i < nb_pkts; i++) {
 393                 const struct szedata_lock *ct_rx_lck_backup;
 394                 unsigned int ct_rx_rem_bytes_backup;
 395                 unsigned char *ct_rx_cur_ptr_backup;
 396
 397                 /* get the next sze packet */
 398                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 399                                 sze->ct_rx_lck->next == NULL) {
 400                         /* unlock old data */
 401                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 402                         sze->ct_rx_lck_orig = NULL;
 403                         sze->ct_rx_lck = NULL;
 404                 }
 405
 406                 /*
 407                  * Store items from sze structure which can be changed
 408                  * before mbuf allocating. Use these items in case of mbuf
 409                  * allocating failure.
 410                  */
 411                 ct_rx_lck_backup = sze->ct_rx_lck;
 412                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 413                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 414
 415                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 416                         /* nothing to read, lock new data */
 417                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 418                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 419
 420                         /*
 421                          * Backup items from sze structure must be updated
 422                          * after locking to contain pointers to new locks.
 423                          */
 424                         ct_rx_lck_backup = sze->ct_rx_lck;
 425                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 426                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 427
 428                         if (sze->ct_rx_lck == NULL)
 429                                 /* nothing to lock */
 430                                 break;
 431
 432                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 433                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 434
 435                         if (!sze->ct_rx_rem_bytes)
 436                                 break;
 437                 }
 438
 439                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 440                         /*
 441                          * cut in header - copy parts of header to merge buffer
 442                          */
 443                         if (sze->ct_rx_lck->next == NULL)
 444                                 break;
 445
 446                         /* copy first part of header */
 447                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 448                                         sze->ct_rx_rem_bytes);
 449
 450                         /* copy second part of header */
 451                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 452                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 453                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 454                                 sze->ct_rx_cur_ptr,
 455                                 RTE_SZE2_PACKET_HEADER_SIZE -
 456                                 sze->ct_rx_rem_bytes);
 457
 458                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 459                                 sze->ct_rx_rem_bytes;
 460                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 461                                 RTE_SZE2_PACKET_HEADER_SIZE +
 462                                 sze->ct_rx_rem_bytes;
 463
 464                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 465                 } else {
 466                         /* not cut */
 467                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 468                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 469                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 470                 }
 471
 472                 sg_size = le16toh(*((uint16_t *)header_ptr));
 473                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 474                 packet_size = sg_size -
 475                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 476
 477
 478                 /* checks if packet all right */
 479                 if (!sg_size)
 480                         errx(5, "Zero segsize");
 481
 482                 /* check sg_size and hwsize */
 483                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 484                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 485                                         "hwsize: %d", sg_size, hw_size);
 486                 }
 487
 488                 hw_data_align =
 489                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 490                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 491
 492                 if (sze->ct_rx_rem_bytes >=
 493                                 (uint16_t)(sg_size -
 494                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 495                         /* no cut */
 496                         /* one packet ready - go to another */
 497                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 498                         packet_len1 = packet_size;
 499                         packet_ptr2 = NULL;
 500                         packet_len2 = 0;
 501
 502                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 503                                 RTE_SZE2_PACKET_HEADER_SIZE;
 504                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 505                                 RTE_SZE2_PACKET_HEADER_SIZE;
 506                 } else {
 507                         /* cut in data */
 508                         if (sze->ct_rx_lck->next == NULL) {
 509                                 errx(6, "Need \"next\" lock, but it is "
 510                                         "missing: %u", sze->ct_rx_rem_bytes);
 511                         }
 512
 513                         /* skip hw data */
 514                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 515                                 uint16_t rem_size = hw_data_align -
 516                                         sze->ct_rx_rem_bytes;
 517
 518                                 /* MOVE to next lock */
 519                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 520                                 sze->ct_rx_cur_ptr =
 521                                         (void *)(((uint8_t *)
 522                                         (sze->ct_rx_lck->start)) + rem_size);
 523
 524                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 525                                 packet_len1 = packet_size;
 526                                 packet_ptr2 = NULL;
 527                                 packet_len2 = 0;
 528
 529                                 sze->ct_rx_cur_ptr +=
 530                                         RTE_SZE2_ALIGN8(packet_size);
 531                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 532                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 533                         } else {
 534                                 /* get pointer and length from first part */
 535                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 536                                         hw_data_align;
 537                                 packet_len1 = sze->ct_rx_rem_bytes -
 538                                         hw_data_align;
 539
 540                                 /* MOVE to next lock */
 541                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 542                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 543
 544                                 /* get pointer and length from second part */
 545                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 546                                 packet_len2 = packet_size - packet_len1;
 547
 548                                 sze->ct_rx_cur_ptr +=
 549                                         RTE_SZE2_ALIGN8(packet_size) -
 550                                         packet_len1;
 551                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 552                                         (RTE_SZE2_ALIGN8(packet_size) -
 553                                          packet_len1);
 554                         }
 555                 }
 556
 557                 if (unlikely(packet_ptr1 == NULL))
 558                         break;
 559
 560                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 561
 562                 if (unlikely(mbuf == NULL)) {
 563                         /*
 564                          * Restore items from sze structure to state after
 565                          * unlocking (eventually locking).
 566                          */
 567                         sze->ct_rx_lck = ct_rx_lck_backup;
 568                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 569                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 570                         break;
 571                 }
 572
 573                 /* get the space available for data in the mbuf */
 574                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 575                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 576                                 RTE_PKTMBUF_HEADROOM);
 577
 578                 if (packet_size <= buf_size) {
 579                         /* sze packet will fit in one mbuf, go ahead and copy */
 580                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 581                                         packet_ptr1, packet_len1);
 582                         if (packet_ptr2 != NULL) {
 583                                 rte_memcpy((void *)
 584                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 585                                         packet_len1), packet_ptr2, packet_len2);
 586                         }
 587                         mbuf->data_len = (uint16_t)packet_size;
 588                 } else {
 589                         /*
 590                          * sze packet will not fit in one mbuf,
 591                          * scatter packet into more mbufs
 592                          */
 593                         struct rte_mbuf *m = mbuf;
 594                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 595
 596                         /* copy first part of packet */
 597                         /* fill first mbuf */
 598                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 599                                 len);
 600                         packet_len1 -= len;
 601                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 602
 603                         while (packet_len1 > 0) {
 604                                 /* fill new mbufs */
 605                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 606
 607                                 if (unlikely(m->next == NULL)) {
 608                                         rte_pktmbuf_free(mbuf);
 609                                         /*
 610                                          * Restore items from sze structure
 611                                          * to state after unlocking (eventually
 612                                          * locking).
 613                                          */
 614                                         sze->ct_rx_lck = ct_rx_lck_backup;
 615                                         sze->ct_rx_rem_bytes =
 616                                                 ct_rx_rem_bytes_backup;
 617                                         sze->ct_rx_cur_ptr =
 618                                                 ct_rx_cur_ptr_backup;
 619                                         goto finish;
 620                                 }
 621
 622                                 m = m->next;
 623
 624                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 625                                         packet_len1);
 626                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 627                                         packet_ptr1, len);
 628
 629                                 (mbuf->nb_segs)++;
 630                                 packet_len1 -= len;
 631                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 632                         }
 633
 634                         if (packet_ptr2 != NULL) {
 635                                 /* copy second part of packet, if exists */
 636                                 /* fill the rest of currently last mbuf */
 637                                 len = rte_pktmbuf_tailroom(m);
 638                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 639                                         packet_ptr2, len);
 640                                 packet_len2 -= len;
 641                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 642
 643                                 while (packet_len2 > 0) {
 644                                         /* fill new mbufs */
 645                                         m->next = rte_pktmbuf_alloc(
 646                                                         sze_q->mb_pool);
 647
 648                                         if (unlikely(m->next == NULL)) {
 649                                                 rte_pktmbuf_free(mbuf);
 650                                                 /*
 651                                                  * Restore items from sze
 652                                                  * structure to state after
 653                                                  * unlocking (eventually
 654                                                  * locking).
 655                                                  */
 656                                                 sze->ct_rx_lck =
 657                                                         ct_rx_lck_backup;
 658                                                 sze->ct_rx_rem_bytes =
 659                                                         ct_rx_rem_bytes_backup;
 660                                                 sze->ct_rx_cur_ptr =
 661                                                         ct_rx_cur_ptr_backup;
 662                                                 goto finish;
 663                                         }
 664
 665                                         m = m->next;
 666
 667                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 668                                                 packet_len2);
 669                                         rte_memcpy(
 670                                                 rte_pktmbuf_append(mbuf, len),
 671                                                 packet_ptr2, len);
 672
 673                                         (mbuf->nb_segs)++;
 674                                         packet_len2 -= len;
 675                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 676                                                 len;
 677                                 }
 678                         }
 679                 }
 680                 mbuf->pkt_len = packet_size;
 681                 mbuf->port = sze_q->in_port;
 682                 bufs[num_rx] = mbuf;
 683                 num_rx++;
 684                 num_bytes += packet_size;
 685         }
 686
 687 finish:
 688         sze_q->rx_pkts += num_rx;
 689         sze_q->rx_bytes += num_bytes;
 690         return num_rx;
 691 }
 692
 693 static uint16_t
 694 eth_szedata2_tx(void *queue,
 695                 struct rte_mbuf **bufs,
 696                 uint16_t nb_pkts)
 697 {
 698         struct rte_mbuf *mbuf;
 699         struct szedata2_tx_queue *sze_q = queue;
 700         uint16_t num_tx = 0;
 701         uint64_t num_bytes = 0;
 702
 703         const struct szedata_lock *lck;
 704         uint32_t lock_size;
 705         uint32_t lock_size2;
 706         void *dst;
 707         uint32_t pkt_len;
 708         uint32_t hwpkt_len;
 709         uint32_t unlock_size;
 710         uint32_t rem_len;
 711         uint8_t mbuf_segs;
 712         uint16_t pkt_left = nb_pkts;
 713
 714         if (sze_q->sze == NULL || nb_pkts == 0)
 715                 return 0;
 716
 717         while (pkt_left > 0) {
 718                 unlock_size = 0;
 719                 lck = szedata_tx_lock_data(sze_q->sze,
 720                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 721                         sze_q->tx_channel);
 722                 if (lck == NULL)
 723                         continue;
 724
 725                 dst = lck->start;
 726                 lock_size = lck->len;
 727                 lock_size2 = lck->next ? lck->next->len : 0;
 728
 729 next_packet:
 730                 mbuf = bufs[nb_pkts - pkt_left];
 731
 732                 pkt_len = mbuf->pkt_len;
 733                 mbuf_segs = mbuf->nb_segs;
 734
 735                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 736                         RTE_SZE2_ALIGN8(pkt_len);
 737
 738                 if (lock_size + lock_size2 < hwpkt_len) {
 739                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 740                         continue;
 741                 }
 742
 743                 num_bytes += pkt_len;
 744
 745                 if (lock_size > hwpkt_len) {
 746                         void *tmp_dst;
 747
 748                         rem_len = 0;
 749
 750                         /* write packet length at first 2 bytes in 8B header */
 751                         *((uint16_t *)dst) = htole16(
 752                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 753                                         pkt_len);
 754                         *(((uint16_t *)dst) + 1) = htole16(0);
 755
 756                         /* copy packet from mbuf */
 757                         tmp_dst = ((uint8_t *)(dst)) +
 758                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 759                         if (mbuf_segs == 1) {
 760                                 /*
 761                                  * non-scattered packet,
 762                                  * transmit from one mbuf
 763                                  */
 764                                 rte_memcpy(tmp_dst,
 765                                         rte_pktmbuf_mtod(mbuf, const void *),
 766                                         pkt_len);
 767                         } else {
 768                                 /* scattered packet, transmit from more mbufs */
 769                                 struct rte_mbuf *m = mbuf;
 770                                 while (m) {
 771                                         rte_memcpy(tmp_dst,
 772                                                 rte_pktmbuf_mtod(m,
 773                                                 const void *),
 774                                                 m->data_len);
 775                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 776                                                 m->data_len;
 777                                         m = m->next;
 778                                 }
 779                         }
 780
 781
 782                         dst = ((uint8_t *)dst) + hwpkt_len;
 783                         unlock_size += hwpkt_len;
 784                         lock_size -= hwpkt_len;
 785
 786                         rte_pktmbuf_free(mbuf);
 787                         num_tx++;
 788                         pkt_left--;
 789                         if (pkt_left == 0) {
 790                                 szedata_tx_unlock_data(sze_q->sze, lck,
 791                                         unlock_size);
 792                                 break;
 793                         }
 794                         goto next_packet;
 795                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 796                         void *tmp_dst;
 797                         uint16_t write_len;
 798
 799                         /* write packet length at first 2 bytes in 8B header */
 800                         *((uint16_t *)dst) =
 801                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 802                                         pkt_len);
 803                         *(((uint16_t *)dst) + 1) = htole16(0);
 804
 805                         /*
 806                          * If the raw packet (pkt_len) is smaller than lock_size
 807                          * get the correct length for memcpy
 808                          */
 809                         write_len =
 810                                 pkt_len < lock_size -
 811                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 812                                 pkt_len :
 813                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 814
 815                         rem_len = hwpkt_len - lock_size;
 816
 817                         tmp_dst = ((uint8_t *)(dst)) +
 818                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 819                         if (mbuf_segs == 1) {
 820                                 /*
 821                                  * non-scattered packet,
 822                                  * transmit from one mbuf
 823                                  */
 824                                 /* copy part of packet to first area */
 825                                 rte_memcpy(tmp_dst,
 826                                         rte_pktmbuf_mtod(mbuf, const void *),
 827                                         write_len);
 828
 829                                 if (lck->next)
 830                                         dst = lck->next->start;
 831
 832                                 /* copy part of packet to second area */
 833                                 rte_memcpy(dst,
 834                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 835                                                         const uint8_t *) +
 836                                         write_len), pkt_len - write_len);
 837                         } else {
 838                                 /* scattered packet, transmit from more mbufs */
 839                                 struct rte_mbuf *m = mbuf;
 840                                 uint16_t written = 0;
 841                                 uint16_t to_write = 0;
 842                                 bool new_mbuf = true;
 843                                 uint16_t write_off = 0;
 844
 845                                 /* copy part of packet to first area */
 846                                 while (m && written < write_len) {
 847                                         to_write = RTE_MIN(m->data_len,
 848                                                         write_len - written);
 849                                         rte_memcpy(tmp_dst,
 850                                                 rte_pktmbuf_mtod(m,
 851                                                         const void *),
 852                                                 to_write);
 853
 854                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 855                                                 to_write;
 856                                         if (m->data_len <= write_len -
 857                                                         written) {
 858                                                 m = m->next;
 859                                                 new_mbuf = true;
 860                                         } else {
 861                                                 new_mbuf = false;
 862                                         }
 863                                         written += to_write;
 864                                 }
 865
 866                                 if (lck->next)
 867                                         dst = lck->next->start;
 868
 869                                 tmp_dst = dst;
 870                                 written = 0;
 871                                 write_off = new_mbuf ? 0 : to_write;
 872
 873                                 /* copy part of packet to second area */
 874                                 while (m && written < pkt_len - write_len) {
 875                                         rte_memcpy(tmp_dst, (const void *)
 876                                                 (rte_pktmbuf_mtod(m,
 877                                                 uint8_t *) + write_off),
 878                                                 m->data_len - write_off);
 879
 880                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 881                                                 (m->data_len - write_off);
 882                                         written += m->data_len - write_off;
 883                                         m = m->next;
 884                                         write_off = 0;
 885                                 }
 886                         }
 887
 888                         dst = ((uint8_t *)dst) + rem_len;
 889                         unlock_size += hwpkt_len;
 890                         lock_size = lock_size2 - rem_len;
 891                         lock_size2 = 0;
 892
 893                         rte_pktmbuf_free(mbuf);
 894                         num_tx++;
 895                 }
 896
 897                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 898                 pkt_left--;
 899         }
 900
 901         sze_q->tx_pkts += num_tx;
 902         sze_q->err_pkts += nb_pkts - num_tx;
 903         sze_q->tx_bytes += num_bytes;
 904         return num_tx;
 905 }
 906
 907 static int
 908 init_rx_channels(struct rte_eth_dev *dev, int v)
 909 {
 910         struct pmd_internals *internals = dev->data->dev_private;
 911         int ret;
 912         uint32_t i;
 913         uint32_t count = internals->num_of_rx;
 914         uint32_t num_sub = 0;
 915         uint32_t x;
 916         uint32_t rx;
 917         uint32_t tx;
 918
 919         rx = internals->sze_rx_req;
 920         tx = 0;
 921
 922         for (i = 0; i < count; i++) {
 923                 /*
 924                  * Open, subscribe rx,tx channels and start device
 925                  */
 926                 if (v)
 927                         RTE_LOG(INFO, PMD, "Opening SZE device %u. time\n", i);
 928
 929                 internals->rx_queue[num_sub].sze =
 930                         szedata_open(internals->sze_dev);
 931                 if (internals->rx_queue[num_sub].sze == NULL)
 932                         return -1;
 933
 934                 /* separate least significant non-zero bit */
 935                 x = rx & ((~rx) + 1);
 936
 937                 if (v)
 938                         RTE_LOG(INFO, PMD, "Subscribing rx channel: 0x%x "
 939                                 "tx channel: 0x%x\n", x, tx);
 940
 941                 ret = szedata_subscribe3(internals->rx_queue[num_sub].sze,
 942                                 &x, &tx);
 943                 if (ret) {
 944                         szedata_close(internals->rx_queue[num_sub].sze);
 945                         internals->rx_queue[num_sub].sze = NULL;
 946                         return -1;
 947                 }
 948
 949                 if (v)
 950                         RTE_LOG(INFO, PMD, "Subscribed rx channel: 0x%x "
 951                                 "tx channel: 0x%x\n", x, tx);
 952
 953                 if (x) {
 954                         if (v)
 955                                 RTE_LOG(INFO, PMD, "Starting SZE device for "
 956                                         "rx queue: %u\n", num_sub);
 957
 958                         ret = szedata_start(internals->rx_queue[num_sub].sze);
 959                         if (ret) {
 960                                 szedata_close(internals->rx_queue[num_sub].sze);
 961                                 internals->rx_queue[num_sub].sze = NULL;
 962                                 return -1;
 963                         }
 964
 965                         /*
 966                          * set to 1 all bits lower than bit set to 1
 967                          * and that bit to 0
 968                          */
 969                         x -= 1;
 970                         internals->rx_queue[num_sub].rx_channel =
 971                                 count_ones(x);
 972
 973                         if (v)
 974                                 RTE_LOG(INFO, PMD, "Subscribed rx channel "
 975                                         "no: %u\n",
 976                                         internals->rx_queue[num_sub].rx_channel
 977                                         );
 978
 979                         num_sub++;
 980                         internals->nb_rx_queues = num_sub;
 981                 } else {
 982                         if (v)
 983                                 RTE_LOG(INFO, PMD,
 984                                         "Could not subscribe any rx channel. "
 985                                         "Closing SZE device\n");
 986
 987                         szedata_close(internals->rx_queue[num_sub].sze);
 988                         internals->rx_queue[num_sub].sze = NULL;
 989                 }
 990
 991                 /* set least significant non-zero bit to zero */
 992                 rx = rx & (rx - 1);
 993         }
 994
 995         dev->data->nb_rx_queues = (uint16_t)num_sub;
 996
 997         if (v)
 998                 RTE_LOG(INFO, PMD, "Successfully opened rx channels: %u\n",
 999                         num_sub);
1000
1001         return 0;
1002 }
1003
1004 static int
1005 init_tx_channels(struct rte_eth_dev *dev, int v)
1006 {
1007         struct pmd_internals *internals = dev->data->dev_private;
1008         int ret;
1009         uint32_t i;
1010         uint32_t count = internals->num_of_tx;
1011         uint32_t num_sub = 0;
1012         uint32_t x;
1013         uint32_t rx;
1014         uint32_t tx;
1015
1016         rx = 0;
1017         tx = internals->sze_tx_req;
1018
1019         for (i = 0; i < count; i++) {
1020                 /*
1021                  * Open, subscribe rx,tx channels and start device
1022                  */
1023                 if (v)
1024                         RTE_LOG(INFO, PMD, "Opening SZE device %u. time\n",
1025                                 i + internals->num_of_rx);
1026
1027                 internals->tx_queue[num_sub].sze =
1028                         szedata_open(internals->sze_dev);
1029                 if (internals->tx_queue[num_sub].sze == NULL)
1030                         return -1;
1031
1032                 /* separate least significant non-zero bit */
1033                 x = tx & ((~tx) + 1);
1034
1035                 if (v)
1036                         RTE_LOG(INFO, PMD, "Subscribing rx channel: 0x%x "
1037                                 "tx channel: 0x%x\n", rx, x);
1038
1039                 ret = szedata_subscribe3(internals->tx_queue[num_sub].sze,
1040                                 &rx, &x);
1041                 if (ret) {
1042                         szedata_close(internals->tx_queue[num_sub].sze);
1043                         internals->tx_queue[num_sub].sze = NULL;
1044                         return -1;
1045                 }
1046
1047                 if (v)
1048                         RTE_LOG(INFO, PMD, "Subscribed rx channel: 0x%x "
1049                                 "tx channel: 0x%x\n", rx, x);
1050
1051                 if (x) {
1052                         if (v)
1053                                 RTE_LOG(INFO, PMD, "Starting SZE device for "
1054                                         "tx queue: %u\n", num_sub);
1055
1056                         ret = szedata_start(internals->tx_queue[num_sub].sze);
1057                         if (ret) {
1058                                 szedata_close(internals->tx_queue[num_sub].sze);
1059                                 internals->tx_queue[num_sub].sze = NULL;
1060                                 return -1;
1061                         }
1062
1063                         /*
1064                          * set to 1 all bits lower than bit set to 1
1065                          * and that bit to 0
1066                          */
1067                         x -= 1;
1068                         internals->tx_queue[num_sub].tx_channel =
1069                                 count_ones(x);
1070
1071                         if (v)
1072                                 RTE_LOG(INFO, PMD, "Subscribed tx channel "
1073                                         "no: %u\n",
1074                                         internals->tx_queue[num_sub].tx_channel
1075                                         );
1076
1077                         num_sub++;
1078                         internals->nb_tx_queues = num_sub;
1079                 } else {
1080                         if (v)
1081                                 RTE_LOG(INFO, PMD,
1082                                         "Could not subscribe any tx channel. "
1083                                         "Closing SZE device\n");
1084
1085                         szedata_close(internals->tx_queue[num_sub].sze);
1086                         internals->tx_queue[num_sub].sze = NULL;
1087                 }
1088
1089                 /* set least significant non-zero bit to zero */
1090                 tx = tx & (tx - 1);
1091         }
1092
1093         dev->data->nb_tx_queues = (uint16_t)num_sub;
1094
1095         if (v)
1096                 RTE_LOG(INFO, PMD, "Successfully opened tx channels: %u\n",
1097                         num_sub);
1098
1099         return 0;
1100 }
1101
1102 static void
1103 close_rx_channels(struct rte_eth_dev *dev)
1104 {
1105         struct pmd_internals *internals = dev->data->dev_private;
1106         uint32_t i;
1107         uint32_t num_sub = internals->nb_rx_queues;
1108
1109         for (i = 0; i < num_sub; i++) {
1110                 if (internals->rx_queue[i].sze != NULL) {
1111                         szedata_close(internals->rx_queue[i].sze);
1112                         internals->rx_queue[i].sze = NULL;
1113                 }
1114         }
1115         /* set number of rx queues to zero */
1116         internals->nb_rx_queues = 0;
1117         dev->data->nb_rx_queues = (uint16_t)0;
1118 }
1119
1120 static void
1121 close_tx_channels(struct rte_eth_dev *dev)
1122 {
1123         struct pmd_internals *internals = dev->data->dev_private;
1124         uint32_t i;
1125         uint32_t num_sub = internals->nb_tx_queues;
1126
1127         for (i = 0; i < num_sub; i++) {
1128                 if (internals->tx_queue[i].sze != NULL) {
1129                         szedata_close(internals->tx_queue[i].sze);
1130                         internals->tx_queue[i].sze = NULL;
1131                 }
1132         }
1133         /* set number of rx queues to zero */
1134         internals->nb_tx_queues = 0;
1135         dev->data->nb_tx_queues = (uint16_t)0;
1136 }
1137
1138 static int
1139 eth_dev_start(struct rte_eth_dev *dev)
1140 {
1141         struct pmd_internals *internals = dev->data->dev_private;
1142         int ret;
1143
1144         if (internals->nb_rx_queues == 0) {
1145                 ret = init_rx_channels(dev, 0);
1146                 if (ret != 0) {
1147                         close_rx_channels(dev);
1148                         return -1;
1149                 }
1150         }
1151
1152         if (internals->nb_tx_queues == 0) {
1153                 ret = init_tx_channels(dev, 0);
1154                 if (ret != 0) {
1155                         close_tx_channels(dev);
1156                         close_rx_channels(dev);
1157                         return -1;
1158                 }
1159         }
1160
1161         dev->data->dev_link.link_status = 1;
1162         return 0;
1163 }
1164
1165 static void
1166 eth_dev_stop(struct rte_eth_dev *dev)
1167 {
1168         unsigned i;
1169         struct pmd_internals *internals = dev->data->dev_private;
1170
1171         for (i = 0; i < internals->nb_rx_queues; i++) {
1172                 if (internals->rx_queue[i].sze != NULL) {
1173                         szedata_close(internals->rx_queue[i].sze);
1174                         internals->rx_queue[i].sze = NULL;
1175                 }
1176         }
1177
1178         for (i = 0; i < internals->nb_tx_queues; i++) {
1179                 if (internals->tx_queue[i].sze != NULL) {
1180                         szedata_close(internals->tx_queue[i].sze);
1181                         internals->tx_queue[i].sze = NULL;
1182                 }
1183         }
1184
1185         internals->nb_rx_queues = 0;
1186         internals->nb_tx_queues = 0;
1187
1188         dev->data->nb_rx_queues = (uint16_t)0;
1189         dev->data->nb_tx_queues = (uint16_t)0;
1190
1191         dev->data->dev_link.link_status = 0;
1192 }
1193
1194 static int
1195 eth_dev_configure(struct rte_eth_dev *dev)
1196 {
1197         struct rte_eth_dev_data *data = dev->data;
1198         if (data->dev_conf.rxmode.enable_scatter == 1) {
1199                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1200                 data->scattered_rx = 1;
1201         } else {
1202                 dev->rx_pkt_burst = eth_szedata2_rx;
1203                 data->scattered_rx = 0;
1204         }
1205         return 0;
1206 }
1207
1208 static void
1209 eth_dev_info(struct rte_eth_dev *dev,
1210                 struct rte_eth_dev_info *dev_info)
1211 {
1212         struct pmd_internals *internals = dev->data->dev_private;
1213         dev_info->driver_name = drivername;
1214         dev_info->if_index = internals->if_index;
1215         dev_info->max_mac_addrs = 1;
1216         dev_info->max_rx_pktlen = (uint32_t)-1;
1217         dev_info->max_rx_queues = (uint16_t)internals->nb_rx_queues;
1218         dev_info->max_tx_queues = (uint16_t)internals->nb_tx_queues;
1219         dev_info->min_rx_bufsize = 0;
1220         dev_info->pci_dev = NULL;
1221 }
1222
1223 static void
1224 eth_stats_get(struct rte_eth_dev *dev,
1225                 struct rte_eth_stats *stats)
1226 {
1227         unsigned i;
1228         uint64_t rx_total = 0;
1229         uint64_t tx_total = 0;
1230         uint64_t tx_err_total = 0;
1231         uint64_t rx_total_bytes = 0;
1232         uint64_t tx_total_bytes = 0;
1233         const struct pmd_internals *internal = dev->data->dev_private;
1234
1235         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
1236                         i < internal->nb_rx_queues; i++) {
1237                 stats->q_ipackets[i] = internal->rx_queue[i].rx_pkts;
1238                 stats->q_ibytes[i] = internal->rx_queue[i].rx_bytes;
1239                 rx_total += stats->q_ipackets[i];
1240                 rx_total_bytes += stats->q_ibytes[i];
1241         }
1242
1243         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
1244                         i < internal->nb_tx_queues; i++) {
1245                 stats->q_opackets[i] = internal->tx_queue[i].tx_pkts;
1246                 stats->q_errors[i] = internal->tx_queue[i].err_pkts;
1247                 stats->q_obytes[i] = internal->tx_queue[i].tx_bytes;
1248                 tx_total += stats->q_opackets[i];
1249                 tx_err_total += stats->q_errors[i];
1250                 tx_total_bytes += stats->q_obytes[i];
1251         }
1252
1253         stats->ipackets = rx_total;
1254         stats->opackets = tx_total;
1255         stats->ibytes = rx_total_bytes;
1256         stats->obytes = tx_total_bytes;
1257         stats->oerrors = tx_err_total;
1258 }
1259
1260 static void
1261 eth_stats_reset(struct rte_eth_dev *dev)
1262 {
1263         unsigned i;
1264         struct pmd_internals *internal = dev->data->dev_private;
1265         for (i = 0; i < internal->nb_rx_queues; i++) {
1266                 internal->rx_queue[i].rx_pkts = 0;
1267                 internal->rx_queue[i].rx_bytes = 0;
1268         }
1269         for (i = 0; i < internal->nb_tx_queues; i++) {
1270                 internal->tx_queue[i].tx_pkts = 0;
1271                 internal->tx_queue[i].err_pkts = 0;
1272                 internal->tx_queue[i].tx_bytes = 0;
1273         }
1274 }
1275
1276 static void
1277 eth_dev_close(struct rte_eth_dev *dev)
1278 {
1279         unsigned i;
1280         struct pmd_internals *internals = dev->data->dev_private;
1281
1282         for (i = 0; i < internals->nb_rx_queues; i++) {
1283                 if (internals->rx_queue[i].sze != NULL) {
1284                         szedata_close(internals->rx_queue[i].sze);
1285                         internals->rx_queue[i].sze = NULL;
1286                 }
1287         }
1288
1289         for (i = 0; i < internals->nb_tx_queues; i++) {
1290                 if (internals->tx_queue[i].sze != NULL) {
1291                         szedata_close(internals->tx_queue[i].sze);
1292                         internals->tx_queue[i].sze = NULL;
1293                 }
1294         }
1295
1296         internals->nb_rx_queues = 0;
1297         internals->nb_tx_queues = 0;
1298
1299         dev->data->nb_rx_queues = (uint16_t)0;
1300         dev->data->nb_tx_queues = (uint16_t)0;
1301 }
1302
1303 static void
1304 eth_queue_release(void *q __rte_unused)
1305 {
1306 }
1307
1308 static int
1309 eth_link_update(struct rte_eth_dev *dev __rte_unused,
1310                 int wait_to_complete __rte_unused)
1311 {
1312         return 0;
1313 }
1314
1315 static int
1316 eth_rx_queue_setup(struct rte_eth_dev *dev,
1317                 uint16_t rx_queue_id,
1318                 uint16_t nb_rx_desc __rte_unused,
1319                 unsigned int socket_id __rte_unused,
1320                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1321                 struct rte_mempool *mb_pool)
1322 {
1323         struct pmd_internals *internals = dev->data->dev_private;
1324         struct szedata2_rx_queue *szedata2_q =
1325                 &internals->rx_queue[rx_queue_id];
1326         szedata2_q->mb_pool = mb_pool;
1327         dev->data->rx_queues[rx_queue_id] = szedata2_q;
1328         szedata2_q->in_port = dev->data->port_id;
1329         return 0;
1330 }
1331
1332 static int
1333 eth_tx_queue_setup(struct rte_eth_dev *dev,
1334                 uint16_t tx_queue_id,
1335                 uint16_t nb_tx_desc __rte_unused,
1336                 unsigned int socket_id __rte_unused,
1337                 const struct rte_eth_txconf *tx_conf __rte_unused)
1338 {
1339         struct pmd_internals *internals = dev->data->dev_private;
1340         dev->data->tx_queues[tx_queue_id] = &internals->tx_queue[tx_queue_id];
1341         return 0;
1342 }
1343
1344 static void
1345 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1346                 struct ether_addr *mac_addr __rte_unused)
1347 {
1348 }
1349
1350 static struct eth_dev_ops ops = {
1351                 .dev_start          = eth_dev_start,
1352                 .dev_stop           = eth_dev_stop,
1353                 .dev_close          = eth_dev_close,
1354                 .dev_configure      = eth_dev_configure,
1355                 .dev_infos_get      = eth_dev_info,
1356                 .rx_queue_setup     = eth_rx_queue_setup,
1357                 .tx_queue_setup     = eth_tx_queue_setup,
1358                 .rx_queue_release   = eth_queue_release,
1359                 .tx_queue_release   = eth_queue_release,
1360                 .link_update        = eth_link_update,
1361                 .stats_get          = eth_stats_get,
1362                 .stats_reset        = eth_stats_reset,
1363                 .mac_addr_set       = eth_mac_addr_set,
1364 };
1365
1366 static int
1367 parse_mask(const char *mask_str, uint32_t *mask_num)
1368 {
1369         char *endptr;
1370         long int value;
1371
1372         value = strtol(mask_str, &endptr, 0);
1373         if (*endptr != '\0' || value > UINT32_MAX || value < 0)
1374                 return -1;
1375
1376         *mask_num = (uint32_t)value;
1377         return 0;
1378 }
1379
1380 static int
1381 add_rx_mask(const char *key __rte_unused, const char *value, void *extra_args)
1382 {
1383         struct rxtx_szedata2 *szedata2 = extra_args;
1384         uint32_t mask;
1385
1386         if (parse_mask(value, &mask) != 0)
1387                 return -1;
1388
1389         szedata2->sze_rx_mask_req |= mask;
1390         return 0;
1391 }
1392
1393 static int
1394 add_tx_mask(const char *key __rte_unused, const char *value, void *extra_args)
1395 {
1396         struct rxtx_szedata2 *szedata2 = extra_args;
1397         uint32_t mask;
1398
1399         if (parse_mask(value, &mask) != 0)
1400                 return -1;
1401
1402         szedata2->sze_tx_mask_req |= mask;
1403         return 0;
1404 }
1405
1406 static int
1407 rte_pmd_init_internals(const char *name, const unsigned nb_rx_queues,
1408                 const unsigned nb_tx_queues,
1409                 const unsigned numa_node,
1410                 struct pmd_internals **internals,
1411                 struct rte_eth_dev **eth_dev)
1412 {
1413         struct rte_eth_dev_data *data = NULL;
1414
1415         RTE_LOG(INFO, PMD,
1416                         "Creating szedata2-backed ethdev on numa socket %u\n",
1417                         numa_node);
1418
1419         /*
1420          * now do all data allocation - for eth_dev structure
1421          * and internal (private) data
1422          */
1423         data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node);
1424         if (data == NULL)
1425                 goto error;
1426
1427         *internals = rte_zmalloc_socket(name, sizeof(**internals), 0,
1428                         numa_node);
1429         if (*internals == NULL)
1430                 goto error;
1431
1432         /* reserve an ethdev entry */
1433         *eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL);
1434         if (*eth_dev == NULL)
1435                 goto error;
1436
1437         /*
1438          * now put it all together
1439          * - store queue data in internals,
1440          * - store numa_node info in pci_driver
1441          * - point eth_dev_data to internals
1442          * - and point eth_dev structure to new eth_dev_data structure
1443          *
1444          * NOTE: we'll replace the data element, of originally allocated eth_dev
1445          * so the rings are local per-process
1446          */
1447
1448         (*internals)->nb_rx_queues = nb_rx_queues;
1449         (*internals)->nb_tx_queues = nb_tx_queues;
1450
1451         (*internals)->if_index = 0;
1452
1453         data->dev_private = *internals;
1454         data->port_id = (*eth_dev)->data->port_id;
1455         snprintf(data->name, sizeof(data->name), "%s", (*eth_dev)->data->name);
1456         data->nb_rx_queues = (uint16_t)nb_rx_queues;
1457         data->nb_tx_queues = (uint16_t)nb_tx_queues;
1458         data->dev_link = pmd_link;
1459         data->mac_addrs = &eth_addr;
1460
1461         (*eth_dev)->data = data;
1462         (*eth_dev)->dev_ops = &ops;
1463         (*eth_dev)->data->dev_flags = RTE_ETH_DEV_DETACHABLE;
1464         (*eth_dev)->driver = NULL;
1465         (*eth_dev)->data->kdrv = RTE_KDRV_NONE;
1466         (*eth_dev)->data->drv_name = drivername;
1467         (*eth_dev)->data->numa_node = numa_node;
1468
1469         return 0;
1470
1471 error:
1472         rte_free(data);
1473         rte_free(*internals);
1474         return -1;
1475 }
1476
1477 static int
1478 rte_eth_from_szedata2(const char *name,
1479                 struct rxtx_szedata2 *szedata2,
1480                 const unsigned numa_node)
1481 {
1482         struct pmd_internals *internals = NULL;
1483         struct rte_eth_dev *eth_dev = NULL;
1484         struct rte_eth_dev_data *data = NULL;
1485         int ret;
1486
1487         if (rte_pmd_init_internals(name, 0, 0, numa_node,
1488                         &internals, &eth_dev) < 0)
1489                 return -1;
1490
1491         data = eth_dev->data;
1492
1493         internals->sze_dev = szedata2->sze_dev;
1494         internals->sze_rx_req = szedata2->sze_rx_mask_req;
1495         internals->sze_tx_req = szedata2->sze_tx_mask_req;
1496         internals->num_of_rx = szedata2->num_of_rx;
1497         internals->num_of_tx = szedata2->num_of_tx;
1498
1499         RTE_LOG(INFO, PMD, "Number of rx channels to open: %u mask: 0x%x\n",
1500                         internals->num_of_rx, internals->sze_rx_req);
1501         RTE_LOG(INFO, PMD, "Number of tx channels to open: %u mask: 0x%x\n",
1502                         internals->num_of_tx, internals->sze_tx_req);
1503
1504         ret = init_rx_channels(eth_dev, 1);
1505         if (ret != 0) {
1506                 close_rx_channels(eth_dev);
1507                 return -1;
1508         }
1509
1510         ret = init_tx_channels(eth_dev, 1);
1511         if (ret != 0) {
1512                 close_tx_channels(eth_dev);
1513                 close_rx_channels(eth_dev);
1514                 return -1;
1515         }
1516
1517         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1518                 data->scattered_rx == 1) {
1519                 eth_dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1520                 data->scattered_rx = 1;
1521         } else {
1522                 eth_dev->rx_pkt_burst = eth_szedata2_rx;
1523                 data->scattered_rx = 0;
1524         }
1525         eth_dev->tx_pkt_burst = eth_szedata2_tx;
1526
1527         return 0;
1528 }
1529
1530
1531 static int
1532 rte_pmd_szedata2_devinit(const char *name, const char *params)
1533 {
1534         unsigned numa_node;
1535         int ret;
1536         struct rte_kvargs *kvlist;
1537         unsigned k_idx;
1538         struct rte_kvargs_pair *pair = NULL;
1539         struct rxtx_szedata2 szedata2 = { 0, 0, 0, 0, NULL };
1540         bool dev_path_missing = true;
1541
1542         RTE_LOG(INFO, PMD, "Initializing pmd_szedata2 for %s\n", name);
1543
1544         numa_node = rte_socket_id();
1545
1546         kvlist = rte_kvargs_parse(params, valid_arguments);
1547         if (kvlist == NULL)
1548                 return -1;
1549
1550         /*
1551          * Get szedata2 device path and rx,tx channels from passed arguments.
1552          */
1553
1554         if (rte_kvargs_count(kvlist, RTE_ETH_SZEDATA2_DEV_PATH_ARG) != 1)
1555                 goto err;
1556
1557         if (rte_kvargs_count(kvlist, RTE_ETH_SZEDATA2_RX_IFACES_ARG) < 1)
1558                 goto err;
1559
1560         if (rte_kvargs_count(kvlist, RTE_ETH_SZEDATA2_TX_IFACES_ARG) < 1)
1561                 goto err;
1562
1563         for (k_idx = 0; k_idx < kvlist->count; k_idx++) {
1564                 pair = &kvlist->pairs[k_idx];
1565                 if (strstr(pair->key, RTE_ETH_SZEDATA2_DEV_PATH_ARG) != NULL) {
1566                         szedata2.sze_dev = pair->value;
1567                         dev_path_missing = false;
1568                         break;
1569                 }
1570         }
1571
1572         if (dev_path_missing)
1573                 goto err;
1574
1575         ret = rte_kvargs_process(kvlist, RTE_ETH_SZEDATA2_RX_IFACES_ARG,
1576                         &add_rx_mask, &szedata2);
1577         if (ret < 0)
1578                 goto err;
1579
1580         ret = rte_kvargs_process(kvlist, RTE_ETH_SZEDATA2_TX_IFACES_ARG,
1581                         &add_tx_mask, &szedata2);
1582         if (ret < 0)
1583                 goto err;
1584
1585         szedata2.num_of_rx = count_ones(szedata2.sze_rx_mask_req);
1586         szedata2.num_of_tx = count_ones(szedata2.sze_tx_mask_req);
1587
1588         RTE_LOG(INFO, PMD, "SZE device found at path %s\n", szedata2.sze_dev);
1589
1590         return rte_eth_from_szedata2(name, &szedata2, numa_node);
1591 err:
1592         rte_kvargs_free(kvlist);
1593         return -1;
1594 }
1595
1596 static int
1597 rte_pmd_szedata2_devuninit(const char *name)
1598 {
1599         struct rte_eth_dev *dev = NULL;
1600
1601         RTE_LOG(INFO, PMD, "Uninitializing pmd_szedata2 for %s "
1602                         "on numa socket %u\n", name, rte_socket_id());
1603
1604         if (name == NULL)
1605                 return -1;
1606
1607         dev = rte_eth_dev_allocated(name);
1608         if (dev == NULL)
1609                 return -1;
1610
1611         rte_free(dev->data->dev_private);
1612         rte_free(dev->data);
1613         rte_eth_dev_release_port(dev);
1614         return 0;
1615 }
1616
1617 static struct rte_driver pmd_szedata2_drv = {
1618         .name = "eth_szedata2",
1619         .type = PMD_VDEV,
1620         .init = rte_pmd_szedata2_devinit,
1621         .uninit = rte_pmd_szedata2_devuninit,
1622 };
1623
1624 PMD_REGISTER_DRIVER(pmd_szedata2_drv);