drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 - 2016 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38 #include <sys/types.h>
  39 #include <dirent.h>
  40 #include <sys/stat.h>
  41 #include <fcntl.h>
  42 #include <sys/mman.h>
  43
  44 #include <libsze2.h>
  45
  46 #include <rte_mbuf.h>
  47 #include <rte_ethdev_driver.h>
  48 #include <rte_ethdev_pci.h>
  49 #include <rte_malloc.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_kvargs.h>
  52 #include <rte_dev.h>
  53
  54 #include "rte_eth_szedata2.h"
  55 #include "szedata2_iobuf.h"
  56
  57 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  58 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  59 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  60
  61 /**
  62  * size of szedata2_packet header with alignment
  63  */
  64 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  65
  66 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2
  67
  68 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
  69
  70 struct szedata2_rx_queue {
  71         struct szedata *sze;
  72         uint8_t rx_channel;
  73         uint16_t in_port;
  74         struct rte_mempool *mb_pool;
  75         volatile uint64_t rx_pkts;
  76         volatile uint64_t rx_bytes;
  77         volatile uint64_t err_pkts;
  78 };
  79
  80 struct szedata2_tx_queue {
  81         struct szedata *sze;
  82         uint8_t tx_channel;
  83         volatile uint64_t tx_pkts;
  84         volatile uint64_t tx_bytes;
  85         volatile uint64_t err_pkts;
  86 };
  87
  88 struct pmd_internals {
  89         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  90         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  91         uint16_t max_rx_queues;
  92         uint16_t max_tx_queues;
  93         char sze_dev[PATH_MAX];
  94         struct rte_mem_resource *pci_rsc;
  95 };
  96
  97 static struct ether_addr eth_addr = {
  98         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
  99 };
 100
 101 static uint16_t
 102 eth_szedata2_rx(void *queue,
 103                 struct rte_mbuf **bufs,
 104                 uint16_t nb_pkts)
 105 {
 106         unsigned int i;
 107         struct rte_mbuf *mbuf;
 108         struct szedata2_rx_queue *sze_q = queue;
 109         struct rte_pktmbuf_pool_private *mbp_priv;
 110         uint16_t num_rx = 0;
 111         uint16_t buf_size;
 112         uint16_t sg_size;
 113         uint16_t hw_size;
 114         uint16_t packet_size;
 115         uint64_t num_bytes = 0;
 116         struct szedata *sze = sze_q->sze;
 117         uint8_t *header_ptr = NULL; /* header of packet */
 118         uint8_t *packet_ptr1 = NULL;
 119         uint8_t *packet_ptr2 = NULL;
 120         uint16_t packet_len1 = 0;
 121         uint16_t packet_len2 = 0;
 122         uint16_t hw_data_align;
 123
 124         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 125                 return 0;
 126
 127         /*
 128          * Reads the given number of packets from szedata2 channel given
 129          * by queue and copies the packet data into a newly allocated mbuf
 130          * to return.
 131          */
 132         for (i = 0; i < nb_pkts; i++) {
 133                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 134
 135                 if (unlikely(mbuf == NULL))
 136                         break;
 137
 138                 /* get the next sze packet */
 139                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 140                                 sze->ct_rx_lck->next == NULL) {
 141                         /* unlock old data */
 142                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 143                         sze->ct_rx_lck_orig = NULL;
 144                         sze->ct_rx_lck = NULL;
 145                 }
 146
 147                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 148                         /* nothing to read, lock new data */
 149                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 150                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 151
 152                         if (sze->ct_rx_lck == NULL) {
 153                                 /* nothing to lock */
 154                                 rte_pktmbuf_free(mbuf);
 155                                 break;
 156                         }
 157
 158                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 159                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 160
 161                         if (!sze->ct_rx_rem_bytes) {
 162                                 rte_pktmbuf_free(mbuf);
 163                                 break;
 164                         }
 165                 }
 166
 167                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 168                         /*
 169                          * cut in header
 170                          * copy parts of header to merge buffer
 171                          */
 172                         if (sze->ct_rx_lck->next == NULL) {
 173                                 rte_pktmbuf_free(mbuf);
 174                                 break;
 175                         }
 176
 177                         /* copy first part of header */
 178                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 179                                         sze->ct_rx_rem_bytes);
 180
 181                         /* copy second part of header */
 182                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 183                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 184                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 185                                 sze->ct_rx_cur_ptr,
 186                                 RTE_SZE2_PACKET_HEADER_SIZE -
 187                                 sze->ct_rx_rem_bytes);
 188
 189                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 190                                 sze->ct_rx_rem_bytes;
 191                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 192                                 RTE_SZE2_PACKET_HEADER_SIZE +
 193                                 sze->ct_rx_rem_bytes;
 194
 195                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 196                 } else {
 197                         /* not cut */
 198                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 199                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 200                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 201                 }
 202
 203                 sg_size = le16toh(*((uint16_t *)header_ptr));
 204                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 205                 packet_size = sg_size -
 206                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 207
 208
 209                 /* checks if packet all right */
 210                 if (!sg_size)
 211                         errx(5, "Zero segsize");
 212
 213                 /* check sg_size and hwsize */
 214                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 215                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 216                                 "hwsize: %d", sg_size, hw_size);
 217                 }
 218
 219                 hw_data_align =
 220                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 221                         RTE_SZE2_PACKET_HEADER_SIZE;
 222
 223                 if (sze->ct_rx_rem_bytes >=
 224                                 (uint16_t)(sg_size -
 225                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 226                         /* no cut */
 227                         /* one packet ready - go to another */
 228                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 229                         packet_len1 = packet_size;
 230                         packet_ptr2 = NULL;
 231                         packet_len2 = 0;
 232
 233                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 234                                 RTE_SZE2_PACKET_HEADER_SIZE;
 235                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 236                                 RTE_SZE2_PACKET_HEADER_SIZE;
 237                 } else {
 238                         /* cut in data */
 239                         if (sze->ct_rx_lck->next == NULL) {
 240                                 errx(6, "Need \"next\" lock, "
 241                                         "but it is missing: %u",
 242                                         sze->ct_rx_rem_bytes);
 243                         }
 244
 245                         /* skip hw data */
 246                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 247                                 uint16_t rem_size = hw_data_align -
 248                                         sze->ct_rx_rem_bytes;
 249
 250                                 /* MOVE to next lock */
 251                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 252                                 sze->ct_rx_cur_ptr =
 253                                         (void *)(((uint8_t *)
 254                                         (sze->ct_rx_lck->start)) + rem_size);
 255
 256                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 257                                 packet_len1 = packet_size;
 258                                 packet_ptr2 = NULL;
 259                                 packet_len2 = 0;
 260
 261                                 sze->ct_rx_cur_ptr +=
 262                                         RTE_SZE2_ALIGN8(packet_size);
 263                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 264                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 265                         } else {
 266                                 /* get pointer and length from first part */
 267                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 268                                         hw_data_align;
 269                                 packet_len1 = sze->ct_rx_rem_bytes -
 270                                         hw_data_align;
 271
 272                                 /* MOVE to next lock */
 273                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 274                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 275
 276                                 /* get pointer and length from second part */
 277                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 278                                 packet_len2 = packet_size - packet_len1;
 279
 280                                 sze->ct_rx_cur_ptr +=
 281                                         RTE_SZE2_ALIGN8(packet_size) -
 282                                         packet_len1;
 283                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 284                                         (RTE_SZE2_ALIGN8(packet_size) -
 285                                          packet_len1);
 286                         }
 287                 }
 288
 289                 if (unlikely(packet_ptr1 == NULL)) {
 290                         rte_pktmbuf_free(mbuf);
 291                         break;
 292                 }
 293
 294                 /* get the space available for data in the mbuf */
 295                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 296                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 297                                 RTE_PKTMBUF_HEADROOM);
 298
 299                 if (packet_size <= buf_size) {
 300                         /* sze packet will fit in one mbuf, go ahead and copy */
 301                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 302                                         packet_ptr1, packet_len1);
 303                         if (packet_ptr2 != NULL) {
 304                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 305                                         uint8_t *) + packet_len1),
 306                                         packet_ptr2, packet_len2);
 307                         }
 308                         mbuf->data_len = (uint16_t)packet_size;
 309
 310                         mbuf->pkt_len = packet_size;
 311                         mbuf->port = sze_q->in_port;
 312                         bufs[num_rx] = mbuf;
 313                         num_rx++;
 314                         num_bytes += packet_size;
 315                 } else {
 316                         /*
 317                          * sze packet will not fit in one mbuf,
 318                          * scattered mode is not enabled, drop packet
 319                          */
 320                         RTE_LOG(ERR, PMD,
 321                                 "SZE segment %d bytes will not fit in one mbuf "
 322                                 "(%d bytes), scattered mode is not enabled, "
 323                                 "drop packet!!\n",
 324                                 packet_size, buf_size);
 325                         rte_pktmbuf_free(mbuf);
 326                 }
 327         }
 328
 329         sze_q->rx_pkts += num_rx;
 330         sze_q->rx_bytes += num_bytes;
 331         return num_rx;
 332 }
 333
 334 static uint16_t
 335 eth_szedata2_rx_scattered(void *queue,
 336                 struct rte_mbuf **bufs,
 337                 uint16_t nb_pkts)
 338 {
 339         unsigned int i;
 340         struct rte_mbuf *mbuf;
 341         struct szedata2_rx_queue *sze_q = queue;
 342         struct rte_pktmbuf_pool_private *mbp_priv;
 343         uint16_t num_rx = 0;
 344         uint16_t buf_size;
 345         uint16_t sg_size;
 346         uint16_t hw_size;
 347         uint16_t packet_size;
 348         uint64_t num_bytes = 0;
 349         struct szedata *sze = sze_q->sze;
 350         uint8_t *header_ptr = NULL; /* header of packet */
 351         uint8_t *packet_ptr1 = NULL;
 352         uint8_t *packet_ptr2 = NULL;
 353         uint16_t packet_len1 = 0;
 354         uint16_t packet_len2 = 0;
 355         uint16_t hw_data_align;
 356
 357         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 358                 return 0;
 359
 360         /*
 361          * Reads the given number of packets from szedata2 channel given
 362          * by queue and copies the packet data into a newly allocated mbuf
 363          * to return.
 364          */
 365         for (i = 0; i < nb_pkts; i++) {
 366                 const struct szedata_lock *ct_rx_lck_backup;
 367                 unsigned int ct_rx_rem_bytes_backup;
 368                 unsigned char *ct_rx_cur_ptr_backup;
 369
 370                 /* get the next sze packet */
 371                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 372                                 sze->ct_rx_lck->next == NULL) {
 373                         /* unlock old data */
 374                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 375                         sze->ct_rx_lck_orig = NULL;
 376                         sze->ct_rx_lck = NULL;
 377                 }
 378
 379                 /*
 380                  * Store items from sze structure which can be changed
 381                  * before mbuf allocating. Use these items in case of mbuf
 382                  * allocating failure.
 383                  */
 384                 ct_rx_lck_backup = sze->ct_rx_lck;
 385                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 386                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 387
 388                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 389                         /* nothing to read, lock new data */
 390                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 391                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 392
 393                         /*
 394                          * Backup items from sze structure must be updated
 395                          * after locking to contain pointers to new locks.
 396                          */
 397                         ct_rx_lck_backup = sze->ct_rx_lck;
 398                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 399                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 400
 401                         if (sze->ct_rx_lck == NULL)
 402                                 /* nothing to lock */
 403                                 break;
 404
 405                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 406                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 407
 408                         if (!sze->ct_rx_rem_bytes)
 409                                 break;
 410                 }
 411
 412                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 413                         /*
 414                          * cut in header - copy parts of header to merge buffer
 415                          */
 416                         if (sze->ct_rx_lck->next == NULL)
 417                                 break;
 418
 419                         /* copy first part of header */
 420                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 421                                         sze->ct_rx_rem_bytes);
 422
 423                         /* copy second part of header */
 424                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 425                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 426                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 427                                 sze->ct_rx_cur_ptr,
 428                                 RTE_SZE2_PACKET_HEADER_SIZE -
 429                                 sze->ct_rx_rem_bytes);
 430
 431                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 432                                 sze->ct_rx_rem_bytes;
 433                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 434                                 RTE_SZE2_PACKET_HEADER_SIZE +
 435                                 sze->ct_rx_rem_bytes;
 436
 437                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 438                 } else {
 439                         /* not cut */
 440                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 441                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 442                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 443                 }
 444
 445                 sg_size = le16toh(*((uint16_t *)header_ptr));
 446                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 447                 packet_size = sg_size -
 448                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 449
 450
 451                 /* checks if packet all right */
 452                 if (!sg_size)
 453                         errx(5, "Zero segsize");
 454
 455                 /* check sg_size and hwsize */
 456                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 457                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 458                                         "hwsize: %d", sg_size, hw_size);
 459                 }
 460
 461                 hw_data_align =
 462                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 463                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 464
 465                 if (sze->ct_rx_rem_bytes >=
 466                                 (uint16_t)(sg_size -
 467                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 468                         /* no cut */
 469                         /* one packet ready - go to another */
 470                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 471                         packet_len1 = packet_size;
 472                         packet_ptr2 = NULL;
 473                         packet_len2 = 0;
 474
 475                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 476                                 RTE_SZE2_PACKET_HEADER_SIZE;
 477                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 478                                 RTE_SZE2_PACKET_HEADER_SIZE;
 479                 } else {
 480                         /* cut in data */
 481                         if (sze->ct_rx_lck->next == NULL) {
 482                                 errx(6, "Need \"next\" lock, but it is "
 483                                         "missing: %u", sze->ct_rx_rem_bytes);
 484                         }
 485
 486                         /* skip hw data */
 487                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 488                                 uint16_t rem_size = hw_data_align -
 489                                         sze->ct_rx_rem_bytes;
 490
 491                                 /* MOVE to next lock */
 492                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 493                                 sze->ct_rx_cur_ptr =
 494                                         (void *)(((uint8_t *)
 495                                         (sze->ct_rx_lck->start)) + rem_size);
 496
 497                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 498                                 packet_len1 = packet_size;
 499                                 packet_ptr2 = NULL;
 500                                 packet_len2 = 0;
 501
 502                                 sze->ct_rx_cur_ptr +=
 503                                         RTE_SZE2_ALIGN8(packet_size);
 504                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 505                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 506                         } else {
 507                                 /* get pointer and length from first part */
 508                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 509                                         hw_data_align;
 510                                 packet_len1 = sze->ct_rx_rem_bytes -
 511                                         hw_data_align;
 512
 513                                 /* MOVE to next lock */
 514                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 515                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 516
 517                                 /* get pointer and length from second part */
 518                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 519                                 packet_len2 = packet_size - packet_len1;
 520
 521                                 sze->ct_rx_cur_ptr +=
 522                                         RTE_SZE2_ALIGN8(packet_size) -
 523                                         packet_len1;
 524                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 525                                         (RTE_SZE2_ALIGN8(packet_size) -
 526                                          packet_len1);
 527                         }
 528                 }
 529
 530                 if (unlikely(packet_ptr1 == NULL))
 531                         break;
 532
 533                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 534
 535                 if (unlikely(mbuf == NULL)) {
 536                         /*
 537                          * Restore items from sze structure to state after
 538                          * unlocking (eventually locking).
 539                          */
 540                         sze->ct_rx_lck = ct_rx_lck_backup;
 541                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 542                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 543                         break;
 544                 }
 545
 546                 /* get the space available for data in the mbuf */
 547                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 548                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 549                                 RTE_PKTMBUF_HEADROOM);
 550
 551                 if (packet_size <= buf_size) {
 552                         /* sze packet will fit in one mbuf, go ahead and copy */
 553                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 554                                         packet_ptr1, packet_len1);
 555                         if (packet_ptr2 != NULL) {
 556                                 rte_memcpy((void *)
 557                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 558                                         packet_len1), packet_ptr2, packet_len2);
 559                         }
 560                         mbuf->data_len = (uint16_t)packet_size;
 561                 } else {
 562                         /*
 563                          * sze packet will not fit in one mbuf,
 564                          * scatter packet into more mbufs
 565                          */
 566                         struct rte_mbuf *m = mbuf;
 567                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 568
 569                         /* copy first part of packet */
 570                         /* fill first mbuf */
 571                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 572                                 len);
 573                         packet_len1 -= len;
 574                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 575
 576                         while (packet_len1 > 0) {
 577                                 /* fill new mbufs */
 578                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 579
 580                                 if (unlikely(m->next == NULL)) {
 581                                         rte_pktmbuf_free(mbuf);
 582                                         /*
 583                                          * Restore items from sze structure
 584                                          * to state after unlocking (eventually
 585                                          * locking).
 586                                          */
 587                                         sze->ct_rx_lck = ct_rx_lck_backup;
 588                                         sze->ct_rx_rem_bytes =
 589                                                 ct_rx_rem_bytes_backup;
 590                                         sze->ct_rx_cur_ptr =
 591                                                 ct_rx_cur_ptr_backup;
 592                                         goto finish;
 593                                 }
 594
 595                                 m = m->next;
 596
 597                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 598                                         packet_len1);
 599                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 600                                         packet_ptr1, len);
 601
 602                                 (mbuf->nb_segs)++;
 603                                 packet_len1 -= len;
 604                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 605                         }
 606
 607                         if (packet_ptr2 != NULL) {
 608                                 /* copy second part of packet, if exists */
 609                                 /* fill the rest of currently last mbuf */
 610                                 len = rte_pktmbuf_tailroom(m);
 611                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 612                                         packet_ptr2, len);
 613                                 packet_len2 -= len;
 614                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 615
 616                                 while (packet_len2 > 0) {
 617                                         /* fill new mbufs */
 618                                         m->next = rte_pktmbuf_alloc(
 619                                                         sze_q->mb_pool);
 620
 621                                         if (unlikely(m->next == NULL)) {
 622                                                 rte_pktmbuf_free(mbuf);
 623                                                 /*
 624                                                  * Restore items from sze
 625                                                  * structure to state after
 626                                                  * unlocking (eventually
 627                                                  * locking).
 628                                                  */
 629                                                 sze->ct_rx_lck =
 630                                                         ct_rx_lck_backup;
 631                                                 sze->ct_rx_rem_bytes =
 632                                                         ct_rx_rem_bytes_backup;
 633                                                 sze->ct_rx_cur_ptr =
 634                                                         ct_rx_cur_ptr_backup;
 635                                                 goto finish;
 636                                         }
 637
 638                                         m = m->next;
 639
 640                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 641                                                 packet_len2);
 642                                         rte_memcpy(
 643                                                 rte_pktmbuf_append(mbuf, len),
 644                                                 packet_ptr2, len);
 645
 646                                         (mbuf->nb_segs)++;
 647                                         packet_len2 -= len;
 648                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 649                                                 len;
 650                                 }
 651                         }
 652                 }
 653                 mbuf->pkt_len = packet_size;
 654                 mbuf->port = sze_q->in_port;
 655                 bufs[num_rx] = mbuf;
 656                 num_rx++;
 657                 num_bytes += packet_size;
 658         }
 659
 660 finish:
 661         sze_q->rx_pkts += num_rx;
 662         sze_q->rx_bytes += num_bytes;
 663         return num_rx;
 664 }
 665
 666 static uint16_t
 667 eth_szedata2_tx(void *queue,
 668                 struct rte_mbuf **bufs,
 669                 uint16_t nb_pkts)
 670 {
 671         struct rte_mbuf *mbuf;
 672         struct szedata2_tx_queue *sze_q = queue;
 673         uint16_t num_tx = 0;
 674         uint64_t num_bytes = 0;
 675
 676         const struct szedata_lock *lck;
 677         uint32_t lock_size;
 678         uint32_t lock_size2;
 679         void *dst;
 680         uint32_t pkt_len;
 681         uint32_t hwpkt_len;
 682         uint32_t unlock_size;
 683         uint32_t rem_len;
 684         uint16_t mbuf_segs;
 685         uint16_t pkt_left = nb_pkts;
 686
 687         if (sze_q->sze == NULL || nb_pkts == 0)
 688                 return 0;
 689
 690         while (pkt_left > 0) {
 691                 unlock_size = 0;
 692                 lck = szedata_tx_lock_data(sze_q->sze,
 693                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 694                         sze_q->tx_channel);
 695                 if (lck == NULL)
 696                         continue;
 697
 698                 dst = lck->start;
 699                 lock_size = lck->len;
 700                 lock_size2 = lck->next ? lck->next->len : 0;
 701
 702 next_packet:
 703                 mbuf = bufs[nb_pkts - pkt_left];
 704
 705                 pkt_len = mbuf->pkt_len;
 706                 mbuf_segs = mbuf->nb_segs;
 707
 708                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 709                         RTE_SZE2_ALIGN8(pkt_len);
 710
 711                 if (lock_size + lock_size2 < hwpkt_len) {
 712                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 713                         continue;
 714                 }
 715
 716                 num_bytes += pkt_len;
 717
 718                 if (lock_size > hwpkt_len) {
 719                         void *tmp_dst;
 720
 721                         rem_len = 0;
 722
 723                         /* write packet length at first 2 bytes in 8B header */
 724                         *((uint16_t *)dst) = htole16(
 725                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 726                                         pkt_len);
 727                         *(((uint16_t *)dst) + 1) = htole16(0);
 728
 729                         /* copy packet from mbuf */
 730                         tmp_dst = ((uint8_t *)(dst)) +
 731                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 732                         if (mbuf_segs == 1) {
 733                                 /*
 734                                  * non-scattered packet,
 735                                  * transmit from one mbuf
 736                                  */
 737                                 rte_memcpy(tmp_dst,
 738                                         rte_pktmbuf_mtod(mbuf, const void *),
 739                                         pkt_len);
 740                         } else {
 741                                 /* scattered packet, transmit from more mbufs */
 742                                 struct rte_mbuf *m = mbuf;
 743                                 while (m) {
 744                                         rte_memcpy(tmp_dst,
 745                                                 rte_pktmbuf_mtod(m,
 746                                                 const void *),
 747                                                 m->data_len);
 748                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 749                                                 m->data_len;
 750                                         m = m->next;
 751                                 }
 752                         }
 753
 754
 755                         dst = ((uint8_t *)dst) + hwpkt_len;
 756                         unlock_size += hwpkt_len;
 757                         lock_size -= hwpkt_len;
 758
 759                         rte_pktmbuf_free(mbuf);
 760                         num_tx++;
 761                         pkt_left--;
 762                         if (pkt_left == 0) {
 763                                 szedata_tx_unlock_data(sze_q->sze, lck,
 764                                         unlock_size);
 765                                 break;
 766                         }
 767                         goto next_packet;
 768                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 769                         void *tmp_dst;
 770                         uint16_t write_len;
 771
 772                         /* write packet length at first 2 bytes in 8B header */
 773                         *((uint16_t *)dst) =
 774                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 775                                         pkt_len);
 776                         *(((uint16_t *)dst) + 1) = htole16(0);
 777
 778                         /*
 779                          * If the raw packet (pkt_len) is smaller than lock_size
 780                          * get the correct length for memcpy
 781                          */
 782                         write_len =
 783                                 pkt_len < lock_size -
 784                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 785                                 pkt_len :
 786                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 787
 788                         rem_len = hwpkt_len - lock_size;
 789
 790                         tmp_dst = ((uint8_t *)(dst)) +
 791                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 792                         if (mbuf_segs == 1) {
 793                                 /*
 794                                  * non-scattered packet,
 795                                  * transmit from one mbuf
 796                                  */
 797                                 /* copy part of packet to first area */
 798                                 rte_memcpy(tmp_dst,
 799                                         rte_pktmbuf_mtod(mbuf, const void *),
 800                                         write_len);
 801
 802                                 if (lck->next)
 803                                         dst = lck->next->start;
 804
 805                                 /* copy part of packet to second area */
 806                                 rte_memcpy(dst,
 807                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 808                                                         const uint8_t *) +
 809                                         write_len), pkt_len - write_len);
 810                         } else {
 811                                 /* scattered packet, transmit from more mbufs */
 812                                 struct rte_mbuf *m = mbuf;
 813                                 uint16_t written = 0;
 814                                 uint16_t to_write = 0;
 815                                 bool new_mbuf = true;
 816                                 uint16_t write_off = 0;
 817
 818                                 /* copy part of packet to first area */
 819                                 while (m && written < write_len) {
 820                                         to_write = RTE_MIN(m->data_len,
 821                                                         write_len - written);
 822                                         rte_memcpy(tmp_dst,
 823                                                 rte_pktmbuf_mtod(m,
 824                                                         const void *),
 825                                                 to_write);
 826
 827                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 828                                                 to_write;
 829                                         if (m->data_len <= write_len -
 830                                                         written) {
 831                                                 m = m->next;
 832                                                 new_mbuf = true;
 833                                         } else {
 834                                                 new_mbuf = false;
 835                                         }
 836                                         written += to_write;
 837                                 }
 838
 839                                 if (lck->next)
 840                                         dst = lck->next->start;
 841
 842                                 tmp_dst = dst;
 843                                 written = 0;
 844                                 write_off = new_mbuf ? 0 : to_write;
 845
 846                                 /* copy part of packet to second area */
 847                                 while (m && written < pkt_len - write_len) {
 848                                         rte_memcpy(tmp_dst, (const void *)
 849                                                 (rte_pktmbuf_mtod(m,
 850                                                 uint8_t *) + write_off),
 851                                                 m->data_len - write_off);
 852
 853                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 854                                                 (m->data_len - write_off);
 855                                         written += m->data_len - write_off;
 856                                         m = m->next;
 857                                         write_off = 0;
 858                                 }
 859                         }
 860
 861                         dst = ((uint8_t *)dst) + rem_len;
 862                         unlock_size += hwpkt_len;
 863                         lock_size = lock_size2 - rem_len;
 864                         lock_size2 = 0;
 865
 866                         rte_pktmbuf_free(mbuf);
 867                         num_tx++;
 868                 }
 869
 870                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 871                 pkt_left--;
 872         }
 873
 874         sze_q->tx_pkts += num_tx;
 875         sze_q->err_pkts += nb_pkts - num_tx;
 876         sze_q->tx_bytes += num_bytes;
 877         return num_tx;
 878 }
 879
 880 static int
 881 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
 882 {
 883         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 884         int ret;
 885         struct pmd_internals *internals = (struct pmd_internals *)
 886                 dev->data->dev_private;
 887
 888         if (rxq->sze == NULL) {
 889                 uint32_t rx = 1 << rxq->rx_channel;
 890                 uint32_t tx = 0;
 891                 rxq->sze = szedata_open(internals->sze_dev);
 892                 if (rxq->sze == NULL)
 893                         return -EINVAL;
 894                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
 895                 if (ret != 0 || rx == 0)
 896                         goto err;
 897         }
 898
 899         ret = szedata_start(rxq->sze);
 900         if (ret != 0)
 901                 goto err;
 902         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 903         return 0;
 904
 905 err:
 906         szedata_close(rxq->sze);
 907         rxq->sze = NULL;
 908         return -EINVAL;
 909 }
 910
 911 static int
 912 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
 913 {
 914         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 915
 916         if (rxq->sze != NULL) {
 917                 szedata_close(rxq->sze);
 918                 rxq->sze = NULL;
 919         }
 920
 921         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 922         return 0;
 923 }
 924
 925 static int
 926 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
 927 {
 928         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 929         int ret;
 930         struct pmd_internals *internals = (struct pmd_internals *)
 931                 dev->data->dev_private;
 932
 933         if (txq->sze == NULL) {
 934                 uint32_t rx = 0;
 935                 uint32_t tx = 1 << txq->tx_channel;
 936                 txq->sze = szedata_open(internals->sze_dev);
 937                 if (txq->sze == NULL)
 938                         return -EINVAL;
 939                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
 940                 if (ret != 0 || tx == 0)
 941                         goto err;
 942         }
 943
 944         ret = szedata_start(txq->sze);
 945         if (ret != 0)
 946                 goto err;
 947         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 948         return 0;
 949
 950 err:
 951         szedata_close(txq->sze);
 952         txq->sze = NULL;
 953         return -EINVAL;
 954 }
 955
 956 static int
 957 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
 958 {
 959         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 960
 961         if (txq->sze != NULL) {
 962                 szedata_close(txq->sze);
 963                 txq->sze = NULL;
 964         }
 965
 966         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 967         return 0;
 968 }
 969
 970 static int
 971 eth_dev_start(struct rte_eth_dev *dev)
 972 {
 973         int ret;
 974         uint16_t i;
 975         uint16_t nb_rx = dev->data->nb_rx_queues;
 976         uint16_t nb_tx = dev->data->nb_tx_queues;
 977
 978         for (i = 0; i < nb_rx; i++) {
 979                 ret = eth_rx_queue_start(dev, i);
 980                 if (ret != 0)
 981                         goto err_rx;
 982         }
 983
 984         for (i = 0; i < nb_tx; i++) {
 985                 ret = eth_tx_queue_start(dev, i);
 986                 if (ret != 0)
 987                         goto err_tx;
 988         }
 989
 990         return 0;
 991
 992 err_tx:
 993         for (i = 0; i < nb_tx; i++)
 994                 eth_tx_queue_stop(dev, i);
 995 err_rx:
 996         for (i = 0; i < nb_rx; i++)
 997                 eth_rx_queue_stop(dev, i);
 998         return ret;
 999 }
1000
1001 static void
1002 eth_dev_stop(struct rte_eth_dev *dev)
1003 {
1004         uint16_t i;
1005         uint16_t nb_rx = dev->data->nb_rx_queues;
1006         uint16_t nb_tx = dev->data->nb_tx_queues;
1007
1008         for (i = 0; i < nb_tx; i++)
1009                 eth_tx_queue_stop(dev, i);
1010
1011         for (i = 0; i < nb_rx; i++)
1012                 eth_rx_queue_stop(dev, i);
1013 }
1014
1015 static int
1016 eth_dev_configure(struct rte_eth_dev *dev)
1017 {
1018         struct rte_eth_dev_data *data = dev->data;
1019         if (data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) {
1020                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1021                 data->scattered_rx = 1;
1022         } else {
1023                 dev->rx_pkt_burst = eth_szedata2_rx;
1024                 data->scattered_rx = 0;
1025         }
1026         return 0;
1027 }
1028
1029 static void
1030 eth_dev_info(struct rte_eth_dev *dev,
1031                 struct rte_eth_dev_info *dev_info)
1032 {
1033         struct pmd_internals *internals = dev->data->dev_private;
1034
1035         dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1036         dev_info->if_index = 0;
1037         dev_info->max_mac_addrs = 1;
1038         dev_info->max_rx_pktlen = (uint32_t)-1;
1039         dev_info->max_rx_queues = internals->max_rx_queues;
1040         dev_info->max_tx_queues = internals->max_tx_queues;
1041         dev_info->min_rx_bufsize = 0;
1042         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_SCATTER;
1043         dev_info->tx_offload_capa = 0;
1044         dev_info->rx_queue_offload_capa = 0;
1045         dev_info->tx_queue_offload_capa = 0;
1046         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1047 }
1048
1049 static int
1050 eth_stats_get(struct rte_eth_dev *dev,
1051                 struct rte_eth_stats *stats)
1052 {
1053         uint16_t i;
1054         uint16_t nb_rx = dev->data->nb_rx_queues;
1055         uint16_t nb_tx = dev->data->nb_tx_queues;
1056         uint64_t rx_total = 0;
1057         uint64_t tx_total = 0;
1058         uint64_t tx_err_total = 0;
1059         uint64_t rx_total_bytes = 0;
1060         uint64_t tx_total_bytes = 0;
1061
1062         for (i = 0; i < nb_rx; i++) {
1063                 struct szedata2_rx_queue *rxq = dev->data->rx_queues[i];
1064
1065                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1066                         stats->q_ipackets[i] = rxq->rx_pkts;
1067                         stats->q_ibytes[i] = rxq->rx_bytes;
1068                 }
1069                 rx_total += rxq->rx_pkts;
1070                 rx_total_bytes += rxq->rx_bytes;
1071         }
1072
1073         for (i = 0; i < nb_tx; i++) {
1074                 struct szedata2_tx_queue *txq = dev->data->tx_queues[i];
1075
1076                 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1077                         stats->q_opackets[i] = txq->tx_pkts;
1078                         stats->q_obytes[i] = txq->tx_bytes;
1079                         stats->q_errors[i] = txq->err_pkts;
1080                 }
1081                 tx_total += txq->tx_pkts;
1082                 tx_total_bytes += txq->tx_bytes;
1083                 tx_err_total += txq->err_pkts;
1084         }
1085
1086         stats->ipackets = rx_total;
1087         stats->opackets = tx_total;
1088         stats->ibytes = rx_total_bytes;
1089         stats->obytes = tx_total_bytes;
1090         stats->oerrors = tx_err_total;
1091
1092         return 0;
1093 }
1094
1095 static void
1096 eth_stats_reset(struct rte_eth_dev *dev)
1097 {
1098         uint16_t i;
1099         uint16_t nb_rx = dev->data->nb_rx_queues;
1100         uint16_t nb_tx = dev->data->nb_tx_queues;
1101         struct pmd_internals *internals = dev->data->dev_private;
1102
1103         for (i = 0; i < nb_rx; i++) {
1104                 internals->rx_queue[i].rx_pkts = 0;
1105                 internals->rx_queue[i].rx_bytes = 0;
1106                 internals->rx_queue[i].err_pkts = 0;
1107         }
1108         for (i = 0; i < nb_tx; i++) {
1109                 internals->tx_queue[i].tx_pkts = 0;
1110                 internals->tx_queue[i].tx_bytes = 0;
1111                 internals->tx_queue[i].err_pkts = 0;
1112         }
1113 }
1114
1115 static void
1116 eth_rx_queue_release(void *q)
1117 {
1118         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1119         if (rxq->sze != NULL) {
1120                 szedata_close(rxq->sze);
1121                 rxq->sze = NULL;
1122         }
1123 }
1124
1125 static void
1126 eth_tx_queue_release(void *q)
1127 {
1128         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1129         if (txq->sze != NULL) {
1130                 szedata_close(txq->sze);
1131                 txq->sze = NULL;
1132         }
1133 }
1134
1135 static void
1136 eth_dev_close(struct rte_eth_dev *dev)
1137 {
1138         uint16_t i;
1139         uint16_t nb_rx = dev->data->nb_rx_queues;
1140         uint16_t nb_tx = dev->data->nb_tx_queues;
1141
1142         eth_dev_stop(dev);
1143
1144         for (i = 0; i < nb_rx; i++) {
1145                 eth_rx_queue_release(dev->data->rx_queues[i]);
1146                 dev->data->rx_queues[i] = NULL;
1147         }
1148         dev->data->nb_rx_queues = 0;
1149         for (i = 0; i < nb_tx; i++) {
1150                 eth_tx_queue_release(dev->data->tx_queues[i]);
1151                 dev->data->tx_queues[i] = NULL;
1152         }
1153         dev->data->nb_tx_queues = 0;
1154 }
1155
1156 /**
1157  * Function takes value from first IBUF status register.
1158  * Values in IBUF and OBUF should be same.
1159  *
1160  * @param internals
1161  *     Pointer to device private structure.
1162  * @return
1163  *     Link speed constant.
1164  */
1165 static inline enum szedata2_link_speed
1166 get_link_speed(const struct pmd_internals *internals)
1167 {
1168         const volatile struct szedata2_ibuf *ibuf =
1169                 ibuf_ptr_by_index(internals->pci_rsc, 0);
1170         uint32_t speed = (szedata2_read32(&ibuf->ibuf_st) & 0x70) >> 4;
1171         switch (speed) {
1172         case 0x03:
1173                 return SZEDATA2_LINK_SPEED_10G;
1174         case 0x04:
1175                 return SZEDATA2_LINK_SPEED_40G;
1176         case 0x05:
1177                 return SZEDATA2_LINK_SPEED_100G;
1178         default:
1179                 return SZEDATA2_LINK_SPEED_DEFAULT;
1180         }
1181 }
1182
1183 static int
1184 eth_link_update(struct rte_eth_dev *dev,
1185                 int wait_to_complete __rte_unused)
1186 {
1187         struct rte_eth_link link;
1188         struct pmd_internals *internals = (struct pmd_internals *)
1189                 dev->data->dev_private;
1190         const volatile struct szedata2_ibuf *ibuf;
1191         uint32_t i;
1192         bool link_is_up = false;
1193
1194         memset(&link, 0, sizeof(link));
1195
1196         switch (get_link_speed(internals)) {
1197         case SZEDATA2_LINK_SPEED_10G:
1198                 link.link_speed = ETH_SPEED_NUM_10G;
1199                 break;
1200         case SZEDATA2_LINK_SPEED_40G:
1201                 link.link_speed = ETH_SPEED_NUM_40G;
1202                 break;
1203         case SZEDATA2_LINK_SPEED_100G:
1204                 link.link_speed = ETH_SPEED_NUM_100G;
1205                 break;
1206         default:
1207                 link.link_speed = ETH_SPEED_NUM_10G;
1208                 break;
1209         }
1210
1211         /* szedata2 uses only full duplex */
1212         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1213
1214         for (i = 0; i < szedata2_ibuf_count; i++) {
1215                 ibuf = ibuf_ptr_by_index(internals->pci_rsc, i);
1216                 /*
1217                  * Link is considered up if at least one ibuf is enabled
1218                  * and up.
1219                  */
1220                 if (ibuf_is_enabled(ibuf) && ibuf_is_link_up(ibuf)) {
1221                         link_is_up = true;
1222                         break;
1223                 }
1224         }
1225
1226         link.link_status = link_is_up ? ETH_LINK_UP : ETH_LINK_DOWN;
1227
1228         link.link_autoneg = ETH_LINK_FIXED;
1229
1230         rte_eth_linkstatus_set(dev, &link);
1231         return 0;
1232 }
1233
1234 static int
1235 eth_dev_set_link_up(struct rte_eth_dev *dev)
1236 {
1237         struct pmd_internals *internals = (struct pmd_internals *)
1238                 dev->data->dev_private;
1239         uint32_t i;
1240
1241         for (i = 0; i < szedata2_ibuf_count; i++)
1242                 ibuf_enable(ibuf_ptr_by_index(internals->pci_rsc, i));
1243         for (i = 0; i < szedata2_obuf_count; i++)
1244                 obuf_enable(obuf_ptr_by_index(internals->pci_rsc, i));
1245         return 0;
1246 }
1247
1248 static int
1249 eth_dev_set_link_down(struct rte_eth_dev *dev)
1250 {
1251         struct pmd_internals *internals = (struct pmd_internals *)
1252                 dev->data->dev_private;
1253         uint32_t i;
1254
1255         for (i = 0; i < szedata2_ibuf_count; i++)
1256                 ibuf_disable(ibuf_ptr_by_index(internals->pci_rsc, i));
1257         for (i = 0; i < szedata2_obuf_count; i++)
1258                 obuf_disable(obuf_ptr_by_index(internals->pci_rsc, i));
1259         return 0;
1260 }
1261
1262 static int
1263 eth_rx_queue_setup(struct rte_eth_dev *dev,
1264                 uint16_t rx_queue_id,
1265                 uint16_t nb_rx_desc __rte_unused,
1266                 unsigned int socket_id __rte_unused,
1267                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1268                 struct rte_mempool *mb_pool)
1269 {
1270         struct pmd_internals *internals = dev->data->dev_private;
1271         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1272         int ret;
1273         uint32_t rx = 1 << rx_queue_id;
1274         uint32_t tx = 0;
1275
1276         rxq->sze = szedata_open(internals->sze_dev);
1277         if (rxq->sze == NULL)
1278                 return -EINVAL;
1279         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1280         if (ret != 0 || rx == 0) {
1281                 szedata_close(rxq->sze);
1282                 rxq->sze = NULL;
1283                 return -EINVAL;
1284         }
1285         rxq->rx_channel = rx_queue_id;
1286         rxq->in_port = dev->data->port_id;
1287         rxq->mb_pool = mb_pool;
1288         rxq->rx_pkts = 0;
1289         rxq->rx_bytes = 0;
1290         rxq->err_pkts = 0;
1291
1292         dev->data->rx_queues[rx_queue_id] = rxq;
1293         return 0;
1294 }
1295
1296 static int
1297 eth_tx_queue_setup(struct rte_eth_dev *dev,
1298                 uint16_t tx_queue_id,
1299                 uint16_t nb_tx_desc __rte_unused,
1300                 unsigned int socket_id __rte_unused,
1301                 const struct rte_eth_txconf *tx_conf __rte_unused)
1302 {
1303         struct pmd_internals *internals = dev->data->dev_private;
1304         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1305         int ret;
1306         uint32_t rx = 0;
1307         uint32_t tx = 1 << tx_queue_id;
1308
1309         txq->sze = szedata_open(internals->sze_dev);
1310         if (txq->sze == NULL)
1311                 return -EINVAL;
1312         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1313         if (ret != 0 || tx == 0) {
1314                 szedata_close(txq->sze);
1315                 txq->sze = NULL;
1316                 return -EINVAL;
1317         }
1318         txq->tx_channel = tx_queue_id;
1319         txq->tx_pkts = 0;
1320         txq->tx_bytes = 0;
1321         txq->err_pkts = 0;
1322
1323         dev->data->tx_queues[tx_queue_id] = txq;
1324         return 0;
1325 }
1326
1327 static void
1328 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1329                 struct ether_addr *mac_addr __rte_unused)
1330 {
1331 }
1332
1333 static void
1334 eth_promiscuous_enable(struct rte_eth_dev *dev)
1335 {
1336         struct pmd_internals *internals = (struct pmd_internals *)
1337                 dev->data->dev_private;
1338         uint32_t i;
1339
1340         for (i = 0; i < szedata2_ibuf_count; i++) {
1341                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1342                                 SZEDATA2_MAC_CHMODE_PROMISC);
1343         }
1344 }
1345
1346 static void
1347 eth_promiscuous_disable(struct rte_eth_dev *dev)
1348 {
1349         struct pmd_internals *internals = (struct pmd_internals *)
1350                 dev->data->dev_private;
1351         uint32_t i;
1352
1353         for (i = 0; i < szedata2_ibuf_count; i++) {
1354                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1355                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1356         }
1357 }
1358
1359 static void
1360 eth_allmulticast_enable(struct rte_eth_dev *dev)
1361 {
1362         struct pmd_internals *internals = (struct pmd_internals *)
1363                 dev->data->dev_private;
1364         uint32_t i;
1365
1366         for (i = 0; i < szedata2_ibuf_count; i++) {
1367                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1368                                 SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1369         }
1370 }
1371
1372 static void
1373 eth_allmulticast_disable(struct rte_eth_dev *dev)
1374 {
1375         struct pmd_internals *internals = (struct pmd_internals *)
1376                 dev->data->dev_private;
1377         uint32_t i;
1378
1379         for (i = 0; i < szedata2_ibuf_count; i++) {
1380                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1381                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1382         }
1383 }
1384
1385 static const struct eth_dev_ops ops = {
1386         .dev_start          = eth_dev_start,
1387         .dev_stop           = eth_dev_stop,
1388         .dev_set_link_up    = eth_dev_set_link_up,
1389         .dev_set_link_down  = eth_dev_set_link_down,
1390         .dev_close          = eth_dev_close,
1391         .dev_configure      = eth_dev_configure,
1392         .dev_infos_get      = eth_dev_info,
1393         .promiscuous_enable   = eth_promiscuous_enable,
1394         .promiscuous_disable  = eth_promiscuous_disable,
1395         .allmulticast_enable  = eth_allmulticast_enable,
1396         .allmulticast_disable = eth_allmulticast_disable,
1397         .rx_queue_start     = eth_rx_queue_start,
1398         .rx_queue_stop      = eth_rx_queue_stop,
1399         .tx_queue_start     = eth_tx_queue_start,
1400         .tx_queue_stop      = eth_tx_queue_stop,
1401         .rx_queue_setup     = eth_rx_queue_setup,
1402         .tx_queue_setup     = eth_tx_queue_setup,
1403         .rx_queue_release   = eth_rx_queue_release,
1404         .tx_queue_release   = eth_tx_queue_release,
1405         .link_update        = eth_link_update,
1406         .stats_get          = eth_stats_get,
1407         .stats_reset        = eth_stats_reset,
1408         .mac_addr_set       = eth_mac_addr_set,
1409 };
1410
1411 /*
1412  * This function goes through sysfs and looks for an index of szedata2
1413  * device file (/dev/szedataIIX, where X is the index).
1414  *
1415  * @return
1416  *           0 on success
1417  *          -1 on error
1418  */
1419 static int
1420 get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index)
1421 {
1422         DIR *dir;
1423         struct dirent *entry;
1424         int ret;
1425         uint32_t tmp_index;
1426         FILE *fd;
1427         char pcislot_path[PATH_MAX];
1428         uint32_t domain;
1429         uint32_t bus;
1430         uint32_t devid;
1431         uint32_t function;
1432
1433         dir = opendir("/sys/class/combo");
1434         if (dir == NULL)
1435                 return -1;
1436
1437         /*
1438          * Iterate through all combosixX directories.
1439          * When the value in /sys/class/combo/combosixX/device/pcislot
1440          * file is the location of the ethernet device dev, "X" is the
1441          * index of the device.
1442          */
1443         while ((entry = readdir(dir)) != NULL) {
1444                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1445                 if (ret != 1)
1446                         continue;
1447
1448                 snprintf(pcislot_path, PATH_MAX,
1449                         "/sys/class/combo/combosix%u/device/pcislot",
1450                         tmp_index);
1451
1452                 fd = fopen(pcislot_path, "r");
1453                 if (fd == NULL)
1454                         continue;
1455
1456                 ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8,
1457                                 &domain, &bus, &devid, &function);
1458                 fclose(fd);
1459                 if (ret != 4)
1460                         continue;
1461
1462                 if (pcislot_addr->domain == domain &&
1463                                 pcislot_addr->bus == bus &&
1464                                 pcislot_addr->devid == devid &&
1465                                 pcislot_addr->function == function) {
1466                         *index = tmp_index;
1467                         closedir(dir);
1468                         return 0;
1469                 }
1470         }
1471
1472         closedir(dir);
1473         return -1;
1474 }
1475
1476 static int
1477 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1478 {
1479         struct rte_eth_dev_data *data = dev->data;
1480         struct pmd_internals *internals = (struct pmd_internals *)
1481                 data->dev_private;
1482         struct szedata *szedata_temp;
1483         int ret;
1484         uint32_t szedata2_index;
1485         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1486         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1487         struct rte_mem_resource *pci_rsc =
1488                 &pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1489         char rsc_filename[PATH_MAX];
1490         void *pci_resource_ptr = NULL;
1491         int fd;
1492
1493         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1494                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1495                         pci_addr->function);
1496
1497         /* Get index of szedata2 device file and create path to device file */
1498         ret = get_szedata2_index(pci_addr, &szedata2_index);
1499         if (ret != 0) {
1500                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1501                 return -ENODEV;
1502         }
1503         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1504                         szedata2_index);
1505
1506         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1507
1508         /*
1509          * Get number of available DMA RX and TX channels, which is maximum
1510          * number of queues that can be created and store it in private device
1511          * data structure.
1512          */
1513         szedata_temp = szedata_open(internals->sze_dev);
1514         if (szedata_temp == NULL) {
1515                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1516                                 internals->sze_dev);
1517                 return -EINVAL;
1518         }
1519         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1520                         SZE2_DIR_RX);
1521         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1522                         SZE2_DIR_TX);
1523         szedata_close(szedata_temp);
1524
1525         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1526                         internals->max_rx_queues, internals->max_tx_queues);
1527
1528         /* Set rx, tx burst functions */
1529         if (data->scattered_rx == 1)
1530                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1531         else
1532                 dev->rx_pkt_burst = eth_szedata2_rx;
1533         dev->tx_pkt_burst = eth_szedata2_tx;
1534
1535         /* Set function callbacks for Ethernet API */
1536         dev->dev_ops = &ops;
1537
1538         rte_eth_copy_pci_info(dev, pci_dev);
1539
1540         /* mmap pci resource0 file to rte_mem_resource structure */
1541         if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1542                         0) {
1543                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1544                                 PCI_RESOURCE_NUMBER);
1545                 return -EINVAL;
1546         }
1547         snprintf(rsc_filename, PATH_MAX,
1548                 "%s/" PCI_PRI_FMT "/resource%u", rte_pci_get_sysfs_path(),
1549                 pci_addr->domain, pci_addr->bus,
1550                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1551         fd = open(rsc_filename, O_RDWR);
1552         if (fd < 0) {
1553                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1554                 return -EINVAL;
1555         }
1556
1557         pci_resource_ptr = mmap(0,
1558                         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1559                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1560         close(fd);
1561         if (pci_resource_ptr == MAP_FAILED) {
1562                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1563                                 rsc_filename, fd);
1564                 return -EINVAL;
1565         }
1566         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr;
1567         internals->pci_rsc = pci_rsc;
1568
1569         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1570                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1571                         (unsigned long long)pci_rsc->phys_addr,
1572                         (unsigned long long)pci_rsc->len,
1573                         (unsigned long long)pci_rsc->addr);
1574
1575         /* Get link state */
1576         eth_link_update(dev, 0);
1577
1578         /* Allocate space for one mac address */
1579         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1580                         RTE_CACHE_LINE_SIZE);
1581         if (data->mac_addrs == NULL) {
1582                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1583                 munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1584                        pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1585                 return -EINVAL;
1586         }
1587
1588         ether_addr_copy(&eth_addr, data->mac_addrs);
1589
1590         /* At initial state COMBO card is in promiscuous mode so disable it */
1591         eth_promiscuous_disable(dev);
1592
1593         RTE_LOG(INFO, PMD, "szedata2 device ("
1594                         PCI_PRI_FMT ") successfully initialized\n",
1595                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1596                         pci_addr->function);
1597
1598         return 0;
1599 }
1600
1601 static int
1602 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1603 {
1604         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1605         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1606
1607         rte_free(dev->data->mac_addrs);
1608         dev->data->mac_addrs = NULL;
1609         munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1610                pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1611
1612         RTE_LOG(INFO, PMD, "szedata2 device ("
1613                         PCI_PRI_FMT ") successfully uninitialized\n",
1614                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1615                         pci_addr->function);
1616
1617         return 0;
1618 }
1619
1620 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1621         {
1622                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1623                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1624         },
1625         {
1626                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1627                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1628         },
1629         {
1630                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1631                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1632         },
1633         {
1634                 .vendor_id = 0,
1635         }
1636 };
1637
1638 static int szedata2_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1639         struct rte_pci_device *pci_dev)
1640 {
1641         return rte_eth_dev_pci_generic_probe(pci_dev,
1642                 sizeof(struct pmd_internals), rte_szedata2_eth_dev_init);
1643 }
1644
1645 static int szedata2_eth_pci_remove(struct rte_pci_device *pci_dev)
1646 {
1647         return rte_eth_dev_pci_generic_remove(pci_dev,
1648                 rte_szedata2_eth_dev_uninit);
1649 }
1650
1651 static struct rte_pci_driver szedata2_eth_driver = {
1652         .id_table = rte_szedata2_pci_id_table,
1653         .probe = szedata2_eth_pci_probe,
1654         .remove = szedata2_eth_pci_remove,
1655 };
1656
1657 RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver);
1658 RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1659 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
1660         "* combo6core & combov3 & szedata2 & szedata2_cv3");