drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 - 2016 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38 #include <sys/types.h>
  39 #include <dirent.h>
  40 #include <sys/stat.h>
  41 #include <fcntl.h>
  42 #include <sys/mman.h>
  43
  44 #include <libsze2.h>
  45
  46 #include <rte_mbuf.h>
  47 #include <rte_ethdev_driver.h>
  48 #include <rte_ethdev_pci.h>
  49 #include <rte_malloc.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_kvargs.h>
  52 #include <rte_dev.h>
  53
  54 #include "rte_eth_szedata2.h"
  55 #include "szedata2_iobuf.h"
  56
  57 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  58 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  59 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  60
  61 /**
  62  * size of szedata2_packet header with alignment
  63  */
  64 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  65
  66 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2
  67
  68 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
  69
  70 struct szedata2_rx_queue {
  71         struct szedata *sze;
  72         uint8_t rx_channel;
  73         uint16_t in_port;
  74         struct rte_mempool *mb_pool;
  75         volatile uint64_t rx_pkts;
  76         volatile uint64_t rx_bytes;
  77         volatile uint64_t err_pkts;
  78 };
  79
  80 struct szedata2_tx_queue {
  81         struct szedata *sze;
  82         uint8_t tx_channel;
  83         volatile uint64_t tx_pkts;
  84         volatile uint64_t tx_bytes;
  85         volatile uint64_t err_pkts;
  86 };
  87
  88 struct pmd_internals {
  89         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  90         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  91         uint16_t max_rx_queues;
  92         uint16_t max_tx_queues;
  93         char sze_dev[PATH_MAX];
  94         struct rte_mem_resource *pci_rsc;
  95 };
  96
  97 static struct ether_addr eth_addr = {
  98         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
  99 };
 100
 101 static uint16_t
 102 eth_szedata2_rx(void *queue,
 103                 struct rte_mbuf **bufs,
 104                 uint16_t nb_pkts)
 105 {
 106         unsigned int i;
 107         struct rte_mbuf *mbuf;
 108         struct szedata2_rx_queue *sze_q = queue;
 109         struct rte_pktmbuf_pool_private *mbp_priv;
 110         uint16_t num_rx = 0;
 111         uint16_t buf_size;
 112         uint16_t sg_size;
 113         uint16_t hw_size;
 114         uint16_t packet_size;
 115         uint64_t num_bytes = 0;
 116         struct szedata *sze = sze_q->sze;
 117         uint8_t *header_ptr = NULL; /* header of packet */
 118         uint8_t *packet_ptr1 = NULL;
 119         uint8_t *packet_ptr2 = NULL;
 120         uint16_t packet_len1 = 0;
 121         uint16_t packet_len2 = 0;
 122         uint16_t hw_data_align;
 123
 124         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 125                 return 0;
 126
 127         /*
 128          * Reads the given number of packets from szedata2 channel given
 129          * by queue and copies the packet data into a newly allocated mbuf
 130          * to return.
 131          */
 132         for (i = 0; i < nb_pkts; i++) {
 133                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 134
 135                 if (unlikely(mbuf == NULL))
 136                         break;
 137
 138                 /* get the next sze packet */
 139                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 140                                 sze->ct_rx_lck->next == NULL) {
 141                         /* unlock old data */
 142                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 143                         sze->ct_rx_lck_orig = NULL;
 144                         sze->ct_rx_lck = NULL;
 145                 }
 146
 147                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 148                         /* nothing to read, lock new data */
 149                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 150                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 151
 152                         if (sze->ct_rx_lck == NULL) {
 153                                 /* nothing to lock */
 154                                 rte_pktmbuf_free(mbuf);
 155                                 break;
 156                         }
 157
 158                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 159                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 160
 161                         if (!sze->ct_rx_rem_bytes) {
 162                                 rte_pktmbuf_free(mbuf);
 163                                 break;
 164                         }
 165                 }
 166
 167                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 168                         /*
 169                          * cut in header
 170                          * copy parts of header to merge buffer
 171                          */
 172                         if (sze->ct_rx_lck->next == NULL) {
 173                                 rte_pktmbuf_free(mbuf);
 174                                 break;
 175                         }
 176
 177                         /* copy first part of header */
 178                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 179                                         sze->ct_rx_rem_bytes);
 180
 181                         /* copy second part of header */
 182                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 183                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 184                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 185                                 sze->ct_rx_cur_ptr,
 186                                 RTE_SZE2_PACKET_HEADER_SIZE -
 187                                 sze->ct_rx_rem_bytes);
 188
 189                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 190                                 sze->ct_rx_rem_bytes;
 191                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 192                                 RTE_SZE2_PACKET_HEADER_SIZE +
 193                                 sze->ct_rx_rem_bytes;
 194
 195                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 196                 } else {
 197                         /* not cut */
 198                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 199                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 200                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 201                 }
 202
 203                 sg_size = le16toh(*((uint16_t *)header_ptr));
 204                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 205                 packet_size = sg_size -
 206                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 207
 208
 209                 /* checks if packet all right */
 210                 if (!sg_size)
 211                         errx(5, "Zero segsize");
 212
 213                 /* check sg_size and hwsize */
 214                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 215                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 216                                 "hwsize: %d", sg_size, hw_size);
 217                 }
 218
 219                 hw_data_align =
 220                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 221                         RTE_SZE2_PACKET_HEADER_SIZE;
 222
 223                 if (sze->ct_rx_rem_bytes >=
 224                                 (uint16_t)(sg_size -
 225                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 226                         /* no cut */
 227                         /* one packet ready - go to another */
 228                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 229                         packet_len1 = packet_size;
 230                         packet_ptr2 = NULL;
 231                         packet_len2 = 0;
 232
 233                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 234                                 RTE_SZE2_PACKET_HEADER_SIZE;
 235                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 236                                 RTE_SZE2_PACKET_HEADER_SIZE;
 237                 } else {
 238                         /* cut in data */
 239                         if (sze->ct_rx_lck->next == NULL) {
 240                                 errx(6, "Need \"next\" lock, "
 241                                         "but it is missing: %u",
 242                                         sze->ct_rx_rem_bytes);
 243                         }
 244
 245                         /* skip hw data */
 246                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 247                                 uint16_t rem_size = hw_data_align -
 248                                         sze->ct_rx_rem_bytes;
 249
 250                                 /* MOVE to next lock */
 251                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 252                                 sze->ct_rx_cur_ptr =
 253                                         (void *)(((uint8_t *)
 254                                         (sze->ct_rx_lck->start)) + rem_size);
 255
 256                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 257                                 packet_len1 = packet_size;
 258                                 packet_ptr2 = NULL;
 259                                 packet_len2 = 0;
 260
 261                                 sze->ct_rx_cur_ptr +=
 262                                         RTE_SZE2_ALIGN8(packet_size);
 263                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 264                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 265                         } else {
 266                                 /* get pointer and length from first part */
 267                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 268                                         hw_data_align;
 269                                 packet_len1 = sze->ct_rx_rem_bytes -
 270                                         hw_data_align;
 271
 272                                 /* MOVE to next lock */
 273                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 274                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 275
 276                                 /* get pointer and length from second part */
 277                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 278                                 packet_len2 = packet_size - packet_len1;
 279
 280                                 sze->ct_rx_cur_ptr +=
 281                                         RTE_SZE2_ALIGN8(packet_size) -
 282                                         packet_len1;
 283                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 284                                         (RTE_SZE2_ALIGN8(packet_size) -
 285                                          packet_len1);
 286                         }
 287                 }
 288
 289                 if (unlikely(packet_ptr1 == NULL)) {
 290                         rte_pktmbuf_free(mbuf);
 291                         break;
 292                 }
 293
 294                 /* get the space available for data in the mbuf */
 295                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 296                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 297                                 RTE_PKTMBUF_HEADROOM);
 298
 299                 if (packet_size <= buf_size) {
 300                         /* sze packet will fit in one mbuf, go ahead and copy */
 301                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 302                                         packet_ptr1, packet_len1);
 303                         if (packet_ptr2 != NULL) {
 304                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 305                                         uint8_t *) + packet_len1),
 306                                         packet_ptr2, packet_len2);
 307                         }
 308                         mbuf->data_len = (uint16_t)packet_size;
 309
 310                         mbuf->pkt_len = packet_size;
 311                         mbuf->port = sze_q->in_port;
 312                         bufs[num_rx] = mbuf;
 313                         num_rx++;
 314                         num_bytes += packet_size;
 315                 } else {
 316                         /*
 317                          * sze packet will not fit in one mbuf,
 318                          * scattered mode is not enabled, drop packet
 319                          */
 320                         RTE_LOG(ERR, PMD,
 321                                 "SZE segment %d bytes will not fit in one mbuf "
 322                                 "(%d bytes), scattered mode is not enabled, "
 323                                 "drop packet!!\n",
 324                                 packet_size, buf_size);
 325                         rte_pktmbuf_free(mbuf);
 326                 }
 327         }
 328
 329         sze_q->rx_pkts += num_rx;
 330         sze_q->rx_bytes += num_bytes;
 331         return num_rx;
 332 }
 333
 334 static uint16_t
 335 eth_szedata2_rx_scattered(void *queue,
 336                 struct rte_mbuf **bufs,
 337                 uint16_t nb_pkts)
 338 {
 339         unsigned int i;
 340         struct rte_mbuf *mbuf;
 341         struct szedata2_rx_queue *sze_q = queue;
 342         struct rte_pktmbuf_pool_private *mbp_priv;
 343         uint16_t num_rx = 0;
 344         uint16_t buf_size;
 345         uint16_t sg_size;
 346         uint16_t hw_size;
 347         uint16_t packet_size;
 348         uint64_t num_bytes = 0;
 349         struct szedata *sze = sze_q->sze;
 350         uint8_t *header_ptr = NULL; /* header of packet */
 351         uint8_t *packet_ptr1 = NULL;
 352         uint8_t *packet_ptr2 = NULL;
 353         uint16_t packet_len1 = 0;
 354         uint16_t packet_len2 = 0;
 355         uint16_t hw_data_align;
 356
 357         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 358                 return 0;
 359
 360         /*
 361          * Reads the given number of packets from szedata2 channel given
 362          * by queue and copies the packet data into a newly allocated mbuf
 363          * to return.
 364          */
 365         for (i = 0; i < nb_pkts; i++) {
 366                 const struct szedata_lock *ct_rx_lck_backup;
 367                 unsigned int ct_rx_rem_bytes_backup;
 368                 unsigned char *ct_rx_cur_ptr_backup;
 369
 370                 /* get the next sze packet */
 371                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 372                                 sze->ct_rx_lck->next == NULL) {
 373                         /* unlock old data */
 374                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 375                         sze->ct_rx_lck_orig = NULL;
 376                         sze->ct_rx_lck = NULL;
 377                 }
 378
 379                 /*
 380                  * Store items from sze structure which can be changed
 381                  * before mbuf allocating. Use these items in case of mbuf
 382                  * allocating failure.
 383                  */
 384                 ct_rx_lck_backup = sze->ct_rx_lck;
 385                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 386                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 387
 388                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 389                         /* nothing to read, lock new data */
 390                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 391                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 392
 393                         /*
 394                          * Backup items from sze structure must be updated
 395                          * after locking to contain pointers to new locks.
 396                          */
 397                         ct_rx_lck_backup = sze->ct_rx_lck;
 398                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 399                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 400
 401                         if (sze->ct_rx_lck == NULL)
 402                                 /* nothing to lock */
 403                                 break;
 404
 405                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 406                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 407
 408                         if (!sze->ct_rx_rem_bytes)
 409                                 break;
 410                 }
 411
 412                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 413                         /*
 414                          * cut in header - copy parts of header to merge buffer
 415                          */
 416                         if (sze->ct_rx_lck->next == NULL)
 417                                 break;
 418
 419                         /* copy first part of header */
 420                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 421                                         sze->ct_rx_rem_bytes);
 422
 423                         /* copy second part of header */
 424                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 425                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 426                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 427                                 sze->ct_rx_cur_ptr,
 428                                 RTE_SZE2_PACKET_HEADER_SIZE -
 429                                 sze->ct_rx_rem_bytes);
 430
 431                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 432                                 sze->ct_rx_rem_bytes;
 433                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 434                                 RTE_SZE2_PACKET_HEADER_SIZE +
 435                                 sze->ct_rx_rem_bytes;
 436
 437                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 438                 } else {
 439                         /* not cut */
 440                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 441                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 442                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 443                 }
 444
 445                 sg_size = le16toh(*((uint16_t *)header_ptr));
 446                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 447                 packet_size = sg_size -
 448                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 449
 450
 451                 /* checks if packet all right */
 452                 if (!sg_size)
 453                         errx(5, "Zero segsize");
 454
 455                 /* check sg_size and hwsize */
 456                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 457                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 458                                         "hwsize: %d", sg_size, hw_size);
 459                 }
 460
 461                 hw_data_align =
 462                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 463                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 464
 465                 if (sze->ct_rx_rem_bytes >=
 466                                 (uint16_t)(sg_size -
 467                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 468                         /* no cut */
 469                         /* one packet ready - go to another */
 470                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 471                         packet_len1 = packet_size;
 472                         packet_ptr2 = NULL;
 473                         packet_len2 = 0;
 474
 475                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 476                                 RTE_SZE2_PACKET_HEADER_SIZE;
 477                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 478                                 RTE_SZE2_PACKET_HEADER_SIZE;
 479                 } else {
 480                         /* cut in data */
 481                         if (sze->ct_rx_lck->next == NULL) {
 482                                 errx(6, "Need \"next\" lock, but it is "
 483                                         "missing: %u", sze->ct_rx_rem_bytes);
 484                         }
 485
 486                         /* skip hw data */
 487                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 488                                 uint16_t rem_size = hw_data_align -
 489                                         sze->ct_rx_rem_bytes;
 490
 491                                 /* MOVE to next lock */
 492                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 493                                 sze->ct_rx_cur_ptr =
 494                                         (void *)(((uint8_t *)
 495                                         (sze->ct_rx_lck->start)) + rem_size);
 496
 497                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 498                                 packet_len1 = packet_size;
 499                                 packet_ptr2 = NULL;
 500                                 packet_len2 = 0;
 501
 502                                 sze->ct_rx_cur_ptr +=
 503                                         RTE_SZE2_ALIGN8(packet_size);
 504                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 505                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 506                         } else {
 507                                 /* get pointer and length from first part */
 508                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 509                                         hw_data_align;
 510                                 packet_len1 = sze->ct_rx_rem_bytes -
 511                                         hw_data_align;
 512
 513                                 /* MOVE to next lock */
 514                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 515                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 516
 517                                 /* get pointer and length from second part */
 518                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 519                                 packet_len2 = packet_size - packet_len1;
 520
 521                                 sze->ct_rx_cur_ptr +=
 522                                         RTE_SZE2_ALIGN8(packet_size) -
 523                                         packet_len1;
 524                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 525                                         (RTE_SZE2_ALIGN8(packet_size) -
 526                                          packet_len1);
 527                         }
 528                 }
 529
 530                 if (unlikely(packet_ptr1 == NULL))
 531                         break;
 532
 533                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 534
 535                 if (unlikely(mbuf == NULL)) {
 536                         /*
 537                          * Restore items from sze structure to state after
 538                          * unlocking (eventually locking).
 539                          */
 540                         sze->ct_rx_lck = ct_rx_lck_backup;
 541                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 542                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 543                         break;
 544                 }
 545
 546                 /* get the space available for data in the mbuf */
 547                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 548                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 549                                 RTE_PKTMBUF_HEADROOM);
 550
 551                 if (packet_size <= buf_size) {
 552                         /* sze packet will fit in one mbuf, go ahead and copy */
 553                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 554                                         packet_ptr1, packet_len1);
 555                         if (packet_ptr2 != NULL) {
 556                                 rte_memcpy((void *)
 557                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 558                                         packet_len1), packet_ptr2, packet_len2);
 559                         }
 560                         mbuf->data_len = (uint16_t)packet_size;
 561                 } else {
 562                         /*
 563                          * sze packet will not fit in one mbuf,
 564                          * scatter packet into more mbufs
 565                          */
 566                         struct rte_mbuf *m = mbuf;
 567                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 568
 569                         /* copy first part of packet */
 570                         /* fill first mbuf */
 571                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 572                                 len);
 573                         packet_len1 -= len;
 574                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 575
 576                         while (packet_len1 > 0) {
 577                                 /* fill new mbufs */
 578                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 579
 580                                 if (unlikely(m->next == NULL)) {
 581                                         rte_pktmbuf_free(mbuf);
 582                                         /*
 583                                          * Restore items from sze structure
 584                                          * to state after unlocking (eventually
 585                                          * locking).
 586                                          */
 587                                         sze->ct_rx_lck = ct_rx_lck_backup;
 588                                         sze->ct_rx_rem_bytes =
 589                                                 ct_rx_rem_bytes_backup;
 590                                         sze->ct_rx_cur_ptr =
 591                                                 ct_rx_cur_ptr_backup;
 592                                         goto finish;
 593                                 }
 594
 595                                 m = m->next;
 596
 597                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 598                                         packet_len1);
 599                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 600                                         packet_ptr1, len);
 601
 602                                 (mbuf->nb_segs)++;
 603                                 packet_len1 -= len;
 604                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 605                         }
 606
 607                         if (packet_ptr2 != NULL) {
 608                                 /* copy second part of packet, if exists */
 609                                 /* fill the rest of currently last mbuf */
 610                                 len = rte_pktmbuf_tailroom(m);
 611                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 612                                         packet_ptr2, len);
 613                                 packet_len2 -= len;
 614                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 615
 616                                 while (packet_len2 > 0) {
 617                                         /* fill new mbufs */
 618                                         m->next = rte_pktmbuf_alloc(
 619                                                         sze_q->mb_pool);
 620
 621                                         if (unlikely(m->next == NULL)) {
 622                                                 rte_pktmbuf_free(mbuf);
 623                                                 /*
 624                                                  * Restore items from sze
 625                                                  * structure to state after
 626                                                  * unlocking (eventually
 627                                                  * locking).
 628                                                  */
 629                                                 sze->ct_rx_lck =
 630                                                         ct_rx_lck_backup;
 631                                                 sze->ct_rx_rem_bytes =
 632                                                         ct_rx_rem_bytes_backup;
 633                                                 sze->ct_rx_cur_ptr =
 634                                                         ct_rx_cur_ptr_backup;
 635                                                 goto finish;
 636                                         }
 637
 638                                         m = m->next;
 639
 640                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 641                                                 packet_len2);
 642                                         rte_memcpy(
 643                                                 rte_pktmbuf_append(mbuf, len),
 644                                                 packet_ptr2, len);
 645
 646                                         (mbuf->nb_segs)++;
 647                                         packet_len2 -= len;
 648                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 649                                                 len;
 650                                 }
 651                         }
 652                 }
 653                 mbuf->pkt_len = packet_size;
 654                 mbuf->port = sze_q->in_port;
 655                 bufs[num_rx] = mbuf;
 656                 num_rx++;
 657                 num_bytes += packet_size;
 658         }
 659
 660 finish:
 661         sze_q->rx_pkts += num_rx;
 662         sze_q->rx_bytes += num_bytes;
 663         return num_rx;
 664 }
 665
 666 static uint16_t
 667 eth_szedata2_tx(void *queue,
 668                 struct rte_mbuf **bufs,
 669                 uint16_t nb_pkts)
 670 {
 671         struct rte_mbuf *mbuf;
 672         struct szedata2_tx_queue *sze_q = queue;
 673         uint16_t num_tx = 0;
 674         uint64_t num_bytes = 0;
 675
 676         const struct szedata_lock *lck;
 677         uint32_t lock_size;
 678         uint32_t lock_size2;
 679         void *dst;
 680         uint32_t pkt_len;
 681         uint32_t hwpkt_len;
 682         uint32_t unlock_size;
 683         uint32_t rem_len;
 684         uint16_t mbuf_segs;
 685         uint16_t pkt_left = nb_pkts;
 686
 687         if (sze_q->sze == NULL || nb_pkts == 0)
 688                 return 0;
 689
 690         while (pkt_left > 0) {
 691                 unlock_size = 0;
 692                 lck = szedata_tx_lock_data(sze_q->sze,
 693                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 694                         sze_q->tx_channel);
 695                 if (lck == NULL)
 696                         continue;
 697
 698                 dst = lck->start;
 699                 lock_size = lck->len;
 700                 lock_size2 = lck->next ? lck->next->len : 0;
 701
 702 next_packet:
 703                 mbuf = bufs[nb_pkts - pkt_left];
 704
 705                 pkt_len = mbuf->pkt_len;
 706                 mbuf_segs = mbuf->nb_segs;
 707
 708                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 709                         RTE_SZE2_ALIGN8(pkt_len);
 710
 711                 if (lock_size + lock_size2 < hwpkt_len) {
 712                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 713                         continue;
 714                 }
 715
 716                 num_bytes += pkt_len;
 717
 718                 if (lock_size > hwpkt_len) {
 719                         void *tmp_dst;
 720
 721                         rem_len = 0;
 722
 723                         /* write packet length at first 2 bytes in 8B header */
 724                         *((uint16_t *)dst) = htole16(
 725                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 726                                         pkt_len);
 727                         *(((uint16_t *)dst) + 1) = htole16(0);
 728
 729                         /* copy packet from mbuf */
 730                         tmp_dst = ((uint8_t *)(dst)) +
 731                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 732                         if (mbuf_segs == 1) {
 733                                 /*
 734                                  * non-scattered packet,
 735                                  * transmit from one mbuf
 736                                  */
 737                                 rte_memcpy(tmp_dst,
 738                                         rte_pktmbuf_mtod(mbuf, const void *),
 739                                         pkt_len);
 740                         } else {
 741                                 /* scattered packet, transmit from more mbufs */
 742                                 struct rte_mbuf *m = mbuf;
 743                                 while (m) {
 744                                         rte_memcpy(tmp_dst,
 745                                                 rte_pktmbuf_mtod(m,
 746                                                 const void *),
 747                                                 m->data_len);
 748                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 749                                                 m->data_len;
 750                                         m = m->next;
 751                                 }
 752                         }
 753
 754
 755                         dst = ((uint8_t *)dst) + hwpkt_len;
 756                         unlock_size += hwpkt_len;
 757                         lock_size -= hwpkt_len;
 758
 759                         rte_pktmbuf_free(mbuf);
 760                         num_tx++;
 761                         pkt_left--;
 762                         if (pkt_left == 0) {
 763                                 szedata_tx_unlock_data(sze_q->sze, lck,
 764                                         unlock_size);
 765                                 break;
 766                         }
 767                         goto next_packet;
 768                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 769                         void *tmp_dst;
 770                         uint16_t write_len;
 771
 772                         /* write packet length at first 2 bytes in 8B header */
 773                         *((uint16_t *)dst) =
 774                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 775                                         pkt_len);
 776                         *(((uint16_t *)dst) + 1) = htole16(0);
 777
 778                         /*
 779                          * If the raw packet (pkt_len) is smaller than lock_size
 780                          * get the correct length for memcpy
 781                          */
 782                         write_len =
 783                                 pkt_len < lock_size -
 784                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 785                                 pkt_len :
 786                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 787
 788                         rem_len = hwpkt_len - lock_size;
 789
 790                         tmp_dst = ((uint8_t *)(dst)) +
 791                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 792                         if (mbuf_segs == 1) {
 793                                 /*
 794                                  * non-scattered packet,
 795                                  * transmit from one mbuf
 796                                  */
 797                                 /* copy part of packet to first area */
 798                                 rte_memcpy(tmp_dst,
 799                                         rte_pktmbuf_mtod(mbuf, const void *),
 800                                         write_len);
 801
 802                                 if (lck->next)
 803                                         dst = lck->next->start;
 804
 805                                 /* copy part of packet to second area */
 806                                 rte_memcpy(dst,
 807                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 808                                                         const uint8_t *) +
 809                                         write_len), pkt_len - write_len);
 810                         } else {
 811                                 /* scattered packet, transmit from more mbufs */
 812                                 struct rte_mbuf *m = mbuf;
 813                                 uint16_t written = 0;
 814                                 uint16_t to_write = 0;
 815                                 bool new_mbuf = true;
 816                                 uint16_t write_off = 0;
 817
 818                                 /* copy part of packet to first area */
 819                                 while (m && written < write_len) {
 820                                         to_write = RTE_MIN(m->data_len,
 821                                                         write_len - written);
 822                                         rte_memcpy(tmp_dst,
 823                                                 rte_pktmbuf_mtod(m,
 824                                                         const void *),
 825                                                 to_write);
 826
 827                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 828                                                 to_write;
 829                                         if (m->data_len <= write_len -
 830                                                         written) {
 831                                                 m = m->next;
 832                                                 new_mbuf = true;
 833                                         } else {
 834                                                 new_mbuf = false;
 835                                         }
 836                                         written += to_write;
 837                                 }
 838
 839                                 if (lck->next)
 840                                         dst = lck->next->start;
 841
 842                                 tmp_dst = dst;
 843                                 written = 0;
 844                                 write_off = new_mbuf ? 0 : to_write;
 845
 846                                 /* copy part of packet to second area */
 847                                 while (m && written < pkt_len - write_len) {
 848                                         rte_memcpy(tmp_dst, (const void *)
 849                                                 (rte_pktmbuf_mtod(m,
 850                                                 uint8_t *) + write_off),
 851                                                 m->data_len - write_off);
 852
 853                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 854                                                 (m->data_len - write_off);
 855                                         written += m->data_len - write_off;
 856                                         m = m->next;
 857                                         write_off = 0;
 858                                 }
 859                         }
 860
 861                         dst = ((uint8_t *)dst) + rem_len;
 862                         unlock_size += hwpkt_len;
 863                         lock_size = lock_size2 - rem_len;
 864                         lock_size2 = 0;
 865
 866                         rte_pktmbuf_free(mbuf);
 867                         num_tx++;
 868                 }
 869
 870                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 871                 pkt_left--;
 872         }
 873
 874         sze_q->tx_pkts += num_tx;
 875         sze_q->err_pkts += nb_pkts - num_tx;
 876         sze_q->tx_bytes += num_bytes;
 877         return num_tx;
 878 }
 879
 880 static int
 881 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
 882 {
 883         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 884         int ret;
 885         struct pmd_internals *internals = (struct pmd_internals *)
 886                 dev->data->dev_private;
 887
 888         if (rxq->sze == NULL) {
 889                 uint32_t rx = 1 << rxq->rx_channel;
 890                 uint32_t tx = 0;
 891                 rxq->sze = szedata_open(internals->sze_dev);
 892                 if (rxq->sze == NULL)
 893                         return -EINVAL;
 894                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
 895                 if (ret != 0 || rx == 0)
 896                         goto err;
 897         }
 898
 899         ret = szedata_start(rxq->sze);
 900         if (ret != 0)
 901                 goto err;
 902         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 903         return 0;
 904
 905 err:
 906         szedata_close(rxq->sze);
 907         rxq->sze = NULL;
 908         return -EINVAL;
 909 }
 910
 911 static int
 912 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
 913 {
 914         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 915
 916         if (rxq->sze != NULL) {
 917                 szedata_close(rxq->sze);
 918                 rxq->sze = NULL;
 919         }
 920
 921         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 922         return 0;
 923 }
 924
 925 static int
 926 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
 927 {
 928         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 929         int ret;
 930         struct pmd_internals *internals = (struct pmd_internals *)
 931                 dev->data->dev_private;
 932
 933         if (txq->sze == NULL) {
 934                 uint32_t rx = 0;
 935                 uint32_t tx = 1 << txq->tx_channel;
 936                 txq->sze = szedata_open(internals->sze_dev);
 937                 if (txq->sze == NULL)
 938                         return -EINVAL;
 939                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
 940                 if (ret != 0 || tx == 0)
 941                         goto err;
 942         }
 943
 944         ret = szedata_start(txq->sze);
 945         if (ret != 0)
 946                 goto err;
 947         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 948         return 0;
 949
 950 err:
 951         szedata_close(txq->sze);
 952         txq->sze = NULL;
 953         return -EINVAL;
 954 }
 955
 956 static int
 957 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
 958 {
 959         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 960
 961         if (txq->sze != NULL) {
 962                 szedata_close(txq->sze);
 963                 txq->sze = NULL;
 964         }
 965
 966         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 967         return 0;
 968 }
 969
 970 static int
 971 eth_dev_start(struct rte_eth_dev *dev)
 972 {
 973         int ret;
 974         uint16_t i;
 975         uint16_t nb_rx = dev->data->nb_rx_queues;
 976         uint16_t nb_tx = dev->data->nb_tx_queues;
 977
 978         for (i = 0; i < nb_rx; i++) {
 979                 ret = eth_rx_queue_start(dev, i);
 980                 if (ret != 0)
 981                         goto err_rx;
 982         }
 983
 984         for (i = 0; i < nb_tx; i++) {
 985                 ret = eth_tx_queue_start(dev, i);
 986                 if (ret != 0)
 987                         goto err_tx;
 988         }
 989
 990         return 0;
 991
 992 err_tx:
 993         for (i = 0; i < nb_tx; i++)
 994                 eth_tx_queue_stop(dev, i);
 995 err_rx:
 996         for (i = 0; i < nb_rx; i++)
 997                 eth_rx_queue_stop(dev, i);
 998         return ret;
 999 }
1000
1001 static void
1002 eth_dev_stop(struct rte_eth_dev *dev)
1003 {
1004         uint16_t i;
1005         uint16_t nb_rx = dev->data->nb_rx_queues;
1006         uint16_t nb_tx = dev->data->nb_tx_queues;
1007
1008         for (i = 0; i < nb_tx; i++)
1009                 eth_tx_queue_stop(dev, i);
1010
1011         for (i = 0; i < nb_rx; i++)
1012                 eth_rx_queue_stop(dev, i);
1013 }
1014
1015 static int
1016 eth_dev_configure(struct rte_eth_dev *dev)
1017 {
1018         struct rte_eth_dev_data *data = dev->data;
1019         if (data->dev_conf.rxmode.enable_scatter == 1) {
1020                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1021                 data->scattered_rx = 1;
1022         } else {
1023                 dev->rx_pkt_burst = eth_szedata2_rx;
1024                 data->scattered_rx = 0;
1025         }
1026         return 0;
1027 }
1028
1029 static void
1030 eth_dev_info(struct rte_eth_dev *dev,
1031                 struct rte_eth_dev_info *dev_info)
1032 {
1033         struct pmd_internals *internals = dev->data->dev_private;
1034         dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1035         dev_info->if_index = 0;
1036         dev_info->max_mac_addrs = 1;
1037         dev_info->max_rx_pktlen = (uint32_t)-1;
1038         dev_info->max_rx_queues = internals->max_rx_queues;
1039         dev_info->max_tx_queues = internals->max_tx_queues;
1040         dev_info->min_rx_bufsize = 0;
1041         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1042 }
1043
1044 static int
1045 eth_stats_get(struct rte_eth_dev *dev,
1046                 struct rte_eth_stats *stats)
1047 {
1048         uint16_t i;
1049         uint16_t nb_rx = dev->data->nb_rx_queues;
1050         uint16_t nb_tx = dev->data->nb_tx_queues;
1051         uint64_t rx_total = 0;
1052         uint64_t tx_total = 0;
1053         uint64_t tx_err_total = 0;
1054         uint64_t rx_total_bytes = 0;
1055         uint64_t tx_total_bytes = 0;
1056         const struct pmd_internals *internals = dev->data->dev_private;
1057
1058         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_rx; i++) {
1059                 stats->q_ipackets[i] = internals->rx_queue[i].rx_pkts;
1060                 stats->q_ibytes[i] = internals->rx_queue[i].rx_bytes;
1061                 rx_total += stats->q_ipackets[i];
1062                 rx_total_bytes += stats->q_ibytes[i];
1063         }
1064
1065         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_tx; i++) {
1066                 stats->q_opackets[i] = internals->tx_queue[i].tx_pkts;
1067                 stats->q_obytes[i] = internals->tx_queue[i].tx_bytes;
1068                 stats->q_errors[i] = internals->tx_queue[i].err_pkts;
1069                 tx_total += stats->q_opackets[i];
1070                 tx_total_bytes += stats->q_obytes[i];
1071                 tx_err_total += stats->q_errors[i];
1072         }
1073
1074         stats->ipackets = rx_total;
1075         stats->opackets = tx_total;
1076         stats->ibytes = rx_total_bytes;
1077         stats->obytes = tx_total_bytes;
1078         stats->oerrors = tx_err_total;
1079
1080         return 0;
1081 }
1082
1083 static void
1084 eth_stats_reset(struct rte_eth_dev *dev)
1085 {
1086         uint16_t i;
1087         uint16_t nb_rx = dev->data->nb_rx_queues;
1088         uint16_t nb_tx = dev->data->nb_tx_queues;
1089         struct pmd_internals *internals = dev->data->dev_private;
1090
1091         for (i = 0; i < nb_rx; i++) {
1092                 internals->rx_queue[i].rx_pkts = 0;
1093                 internals->rx_queue[i].rx_bytes = 0;
1094                 internals->rx_queue[i].err_pkts = 0;
1095         }
1096         for (i = 0; i < nb_tx; i++) {
1097                 internals->tx_queue[i].tx_pkts = 0;
1098                 internals->tx_queue[i].tx_bytes = 0;
1099                 internals->tx_queue[i].err_pkts = 0;
1100         }
1101 }
1102
1103 static void
1104 eth_rx_queue_release(void *q)
1105 {
1106         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1107         if (rxq->sze != NULL) {
1108                 szedata_close(rxq->sze);
1109                 rxq->sze = NULL;
1110         }
1111 }
1112
1113 static void
1114 eth_tx_queue_release(void *q)
1115 {
1116         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1117         if (txq->sze != NULL) {
1118                 szedata_close(txq->sze);
1119                 txq->sze = NULL;
1120         }
1121 }
1122
1123 static void
1124 eth_dev_close(struct rte_eth_dev *dev)
1125 {
1126         uint16_t i;
1127         uint16_t nb_rx = dev->data->nb_rx_queues;
1128         uint16_t nb_tx = dev->data->nb_tx_queues;
1129
1130         eth_dev_stop(dev);
1131
1132         for (i = 0; i < nb_rx; i++) {
1133                 eth_rx_queue_release(dev->data->rx_queues[i]);
1134                 dev->data->rx_queues[i] = NULL;
1135         }
1136         dev->data->nb_rx_queues = 0;
1137         for (i = 0; i < nb_tx; i++) {
1138                 eth_tx_queue_release(dev->data->tx_queues[i]);
1139                 dev->data->tx_queues[i] = NULL;
1140         }
1141         dev->data->nb_tx_queues = 0;
1142 }
1143
1144 /**
1145  * Function takes value from first IBUF status register.
1146  * Values in IBUF and OBUF should be same.
1147  *
1148  * @param internals
1149  *     Pointer to device private structure.
1150  * @return
1151  *     Link speed constant.
1152  */
1153 static inline enum szedata2_link_speed
1154 get_link_speed(const struct pmd_internals *internals)
1155 {
1156         const volatile struct szedata2_ibuf *ibuf =
1157                 ibuf_ptr_by_index(internals->pci_rsc, 0);
1158         uint32_t speed = (szedata2_read32(&ibuf->ibuf_st) & 0x70) >> 4;
1159         switch (speed) {
1160         case 0x03:
1161                 return SZEDATA2_LINK_SPEED_10G;
1162         case 0x04:
1163                 return SZEDATA2_LINK_SPEED_40G;
1164         case 0x05:
1165                 return SZEDATA2_LINK_SPEED_100G;
1166         default:
1167                 return SZEDATA2_LINK_SPEED_DEFAULT;
1168         }
1169 }
1170
1171 static int
1172 eth_link_update(struct rte_eth_dev *dev,
1173                 int wait_to_complete __rte_unused)
1174 {
1175         struct rte_eth_link link;
1176         struct pmd_internals *internals = (struct pmd_internals *)
1177                 dev->data->dev_private;
1178         const volatile struct szedata2_ibuf *ibuf;
1179         uint32_t i;
1180         bool link_is_up = false;
1181
1182         memset(&link, 0, sizeof(link));
1183
1184         switch (get_link_speed(internals)) {
1185         case SZEDATA2_LINK_SPEED_10G:
1186                 link.link_speed = ETH_SPEED_NUM_10G;
1187                 break;
1188         case SZEDATA2_LINK_SPEED_40G:
1189                 link.link_speed = ETH_SPEED_NUM_40G;
1190                 break;
1191         case SZEDATA2_LINK_SPEED_100G:
1192                 link.link_speed = ETH_SPEED_NUM_100G;
1193                 break;
1194         default:
1195                 link.link_speed = ETH_SPEED_NUM_10G;
1196                 break;
1197         }
1198
1199         /* szedata2 uses only full duplex */
1200         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1201
1202         for (i = 0; i < szedata2_ibuf_count; i++) {
1203                 ibuf = ibuf_ptr_by_index(internals->pci_rsc, i);
1204                 /*
1205                  * Link is considered up if at least one ibuf is enabled
1206                  * and up.
1207                  */
1208                 if (ibuf_is_enabled(ibuf) && ibuf_is_link_up(ibuf)) {
1209                         link_is_up = true;
1210                         break;
1211                 }
1212         }
1213
1214         link.link_status = link_is_up ? ETH_LINK_UP : ETH_LINK_DOWN;
1215
1216         link.link_autoneg = ETH_LINK_FIXED;
1217
1218         rte_eth_linkstatus_set(dev, &link);
1219         return 0;
1220 }
1221
1222 static int
1223 eth_dev_set_link_up(struct rte_eth_dev *dev)
1224 {
1225         struct pmd_internals *internals = (struct pmd_internals *)
1226                 dev->data->dev_private;
1227         uint32_t i;
1228
1229         for (i = 0; i < szedata2_ibuf_count; i++)
1230                 ibuf_enable(ibuf_ptr_by_index(internals->pci_rsc, i));
1231         for (i = 0; i < szedata2_obuf_count; i++)
1232                 obuf_enable(obuf_ptr_by_index(internals->pci_rsc, i));
1233         return 0;
1234 }
1235
1236 static int
1237 eth_dev_set_link_down(struct rte_eth_dev *dev)
1238 {
1239         struct pmd_internals *internals = (struct pmd_internals *)
1240                 dev->data->dev_private;
1241         uint32_t i;
1242
1243         for (i = 0; i < szedata2_ibuf_count; i++)
1244                 ibuf_disable(ibuf_ptr_by_index(internals->pci_rsc, i));
1245         for (i = 0; i < szedata2_obuf_count; i++)
1246                 obuf_disable(obuf_ptr_by_index(internals->pci_rsc, i));
1247         return 0;
1248 }
1249
1250 static int
1251 eth_rx_queue_setup(struct rte_eth_dev *dev,
1252                 uint16_t rx_queue_id,
1253                 uint16_t nb_rx_desc __rte_unused,
1254                 unsigned int socket_id __rte_unused,
1255                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1256                 struct rte_mempool *mb_pool)
1257 {
1258         struct pmd_internals *internals = dev->data->dev_private;
1259         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1260         int ret;
1261         uint32_t rx = 1 << rx_queue_id;
1262         uint32_t tx = 0;
1263
1264         rxq->sze = szedata_open(internals->sze_dev);
1265         if (rxq->sze == NULL)
1266                 return -EINVAL;
1267         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1268         if (ret != 0 || rx == 0) {
1269                 szedata_close(rxq->sze);
1270                 rxq->sze = NULL;
1271                 return -EINVAL;
1272         }
1273         rxq->rx_channel = rx_queue_id;
1274         rxq->in_port = dev->data->port_id;
1275         rxq->mb_pool = mb_pool;
1276         rxq->rx_pkts = 0;
1277         rxq->rx_bytes = 0;
1278         rxq->err_pkts = 0;
1279
1280         dev->data->rx_queues[rx_queue_id] = rxq;
1281         return 0;
1282 }
1283
1284 static int
1285 eth_tx_queue_setup(struct rte_eth_dev *dev,
1286                 uint16_t tx_queue_id,
1287                 uint16_t nb_tx_desc __rte_unused,
1288                 unsigned int socket_id __rte_unused,
1289                 const struct rte_eth_txconf *tx_conf __rte_unused)
1290 {
1291         struct pmd_internals *internals = dev->data->dev_private;
1292         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1293         int ret;
1294         uint32_t rx = 0;
1295         uint32_t tx = 1 << tx_queue_id;
1296
1297         txq->sze = szedata_open(internals->sze_dev);
1298         if (txq->sze == NULL)
1299                 return -EINVAL;
1300         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1301         if (ret != 0 || tx == 0) {
1302                 szedata_close(txq->sze);
1303                 txq->sze = NULL;
1304                 return -EINVAL;
1305         }
1306         txq->tx_channel = tx_queue_id;
1307         txq->tx_pkts = 0;
1308         txq->tx_bytes = 0;
1309         txq->err_pkts = 0;
1310
1311         dev->data->tx_queues[tx_queue_id] = txq;
1312         return 0;
1313 }
1314
1315 static void
1316 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1317                 struct ether_addr *mac_addr __rte_unused)
1318 {
1319 }
1320
1321 static void
1322 eth_promiscuous_enable(struct rte_eth_dev *dev)
1323 {
1324         struct pmd_internals *internals = (struct pmd_internals *)
1325                 dev->data->dev_private;
1326         uint32_t i;
1327
1328         for (i = 0; i < szedata2_ibuf_count; i++) {
1329                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1330                                 SZEDATA2_MAC_CHMODE_PROMISC);
1331         }
1332 }
1333
1334 static void
1335 eth_promiscuous_disable(struct rte_eth_dev *dev)
1336 {
1337         struct pmd_internals *internals = (struct pmd_internals *)
1338                 dev->data->dev_private;
1339         uint32_t i;
1340
1341         for (i = 0; i < szedata2_ibuf_count; i++) {
1342                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1343                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1344         }
1345 }
1346
1347 static void
1348 eth_allmulticast_enable(struct rte_eth_dev *dev)
1349 {
1350         struct pmd_internals *internals = (struct pmd_internals *)
1351                 dev->data->dev_private;
1352         uint32_t i;
1353
1354         for (i = 0; i < szedata2_ibuf_count; i++) {
1355                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1356                                 SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1357         }
1358 }
1359
1360 static void
1361 eth_allmulticast_disable(struct rte_eth_dev *dev)
1362 {
1363         struct pmd_internals *internals = (struct pmd_internals *)
1364                 dev->data->dev_private;
1365         uint32_t i;
1366
1367         for (i = 0; i < szedata2_ibuf_count; i++) {
1368                 ibuf_mac_mode_write(ibuf_ptr_by_index(internals->pci_rsc, i),
1369                                 SZEDATA2_MAC_CHMODE_ONLY_VALID);
1370         }
1371 }
1372
1373 static const struct eth_dev_ops ops = {
1374         .dev_start          = eth_dev_start,
1375         .dev_stop           = eth_dev_stop,
1376         .dev_set_link_up    = eth_dev_set_link_up,
1377         .dev_set_link_down  = eth_dev_set_link_down,
1378         .dev_close          = eth_dev_close,
1379         .dev_configure      = eth_dev_configure,
1380         .dev_infos_get      = eth_dev_info,
1381         .promiscuous_enable   = eth_promiscuous_enable,
1382         .promiscuous_disable  = eth_promiscuous_disable,
1383         .allmulticast_enable  = eth_allmulticast_enable,
1384         .allmulticast_disable = eth_allmulticast_disable,
1385         .rx_queue_start     = eth_rx_queue_start,
1386         .rx_queue_stop      = eth_rx_queue_stop,
1387         .tx_queue_start     = eth_tx_queue_start,
1388         .tx_queue_stop      = eth_tx_queue_stop,
1389         .rx_queue_setup     = eth_rx_queue_setup,
1390         .tx_queue_setup     = eth_tx_queue_setup,
1391         .rx_queue_release   = eth_rx_queue_release,
1392         .tx_queue_release   = eth_tx_queue_release,
1393         .link_update        = eth_link_update,
1394         .stats_get          = eth_stats_get,
1395         .stats_reset        = eth_stats_reset,
1396         .mac_addr_set       = eth_mac_addr_set,
1397 };
1398
1399 /*
1400  * This function goes through sysfs and looks for an index of szedata2
1401  * device file (/dev/szedataIIX, where X is the index).
1402  *
1403  * @return
1404  *           0 on success
1405  *          -1 on error
1406  */
1407 static int
1408 get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index)
1409 {
1410         DIR *dir;
1411         struct dirent *entry;
1412         int ret;
1413         uint32_t tmp_index;
1414         FILE *fd;
1415         char pcislot_path[PATH_MAX];
1416         uint32_t domain;
1417         uint32_t bus;
1418         uint32_t devid;
1419         uint32_t function;
1420
1421         dir = opendir("/sys/class/combo");
1422         if (dir == NULL)
1423                 return -1;
1424
1425         /*
1426          * Iterate through all combosixX directories.
1427          * When the value in /sys/class/combo/combosixX/device/pcislot
1428          * file is the location of the ethernet device dev, "X" is the
1429          * index of the device.
1430          */
1431         while ((entry = readdir(dir)) != NULL) {
1432                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1433                 if (ret != 1)
1434                         continue;
1435
1436                 snprintf(pcislot_path, PATH_MAX,
1437                         "/sys/class/combo/combosix%u/device/pcislot",
1438                         tmp_index);
1439
1440                 fd = fopen(pcislot_path, "r");
1441                 if (fd == NULL)
1442                         continue;
1443
1444                 ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8,
1445                                 &domain, &bus, &devid, &function);
1446                 fclose(fd);
1447                 if (ret != 4)
1448                         continue;
1449
1450                 if (pcislot_addr->domain == domain &&
1451                                 pcislot_addr->bus == bus &&
1452                                 pcislot_addr->devid == devid &&
1453                                 pcislot_addr->function == function) {
1454                         *index = tmp_index;
1455                         closedir(dir);
1456                         return 0;
1457                 }
1458         }
1459
1460         closedir(dir);
1461         return -1;
1462 }
1463
1464 static int
1465 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1466 {
1467         struct rte_eth_dev_data *data = dev->data;
1468         struct pmd_internals *internals = (struct pmd_internals *)
1469                 data->dev_private;
1470         struct szedata *szedata_temp;
1471         int ret;
1472         uint32_t szedata2_index;
1473         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1474         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1475         struct rte_mem_resource *pci_rsc =
1476                 &pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1477         char rsc_filename[PATH_MAX];
1478         void *pci_resource_ptr = NULL;
1479         int fd;
1480
1481         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1482                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1483                         pci_addr->function);
1484
1485         /* Get index of szedata2 device file and create path to device file */
1486         ret = get_szedata2_index(pci_addr, &szedata2_index);
1487         if (ret != 0) {
1488                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1489                 return -ENODEV;
1490         }
1491         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1492                         szedata2_index);
1493
1494         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1495
1496         /*
1497          * Get number of available DMA RX and TX channels, which is maximum
1498          * number of queues that can be created and store it in private device
1499          * data structure.
1500          */
1501         szedata_temp = szedata_open(internals->sze_dev);
1502         if (szedata_temp == NULL) {
1503                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1504                                 internals->sze_dev);
1505                 return -EINVAL;
1506         }
1507         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1508                         SZE2_DIR_RX);
1509         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1510                         SZE2_DIR_TX);
1511         szedata_close(szedata_temp);
1512
1513         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1514                         internals->max_rx_queues, internals->max_tx_queues);
1515
1516         /* Set rx, tx burst functions */
1517         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1518                 data->scattered_rx == 1) {
1519                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1520                 data->scattered_rx = 1;
1521         } else {
1522                 dev->rx_pkt_burst = eth_szedata2_rx;
1523                 data->scattered_rx = 0;
1524         }
1525         dev->tx_pkt_burst = eth_szedata2_tx;
1526
1527         /* Set function callbacks for Ethernet API */
1528         dev->dev_ops = &ops;
1529
1530         rte_eth_copy_pci_info(dev, pci_dev);
1531
1532         /* mmap pci resource0 file to rte_mem_resource structure */
1533         if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1534                         0) {
1535                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1536                                 PCI_RESOURCE_NUMBER);
1537                 return -EINVAL;
1538         }
1539         snprintf(rsc_filename, PATH_MAX,
1540                 "%s/" PCI_PRI_FMT "/resource%u", rte_pci_get_sysfs_path(),
1541                 pci_addr->domain, pci_addr->bus,
1542                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1543         fd = open(rsc_filename, O_RDWR);
1544         if (fd < 0) {
1545                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1546                 return -EINVAL;
1547         }
1548
1549         pci_resource_ptr = mmap(0,
1550                         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1551                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1552         close(fd);
1553         if (pci_resource_ptr == MAP_FAILED) {
1554                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1555                                 rsc_filename, fd);
1556                 return -EINVAL;
1557         }
1558         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr;
1559         internals->pci_rsc = pci_rsc;
1560
1561         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1562                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1563                         (unsigned long long)pci_rsc->phys_addr,
1564                         (unsigned long long)pci_rsc->len,
1565                         (unsigned long long)pci_rsc->addr);
1566
1567         /* Get link state */
1568         eth_link_update(dev, 0);
1569
1570         /* Allocate space for one mac address */
1571         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1572                         RTE_CACHE_LINE_SIZE);
1573         if (data->mac_addrs == NULL) {
1574                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1575                 munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1576                        pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1577                 return -EINVAL;
1578         }
1579
1580         ether_addr_copy(&eth_addr, data->mac_addrs);
1581
1582         /* At initial state COMBO card is in promiscuous mode so disable it */
1583         eth_promiscuous_disable(dev);
1584
1585         RTE_LOG(INFO, PMD, "szedata2 device ("
1586                         PCI_PRI_FMT ") successfully initialized\n",
1587                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1588                         pci_addr->function);
1589
1590         return 0;
1591 }
1592
1593 static int
1594 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1595 {
1596         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1597         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1598
1599         rte_free(dev->data->mac_addrs);
1600         dev->data->mac_addrs = NULL;
1601         munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1602                pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1603
1604         RTE_LOG(INFO, PMD, "szedata2 device ("
1605                         PCI_PRI_FMT ") successfully uninitialized\n",
1606                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1607                         pci_addr->function);
1608
1609         return 0;
1610 }
1611
1612 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1613         {
1614                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1615                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1616         },
1617         {
1618                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1619                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1620         },
1621         {
1622                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1623                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1624         },
1625         {
1626                 .vendor_id = 0,
1627         }
1628 };
1629
1630 static int szedata2_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1631         struct rte_pci_device *pci_dev)
1632 {
1633         return rte_eth_dev_pci_generic_probe(pci_dev,
1634                 sizeof(struct pmd_internals), rte_szedata2_eth_dev_init);
1635 }
1636
1637 static int szedata2_eth_pci_remove(struct rte_pci_device *pci_dev)
1638 {
1639         return rte_eth_dev_pci_generic_remove(pci_dev,
1640                 rte_szedata2_eth_dev_uninit);
1641 }
1642
1643 static struct rte_pci_driver szedata2_eth_driver = {
1644         .id_table = rte_szedata2_pci_id_table,
1645         .probe = szedata2_eth_pci_probe,
1646         .remove = szedata2_eth_pci_remove,
1647 };
1648
1649 RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver);
1650 RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1651 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
1652         "* combo6core & combov3 & szedata2 & szedata2_cv3");