drivers/net/szedata2/rte_eth_szedata2.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) 2015 - 2016 CESNET
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of CESNET nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdint.h>
  35 #include <unistd.h>
  36 #include <stdbool.h>
  37 #include <err.h>
  38 #include <sys/types.h>
  39 #include <dirent.h>
  40 #include <sys/stat.h>
  41 #include <fcntl.h>
  42 #include <sys/mman.h>
  43
  44 #include <libsze2.h>
  45
  46 #include <rte_mbuf.h>
  47 #include <rte_ethdev.h>
  48 #include <rte_malloc.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_kvargs.h>
  51 #include <rte_dev.h>
  52 #include <rte_atomic.h>
  53
  54 #include "rte_eth_szedata2.h"
  55
  56 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
  57 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
  58 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
  59
  60 /**
  61  * size of szedata2_packet header with alignment
  62  */
  63 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
  64
  65 #define RTE_SZEDATA2_DRIVER_NAME net_szedata2
  66 #define RTE_SZEDATA2_PCI_DRIVER_NAME "rte_szedata2_pmd"
  67
  68 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
  69
  70 struct szedata2_rx_queue {
  71         struct szedata *sze;
  72         uint8_t rx_channel;
  73         uint8_t in_port;
  74         struct rte_mempool *mb_pool;
  75         volatile uint64_t rx_pkts;
  76         volatile uint64_t rx_bytes;
  77         volatile uint64_t err_pkts;
  78 };
  79
  80 struct szedata2_tx_queue {
  81         struct szedata *sze;
  82         uint8_t tx_channel;
  83         volatile uint64_t tx_pkts;
  84         volatile uint64_t tx_bytes;
  85         volatile uint64_t err_pkts;
  86 };
  87
  88 struct pmd_internals {
  89         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
  90         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
  91         uint16_t max_rx_queues;
  92         uint16_t max_tx_queues;
  93         char sze_dev[PATH_MAX];
  94         struct rte_mem_resource *pci_rsc;
  95 };
  96
  97 static struct ether_addr eth_addr = {
  98         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
  99 };
 100
 101 static uint16_t
 102 eth_szedata2_rx(void *queue,
 103                 struct rte_mbuf **bufs,
 104                 uint16_t nb_pkts)
 105 {
 106         unsigned int i;
 107         struct rte_mbuf *mbuf;
 108         struct szedata2_rx_queue *sze_q = queue;
 109         struct rte_pktmbuf_pool_private *mbp_priv;
 110         uint16_t num_rx = 0;
 111         uint16_t buf_size;
 112         uint16_t sg_size;
 113         uint16_t hw_size;
 114         uint16_t packet_size;
 115         uint64_t num_bytes = 0;
 116         struct szedata *sze = sze_q->sze;
 117         uint8_t *header_ptr = NULL; /* header of packet */
 118         uint8_t *packet_ptr1 = NULL;
 119         uint8_t *packet_ptr2 = NULL;
 120         uint16_t packet_len1 = 0;
 121         uint16_t packet_len2 = 0;
 122         uint16_t hw_data_align;
 123
 124         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 125                 return 0;
 126
 127         /*
 128          * Reads the given number of packets from szedata2 channel given
 129          * by queue and copies the packet data into a newly allocated mbuf
 130          * to return.
 131          */
 132         for (i = 0; i < nb_pkts; i++) {
 133                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 134
 135                 if (unlikely(mbuf == NULL))
 136                         break;
 137
 138                 /* get the next sze packet */
 139                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 140                                 sze->ct_rx_lck->next == NULL) {
 141                         /* unlock old data */
 142                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 143                         sze->ct_rx_lck_orig = NULL;
 144                         sze->ct_rx_lck = NULL;
 145                 }
 146
 147                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 148                         /* nothing to read, lock new data */
 149                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 150                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 151
 152                         if (sze->ct_rx_lck == NULL) {
 153                                 /* nothing to lock */
 154                                 rte_pktmbuf_free(mbuf);
 155                                 break;
 156                         }
 157
 158                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 159                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 160
 161                         if (!sze->ct_rx_rem_bytes) {
 162                                 rte_pktmbuf_free(mbuf);
 163                                 break;
 164                         }
 165                 }
 166
 167                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 168                         /*
 169                          * cut in header
 170                          * copy parts of header to merge buffer
 171                          */
 172                         if (sze->ct_rx_lck->next == NULL) {
 173                                 rte_pktmbuf_free(mbuf);
 174                                 break;
 175                         }
 176
 177                         /* copy first part of header */
 178                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 179                                         sze->ct_rx_rem_bytes);
 180
 181                         /* copy second part of header */
 182                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 183                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 184                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 185                                 sze->ct_rx_cur_ptr,
 186                                 RTE_SZE2_PACKET_HEADER_SIZE -
 187                                 sze->ct_rx_rem_bytes);
 188
 189                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 190                                 sze->ct_rx_rem_bytes;
 191                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 192                                 RTE_SZE2_PACKET_HEADER_SIZE +
 193                                 sze->ct_rx_rem_bytes;
 194
 195                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 196                 } else {
 197                         /* not cut */
 198                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 199                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 200                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 201                 }
 202
 203                 sg_size = le16toh(*((uint16_t *)header_ptr));
 204                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 205                 packet_size = sg_size -
 206                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 207
 208
 209                 /* checks if packet all right */
 210                 if (!sg_size)
 211                         errx(5, "Zero segsize");
 212
 213                 /* check sg_size and hwsize */
 214                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 215                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 216                                 "hwsize: %d", sg_size, hw_size);
 217                 }
 218
 219                 hw_data_align =
 220                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
 221                         RTE_SZE2_PACKET_HEADER_SIZE;
 222
 223                 if (sze->ct_rx_rem_bytes >=
 224                                 (uint16_t)(sg_size -
 225                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 226                         /* no cut */
 227                         /* one packet ready - go to another */
 228                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 229                         packet_len1 = packet_size;
 230                         packet_ptr2 = NULL;
 231                         packet_len2 = 0;
 232
 233                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 234                                 RTE_SZE2_PACKET_HEADER_SIZE;
 235                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 236                                 RTE_SZE2_PACKET_HEADER_SIZE;
 237                 } else {
 238                         /* cut in data */
 239                         if (sze->ct_rx_lck->next == NULL) {
 240                                 errx(6, "Need \"next\" lock, "
 241                                         "but it is missing: %u",
 242                                         sze->ct_rx_rem_bytes);
 243                         }
 244
 245                         /* skip hw data */
 246                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 247                                 uint16_t rem_size = hw_data_align -
 248                                         sze->ct_rx_rem_bytes;
 249
 250                                 /* MOVE to next lock */
 251                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 252                                 sze->ct_rx_cur_ptr =
 253                                         (void *)(((uint8_t *)
 254                                         (sze->ct_rx_lck->start)) + rem_size);
 255
 256                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 257                                 packet_len1 = packet_size;
 258                                 packet_ptr2 = NULL;
 259                                 packet_len2 = 0;
 260
 261                                 sze->ct_rx_cur_ptr +=
 262                                         RTE_SZE2_ALIGN8(packet_size);
 263                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 264                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 265                         } else {
 266                                 /* get pointer and length from first part */
 267                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 268                                         hw_data_align;
 269                                 packet_len1 = sze->ct_rx_rem_bytes -
 270                                         hw_data_align;
 271
 272                                 /* MOVE to next lock */
 273                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 274                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 275
 276                                 /* get pointer and length from second part */
 277                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 278                                 packet_len2 = packet_size - packet_len1;
 279
 280                                 sze->ct_rx_cur_ptr +=
 281                                         RTE_SZE2_ALIGN8(packet_size) -
 282                                         packet_len1;
 283                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 284                                         (RTE_SZE2_ALIGN8(packet_size) -
 285                                          packet_len1);
 286                         }
 287                 }
 288
 289                 if (unlikely(packet_ptr1 == NULL)) {
 290                         rte_pktmbuf_free(mbuf);
 291                         break;
 292                 }
 293
 294                 /* get the space available for data in the mbuf */
 295                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 296                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 297                                 RTE_PKTMBUF_HEADROOM);
 298
 299                 if (packet_size <= buf_size) {
 300                         /* sze packet will fit in one mbuf, go ahead and copy */
 301                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 302                                         packet_ptr1, packet_len1);
 303                         if (packet_ptr2 != NULL) {
 304                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
 305                                         uint8_t *) + packet_len1),
 306                                         packet_ptr2, packet_len2);
 307                         }
 308                         mbuf->data_len = (uint16_t)packet_size;
 309
 310                         mbuf->pkt_len = packet_size;
 311                         mbuf->port = sze_q->in_port;
 312                         bufs[num_rx] = mbuf;
 313                         num_rx++;
 314                         num_bytes += packet_size;
 315                 } else {
 316                         /*
 317                          * sze packet will not fit in one mbuf,
 318                          * scattered mode is not enabled, drop packet
 319                          */
 320                         RTE_LOG(ERR, PMD,
 321                                 "SZE segment %d bytes will not fit in one mbuf "
 322                                 "(%d bytes), scattered mode is not enabled, "
 323                                 "drop packet!!\n",
 324                                 packet_size, buf_size);
 325                         rte_pktmbuf_free(mbuf);
 326                 }
 327         }
 328
 329         sze_q->rx_pkts += num_rx;
 330         sze_q->rx_bytes += num_bytes;
 331         return num_rx;
 332 }
 333
 334 static uint16_t
 335 eth_szedata2_rx_scattered(void *queue,
 336                 struct rte_mbuf **bufs,
 337                 uint16_t nb_pkts)
 338 {
 339         unsigned int i;
 340         struct rte_mbuf *mbuf;
 341         struct szedata2_rx_queue *sze_q = queue;
 342         struct rte_pktmbuf_pool_private *mbp_priv;
 343         uint16_t num_rx = 0;
 344         uint16_t buf_size;
 345         uint16_t sg_size;
 346         uint16_t hw_size;
 347         uint16_t packet_size;
 348         uint64_t num_bytes = 0;
 349         struct szedata *sze = sze_q->sze;
 350         uint8_t *header_ptr = NULL; /* header of packet */
 351         uint8_t *packet_ptr1 = NULL;
 352         uint8_t *packet_ptr2 = NULL;
 353         uint16_t packet_len1 = 0;
 354         uint16_t packet_len2 = 0;
 355         uint16_t hw_data_align;
 356
 357         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
 358                 return 0;
 359
 360         /*
 361          * Reads the given number of packets from szedata2 channel given
 362          * by queue and copies the packet data into a newly allocated mbuf
 363          * to return.
 364          */
 365         for (i = 0; i < nb_pkts; i++) {
 366                 const struct szedata_lock *ct_rx_lck_backup;
 367                 unsigned int ct_rx_rem_bytes_backup;
 368                 unsigned char *ct_rx_cur_ptr_backup;
 369
 370                 /* get the next sze packet */
 371                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
 372                                 sze->ct_rx_lck->next == NULL) {
 373                         /* unlock old data */
 374                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
 375                         sze->ct_rx_lck_orig = NULL;
 376                         sze->ct_rx_lck = NULL;
 377                 }
 378
 379                 /*
 380                  * Store items from sze structure which can be changed
 381                  * before mbuf allocating. Use these items in case of mbuf
 382                  * allocating failure.
 383                  */
 384                 ct_rx_lck_backup = sze->ct_rx_lck;
 385                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 386                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 387
 388                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
 389                         /* nothing to read, lock new data */
 390                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
 391                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
 392
 393                         /*
 394                          * Backup items from sze structure must be updated
 395                          * after locking to contain pointers to new locks.
 396                          */
 397                         ct_rx_lck_backup = sze->ct_rx_lck;
 398                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
 399                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
 400
 401                         if (sze->ct_rx_lck == NULL)
 402                                 /* nothing to lock */
 403                                 break;
 404
 405                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 406                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
 407
 408                         if (!sze->ct_rx_rem_bytes)
 409                                 break;
 410                 }
 411
 412                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
 413                         /*
 414                          * cut in header - copy parts of header to merge buffer
 415                          */
 416                         if (sze->ct_rx_lck->next == NULL)
 417                                 break;
 418
 419                         /* copy first part of header */
 420                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
 421                                         sze->ct_rx_rem_bytes);
 422
 423                         /* copy second part of header */
 424                         sze->ct_rx_lck = sze->ct_rx_lck->next;
 425                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 426                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
 427                                 sze->ct_rx_cur_ptr,
 428                                 RTE_SZE2_PACKET_HEADER_SIZE -
 429                                 sze->ct_rx_rem_bytes);
 430
 431                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
 432                                 sze->ct_rx_rem_bytes;
 433                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 434                                 RTE_SZE2_PACKET_HEADER_SIZE +
 435                                 sze->ct_rx_rem_bytes;
 436
 437                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
 438                 } else {
 439                         /* not cut */
 440                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
 441                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
 442                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
 443                 }
 444
 445                 sg_size = le16toh(*((uint16_t *)header_ptr));
 446                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
 447                 packet_size = sg_size -
 448                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
 449
 450
 451                 /* checks if packet all right */
 452                 if (!sg_size)
 453                         errx(5, "Zero segsize");
 454
 455                 /* check sg_size and hwsize */
 456                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
 457                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
 458                                         "hwsize: %d", sg_size, hw_size);
 459                 }
 460
 461                 hw_data_align =
 462                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
 463                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
 464
 465                 if (sze->ct_rx_rem_bytes >=
 466                                 (uint16_t)(sg_size -
 467                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
 468                         /* no cut */
 469                         /* one packet ready - go to another */
 470                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
 471                         packet_len1 = packet_size;
 472                         packet_ptr2 = NULL;
 473                         packet_len2 = 0;
 474
 475                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
 476                                 RTE_SZE2_PACKET_HEADER_SIZE;
 477                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
 478                                 RTE_SZE2_PACKET_HEADER_SIZE;
 479                 } else {
 480                         /* cut in data */
 481                         if (sze->ct_rx_lck->next == NULL) {
 482                                 errx(6, "Need \"next\" lock, but it is "
 483                                         "missing: %u", sze->ct_rx_rem_bytes);
 484                         }
 485
 486                         /* skip hw data */
 487                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
 488                                 uint16_t rem_size = hw_data_align -
 489                                         sze->ct_rx_rem_bytes;
 490
 491                                 /* MOVE to next lock */
 492                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 493                                 sze->ct_rx_cur_ptr =
 494                                         (void *)(((uint8_t *)
 495                                         (sze->ct_rx_lck->start)) + rem_size);
 496
 497                                 packet_ptr1 = sze->ct_rx_cur_ptr;
 498                                 packet_len1 = packet_size;
 499                                 packet_ptr2 = NULL;
 500                                 packet_len2 = 0;
 501
 502                                 sze->ct_rx_cur_ptr +=
 503                                         RTE_SZE2_ALIGN8(packet_size);
 504                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 505                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
 506                         } else {
 507                                 /* get pointer and length from first part */
 508                                 packet_ptr1 = sze->ct_rx_cur_ptr +
 509                                         hw_data_align;
 510                                 packet_len1 = sze->ct_rx_rem_bytes -
 511                                         hw_data_align;
 512
 513                                 /* MOVE to next lock */
 514                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
 515                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
 516
 517                                 /* get pointer and length from second part */
 518                                 packet_ptr2 = sze->ct_rx_cur_ptr;
 519                                 packet_len2 = packet_size - packet_len1;
 520
 521                                 sze->ct_rx_cur_ptr +=
 522                                         RTE_SZE2_ALIGN8(packet_size) -
 523                                         packet_len1;
 524                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
 525                                         (RTE_SZE2_ALIGN8(packet_size) -
 526                                          packet_len1);
 527                         }
 528                 }
 529
 530                 if (unlikely(packet_ptr1 == NULL))
 531                         break;
 532
 533                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
 534
 535                 if (unlikely(mbuf == NULL)) {
 536                         /*
 537                          * Restore items from sze structure to state after
 538                          * unlocking (eventually locking).
 539                          */
 540                         sze->ct_rx_lck = ct_rx_lck_backup;
 541                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
 542                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
 543                         break;
 544                 }
 545
 546                 /* get the space available for data in the mbuf */
 547                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
 548                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
 549                                 RTE_PKTMBUF_HEADROOM);
 550
 551                 if (packet_size <= buf_size) {
 552                         /* sze packet will fit in one mbuf, go ahead and copy */
 553                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
 554                                         packet_ptr1, packet_len1);
 555                         if (packet_ptr2 != NULL) {
 556                                 rte_memcpy((void *)
 557                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
 558                                         packet_len1), packet_ptr2, packet_len2);
 559                         }
 560                         mbuf->data_len = (uint16_t)packet_size;
 561                 } else {
 562                         /*
 563                          * sze packet will not fit in one mbuf,
 564                          * scatter packet into more mbufs
 565                          */
 566                         struct rte_mbuf *m = mbuf;
 567                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
 568
 569                         /* copy first part of packet */
 570                         /* fill first mbuf */
 571                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
 572                                 len);
 573                         packet_len1 -= len;
 574                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 575
 576                         while (packet_len1 > 0) {
 577                                 /* fill new mbufs */
 578                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
 579
 580                                 if (unlikely(m->next == NULL)) {
 581                                         rte_pktmbuf_free(mbuf);
 582                                         /*
 583                                          * Restore items from sze structure
 584                                          * to state after unlocking (eventually
 585                                          * locking).
 586                                          */
 587                                         sze->ct_rx_lck = ct_rx_lck_backup;
 588                                         sze->ct_rx_rem_bytes =
 589                                                 ct_rx_rem_bytes_backup;
 590                                         sze->ct_rx_cur_ptr =
 591                                                 ct_rx_cur_ptr_backup;
 592                                         goto finish;
 593                                 }
 594
 595                                 m = m->next;
 596
 597                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
 598                                         packet_len1);
 599                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 600                                         packet_ptr1, len);
 601
 602                                 (mbuf->nb_segs)++;
 603                                 packet_len1 -= len;
 604                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
 605                         }
 606
 607                         if (packet_ptr2 != NULL) {
 608                                 /* copy second part of packet, if exists */
 609                                 /* fill the rest of currently last mbuf */
 610                                 len = rte_pktmbuf_tailroom(m);
 611                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
 612                                         packet_ptr2, len);
 613                                 packet_len2 -= len;
 614                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
 615
 616                                 while (packet_len2 > 0) {
 617                                         /* fill new mbufs */
 618                                         m->next = rte_pktmbuf_alloc(
 619                                                         sze_q->mb_pool);
 620
 621                                         if (unlikely(m->next == NULL)) {
 622                                                 rte_pktmbuf_free(mbuf);
 623                                                 /*
 624                                                  * Restore items from sze
 625                                                  * structure to state after
 626                                                  * unlocking (eventually
 627                                                  * locking).
 628                                                  */
 629                                                 sze->ct_rx_lck =
 630                                                         ct_rx_lck_backup;
 631                                                 sze->ct_rx_rem_bytes =
 632                                                         ct_rx_rem_bytes_backup;
 633                                                 sze->ct_rx_cur_ptr =
 634                                                         ct_rx_cur_ptr_backup;
 635                                                 goto finish;
 636                                         }
 637
 638                                         m = m->next;
 639
 640                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
 641                                                 packet_len2);
 642                                         rte_memcpy(
 643                                                 rte_pktmbuf_append(mbuf, len),
 644                                                 packet_ptr2, len);
 645
 646                                         (mbuf->nb_segs)++;
 647                                         packet_len2 -= len;
 648                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
 649                                                 len;
 650                                 }
 651                         }
 652                 }
 653                 mbuf->pkt_len = packet_size;
 654                 mbuf->port = sze_q->in_port;
 655                 bufs[num_rx] = mbuf;
 656                 num_rx++;
 657                 num_bytes += packet_size;
 658         }
 659
 660 finish:
 661         sze_q->rx_pkts += num_rx;
 662         sze_q->rx_bytes += num_bytes;
 663         return num_rx;
 664 }
 665
 666 static uint16_t
 667 eth_szedata2_tx(void *queue,
 668                 struct rte_mbuf **bufs,
 669                 uint16_t nb_pkts)
 670 {
 671         struct rte_mbuf *mbuf;
 672         struct szedata2_tx_queue *sze_q = queue;
 673         uint16_t num_tx = 0;
 674         uint64_t num_bytes = 0;
 675
 676         const struct szedata_lock *lck;
 677         uint32_t lock_size;
 678         uint32_t lock_size2;
 679         void *dst;
 680         uint32_t pkt_len;
 681         uint32_t hwpkt_len;
 682         uint32_t unlock_size;
 683         uint32_t rem_len;
 684         uint8_t mbuf_segs;
 685         uint16_t pkt_left = nb_pkts;
 686
 687         if (sze_q->sze == NULL || nb_pkts == 0)
 688                 return 0;
 689
 690         while (pkt_left > 0) {
 691                 unlock_size = 0;
 692                 lck = szedata_tx_lock_data(sze_q->sze,
 693                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
 694                         sze_q->tx_channel);
 695                 if (lck == NULL)
 696                         continue;
 697
 698                 dst = lck->start;
 699                 lock_size = lck->len;
 700                 lock_size2 = lck->next ? lck->next->len : 0;
 701
 702 next_packet:
 703                 mbuf = bufs[nb_pkts - pkt_left];
 704
 705                 pkt_len = mbuf->pkt_len;
 706                 mbuf_segs = mbuf->nb_segs;
 707
 708                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 709                         RTE_SZE2_ALIGN8(pkt_len);
 710
 711                 if (lock_size + lock_size2 < hwpkt_len) {
 712                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 713                         continue;
 714                 }
 715
 716                 num_bytes += pkt_len;
 717
 718                 if (lock_size > hwpkt_len) {
 719                         void *tmp_dst;
 720
 721                         rem_len = 0;
 722
 723                         /* write packet length at first 2 bytes in 8B header */
 724                         *((uint16_t *)dst) = htole16(
 725                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 726                                         pkt_len);
 727                         *(((uint16_t *)dst) + 1) = htole16(0);
 728
 729                         /* copy packet from mbuf */
 730                         tmp_dst = ((uint8_t *)(dst)) +
 731                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 732                         if (mbuf_segs == 1) {
 733                                 /*
 734                                  * non-scattered packet,
 735                                  * transmit from one mbuf
 736                                  */
 737                                 rte_memcpy(tmp_dst,
 738                                         rte_pktmbuf_mtod(mbuf, const void *),
 739                                         pkt_len);
 740                         } else {
 741                                 /* scattered packet, transmit from more mbufs */
 742                                 struct rte_mbuf *m = mbuf;
 743                                 while (m) {
 744                                         rte_memcpy(tmp_dst,
 745                                                 rte_pktmbuf_mtod(m,
 746                                                 const void *),
 747                                                 m->data_len);
 748                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 749                                                 m->data_len;
 750                                         m = m->next;
 751                                 }
 752                         }
 753
 754
 755                         dst = ((uint8_t *)dst) + hwpkt_len;
 756                         unlock_size += hwpkt_len;
 757                         lock_size -= hwpkt_len;
 758
 759                         rte_pktmbuf_free(mbuf);
 760                         num_tx++;
 761                         pkt_left--;
 762                         if (pkt_left == 0) {
 763                                 szedata_tx_unlock_data(sze_q->sze, lck,
 764                                         unlock_size);
 765                                 break;
 766                         }
 767                         goto next_packet;
 768                 } else if (lock_size + lock_size2 >= hwpkt_len) {
 769                         void *tmp_dst;
 770                         uint16_t write_len;
 771
 772                         /* write packet length at first 2 bytes in 8B header */
 773                         *((uint16_t *)dst) =
 774                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
 775                                         pkt_len);
 776                         *(((uint16_t *)dst) + 1) = htole16(0);
 777
 778                         /*
 779                          * If the raw packet (pkt_len) is smaller than lock_size
 780                          * get the correct length for memcpy
 781                          */
 782                         write_len =
 783                                 pkt_len < lock_size -
 784                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
 785                                 pkt_len :
 786                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 787
 788                         rem_len = hwpkt_len - lock_size;
 789
 790                         tmp_dst = ((uint8_t *)(dst)) +
 791                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
 792                         if (mbuf_segs == 1) {
 793                                 /*
 794                                  * non-scattered packet,
 795                                  * transmit from one mbuf
 796                                  */
 797                                 /* copy part of packet to first area */
 798                                 rte_memcpy(tmp_dst,
 799                                         rte_pktmbuf_mtod(mbuf, const void *),
 800                                         write_len);
 801
 802                                 if (lck->next)
 803                                         dst = lck->next->start;
 804
 805                                 /* copy part of packet to second area */
 806                                 rte_memcpy(dst,
 807                                         (const void *)(rte_pktmbuf_mtod(mbuf,
 808                                                         const uint8_t *) +
 809                                         write_len), pkt_len - write_len);
 810                         } else {
 811                                 /* scattered packet, transmit from more mbufs */
 812                                 struct rte_mbuf *m = mbuf;
 813                                 uint16_t written = 0;
 814                                 uint16_t to_write = 0;
 815                                 bool new_mbuf = true;
 816                                 uint16_t write_off = 0;
 817
 818                                 /* copy part of packet to first area */
 819                                 while (m && written < write_len) {
 820                                         to_write = RTE_MIN(m->data_len,
 821                                                         write_len - written);
 822                                         rte_memcpy(tmp_dst,
 823                                                 rte_pktmbuf_mtod(m,
 824                                                         const void *),
 825                                                 to_write);
 826
 827                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 828                                                 to_write;
 829                                         if (m->data_len <= write_len -
 830                                                         written) {
 831                                                 m = m->next;
 832                                                 new_mbuf = true;
 833                                         } else {
 834                                                 new_mbuf = false;
 835                                         }
 836                                         written += to_write;
 837                                 }
 838
 839                                 if (lck->next)
 840                                         dst = lck->next->start;
 841
 842                                 tmp_dst = dst;
 843                                 written = 0;
 844                                 write_off = new_mbuf ? 0 : to_write;
 845
 846                                 /* copy part of packet to second area */
 847                                 while (m && written < pkt_len - write_len) {
 848                                         rte_memcpy(tmp_dst, (const void *)
 849                                                 (rte_pktmbuf_mtod(m,
 850                                                 uint8_t *) + write_off),
 851                                                 m->data_len - write_off);
 852
 853                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
 854                                                 (m->data_len - write_off);
 855                                         written += m->data_len - write_off;
 856                                         m = m->next;
 857                                         write_off = 0;
 858                                 }
 859                         }
 860
 861                         dst = ((uint8_t *)dst) + rem_len;
 862                         unlock_size += hwpkt_len;
 863                         lock_size = lock_size2 - rem_len;
 864                         lock_size2 = 0;
 865
 866                         rte_pktmbuf_free(mbuf);
 867                         num_tx++;
 868                 }
 869
 870                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
 871                 pkt_left--;
 872         }
 873
 874         sze_q->tx_pkts += num_tx;
 875         sze_q->err_pkts += nb_pkts - num_tx;
 876         sze_q->tx_bytes += num_bytes;
 877         return num_tx;
 878 }
 879
 880 static int
 881 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
 882 {
 883         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 884         int ret;
 885         struct pmd_internals *internals = (struct pmd_internals *)
 886                 dev->data->dev_private;
 887
 888         if (rxq->sze == NULL) {
 889                 uint32_t rx = 1 << rxq->rx_channel;
 890                 uint32_t tx = 0;
 891                 rxq->sze = szedata_open(internals->sze_dev);
 892                 if (rxq->sze == NULL)
 893                         return -EINVAL;
 894                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
 895                 if (ret != 0 || rx == 0)
 896                         goto err;
 897         }
 898
 899         ret = szedata_start(rxq->sze);
 900         if (ret != 0)
 901                 goto err;
 902         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 903         return 0;
 904
 905 err:
 906         szedata_close(rxq->sze);
 907         rxq->sze = NULL;
 908         return -EINVAL;
 909 }
 910
 911 static int
 912 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
 913 {
 914         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
 915
 916         if (rxq->sze != NULL) {
 917                 szedata_close(rxq->sze);
 918                 rxq->sze = NULL;
 919         }
 920
 921         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 922         return 0;
 923 }
 924
 925 static int
 926 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
 927 {
 928         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 929         int ret;
 930         struct pmd_internals *internals = (struct pmd_internals *)
 931                 dev->data->dev_private;
 932
 933         if (txq->sze == NULL) {
 934                 uint32_t rx = 0;
 935                 uint32_t tx = 1 << txq->tx_channel;
 936                 txq->sze = szedata_open(internals->sze_dev);
 937                 if (txq->sze == NULL)
 938                         return -EINVAL;
 939                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
 940                 if (ret != 0 || tx == 0)
 941                         goto err;
 942         }
 943
 944         ret = szedata_start(txq->sze);
 945         if (ret != 0)
 946                 goto err;
 947         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
 948         return 0;
 949
 950 err:
 951         szedata_close(txq->sze);
 952         txq->sze = NULL;
 953         return -EINVAL;
 954 }
 955
 956 static int
 957 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
 958 {
 959         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
 960
 961         if (txq->sze != NULL) {
 962                 szedata_close(txq->sze);
 963                 txq->sze = NULL;
 964         }
 965
 966         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 967         return 0;
 968 }
 969
 970 static int
 971 eth_dev_start(struct rte_eth_dev *dev)
 972 {
 973         int ret;
 974         uint16_t i;
 975         uint16_t nb_rx = dev->data->nb_rx_queues;
 976         uint16_t nb_tx = dev->data->nb_tx_queues;
 977
 978         for (i = 0; i < nb_rx; i++) {
 979                 ret = eth_rx_queue_start(dev, i);
 980                 if (ret != 0)
 981                         goto err_rx;
 982         }
 983
 984         for (i = 0; i < nb_tx; i++) {
 985                 ret = eth_tx_queue_start(dev, i);
 986                 if (ret != 0)
 987                         goto err_tx;
 988         }
 989
 990         return 0;
 991
 992 err_tx:
 993         for (i = 0; i < nb_tx; i++)
 994                 eth_tx_queue_stop(dev, i);
 995 err_rx:
 996         for (i = 0; i < nb_rx; i++)
 997                 eth_rx_queue_stop(dev, i);
 998         return ret;
 999 }
1000
1001 static void
1002 eth_dev_stop(struct rte_eth_dev *dev)
1003 {
1004         uint16_t i;
1005         uint16_t nb_rx = dev->data->nb_rx_queues;
1006         uint16_t nb_tx = dev->data->nb_tx_queues;
1007
1008         for (i = 0; i < nb_tx; i++)
1009                 eth_tx_queue_stop(dev, i);
1010
1011         for (i = 0; i < nb_rx; i++)
1012                 eth_rx_queue_stop(dev, i);
1013 }
1014
1015 static int
1016 eth_dev_configure(struct rte_eth_dev *dev)
1017 {
1018         struct rte_eth_dev_data *data = dev->data;
1019         if (data->dev_conf.rxmode.enable_scatter == 1) {
1020                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1021                 data->scattered_rx = 1;
1022         } else {
1023                 dev->rx_pkt_burst = eth_szedata2_rx;
1024                 data->scattered_rx = 0;
1025         }
1026         return 0;
1027 }
1028
1029 static void
1030 eth_dev_info(struct rte_eth_dev *dev,
1031                 struct rte_eth_dev_info *dev_info)
1032 {
1033         struct pmd_internals *internals = dev->data->dev_private;
1034         dev_info->pci_dev = RTE_DEV_TO_PCI(dev->device);
1035         dev_info->if_index = 0;
1036         dev_info->max_mac_addrs = 1;
1037         dev_info->max_rx_pktlen = (uint32_t)-1;
1038         dev_info->max_rx_queues = internals->max_rx_queues;
1039         dev_info->max_tx_queues = internals->max_tx_queues;
1040         dev_info->min_rx_bufsize = 0;
1041         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1042 }
1043
1044 static void
1045 eth_stats_get(struct rte_eth_dev *dev,
1046                 struct rte_eth_stats *stats)
1047 {
1048         uint16_t i;
1049         uint16_t nb_rx = dev->data->nb_rx_queues;
1050         uint16_t nb_tx = dev->data->nb_tx_queues;
1051         uint64_t rx_total = 0;
1052         uint64_t tx_total = 0;
1053         uint64_t tx_err_total = 0;
1054         uint64_t rx_total_bytes = 0;
1055         uint64_t tx_total_bytes = 0;
1056         const struct pmd_internals *internals = dev->data->dev_private;
1057
1058         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_rx; i++) {
1059                 stats->q_ipackets[i] = internals->rx_queue[i].rx_pkts;
1060                 stats->q_ibytes[i] = internals->rx_queue[i].rx_bytes;
1061                 rx_total += stats->q_ipackets[i];
1062                 rx_total_bytes += stats->q_ibytes[i];
1063         }
1064
1065         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_tx; i++) {
1066                 stats->q_opackets[i] = internals->tx_queue[i].tx_pkts;
1067                 stats->q_obytes[i] = internals->tx_queue[i].tx_bytes;
1068                 stats->q_errors[i] = internals->tx_queue[i].err_pkts;
1069                 tx_total += stats->q_opackets[i];
1070                 tx_total_bytes += stats->q_obytes[i];
1071                 tx_err_total += stats->q_errors[i];
1072         }
1073
1074         stats->ipackets = rx_total;
1075         stats->opackets = tx_total;
1076         stats->ibytes = rx_total_bytes;
1077         stats->obytes = tx_total_bytes;
1078         stats->oerrors = tx_err_total;
1079 }
1080
1081 static void
1082 eth_stats_reset(struct rte_eth_dev *dev)
1083 {
1084         uint16_t i;
1085         uint16_t nb_rx = dev->data->nb_rx_queues;
1086         uint16_t nb_tx = dev->data->nb_tx_queues;
1087         struct pmd_internals *internals = dev->data->dev_private;
1088
1089         for (i = 0; i < nb_rx; i++) {
1090                 internals->rx_queue[i].rx_pkts = 0;
1091                 internals->rx_queue[i].rx_bytes = 0;
1092                 internals->rx_queue[i].err_pkts = 0;
1093         }
1094         for (i = 0; i < nb_tx; i++) {
1095                 internals->tx_queue[i].tx_pkts = 0;
1096                 internals->tx_queue[i].tx_bytes = 0;
1097                 internals->tx_queue[i].err_pkts = 0;
1098         }
1099 }
1100
1101 static void
1102 eth_rx_queue_release(void *q)
1103 {
1104         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1105         if (rxq->sze != NULL) {
1106                 szedata_close(rxq->sze);
1107                 rxq->sze = NULL;
1108         }
1109 }
1110
1111 static void
1112 eth_tx_queue_release(void *q)
1113 {
1114         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1115         if (txq->sze != NULL) {
1116                 szedata_close(txq->sze);
1117                 txq->sze = NULL;
1118         }
1119 }
1120
1121 static void
1122 eth_dev_close(struct rte_eth_dev *dev)
1123 {
1124         uint16_t i;
1125         uint16_t nb_rx = dev->data->nb_rx_queues;
1126         uint16_t nb_tx = dev->data->nb_tx_queues;
1127
1128         eth_dev_stop(dev);
1129
1130         for (i = 0; i < nb_rx; i++) {
1131                 eth_rx_queue_release(dev->data->rx_queues[i]);
1132                 dev->data->rx_queues[i] = NULL;
1133         }
1134         dev->data->nb_rx_queues = 0;
1135         for (i = 0; i < nb_tx; i++) {
1136                 eth_tx_queue_release(dev->data->tx_queues[i]);
1137                 dev->data->tx_queues[i] = NULL;
1138         }
1139         dev->data->nb_tx_queues = 0;
1140 }
1141
1142 static int
1143 eth_link_update(struct rte_eth_dev *dev,
1144                 int wait_to_complete __rte_unused)
1145 {
1146         struct rte_eth_link link;
1147         struct rte_eth_link *link_ptr = &link;
1148         struct rte_eth_link *dev_link = &dev->data->dev_link;
1149         struct pmd_internals *internals = (struct pmd_internals *)
1150                 dev->data->dev_private;
1151         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1152                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1153                         volatile struct szedata2_cgmii_ibuf *);
1154
1155         switch (cgmii_link_speed(ibuf)) {
1156         case SZEDATA2_LINK_SPEED_10G:
1157                 link.link_speed = ETH_SPEED_NUM_10G;
1158                 break;
1159         case SZEDATA2_LINK_SPEED_40G:
1160                 link.link_speed = ETH_SPEED_NUM_40G;
1161                 break;
1162         case SZEDATA2_LINK_SPEED_100G:
1163                 link.link_speed = ETH_SPEED_NUM_100G;
1164                 break;
1165         default:
1166                 link.link_speed = ETH_SPEED_NUM_10G;
1167                 break;
1168         }
1169
1170         /* szedata2 uses only full duplex */
1171         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1172
1173         link.link_status = (cgmii_ibuf_is_enabled(ibuf) &&
1174                         cgmii_ibuf_is_link_up(ibuf)) ? ETH_LINK_UP : ETH_LINK_DOWN;
1175
1176         link.link_autoneg = ETH_LINK_SPEED_FIXED;
1177
1178         rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link,
1179                         *(uint64_t *)link_ptr);
1180
1181         return 0;
1182 }
1183
1184 static int
1185 eth_dev_set_link_up(struct rte_eth_dev *dev)
1186 {
1187         struct pmd_internals *internals = (struct pmd_internals *)
1188                 dev->data->dev_private;
1189         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1190                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1191                         volatile struct szedata2_cgmii_ibuf *);
1192         volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1193                         internals->pci_rsc, SZEDATA2_CGMII_OBUF_BASE_OFF,
1194                         volatile struct szedata2_cgmii_obuf *);
1195
1196         cgmii_ibuf_enable(ibuf);
1197         cgmii_obuf_enable(obuf);
1198         return 0;
1199 }
1200
1201 static int
1202 eth_dev_set_link_down(struct rte_eth_dev *dev)
1203 {
1204         struct pmd_internals *internals = (struct pmd_internals *)
1205                 dev->data->dev_private;
1206         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1207                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1208                         volatile struct szedata2_cgmii_ibuf *);
1209         volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1210                         internals->pci_rsc, SZEDATA2_CGMII_OBUF_BASE_OFF,
1211                         volatile struct szedata2_cgmii_obuf *);
1212
1213         cgmii_ibuf_disable(ibuf);
1214         cgmii_obuf_disable(obuf);
1215         return 0;
1216 }
1217
1218 static int
1219 eth_rx_queue_setup(struct rte_eth_dev *dev,
1220                 uint16_t rx_queue_id,
1221                 uint16_t nb_rx_desc __rte_unused,
1222                 unsigned int socket_id __rte_unused,
1223                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1224                 struct rte_mempool *mb_pool)
1225 {
1226         struct pmd_internals *internals = dev->data->dev_private;
1227         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1228         int ret;
1229         uint32_t rx = 1 << rx_queue_id;
1230         uint32_t tx = 0;
1231
1232         rxq->sze = szedata_open(internals->sze_dev);
1233         if (rxq->sze == NULL)
1234                 return -EINVAL;
1235         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1236         if (ret != 0 || rx == 0) {
1237                 szedata_close(rxq->sze);
1238                 rxq->sze = NULL;
1239                 return -EINVAL;
1240         }
1241         rxq->rx_channel = rx_queue_id;
1242         rxq->in_port = dev->data->port_id;
1243         rxq->mb_pool = mb_pool;
1244         rxq->rx_pkts = 0;
1245         rxq->rx_bytes = 0;
1246         rxq->err_pkts = 0;
1247
1248         dev->data->rx_queues[rx_queue_id] = rxq;
1249         return 0;
1250 }
1251
1252 static int
1253 eth_tx_queue_setup(struct rte_eth_dev *dev,
1254                 uint16_t tx_queue_id,
1255                 uint16_t nb_tx_desc __rte_unused,
1256                 unsigned int socket_id __rte_unused,
1257                 const struct rte_eth_txconf *tx_conf __rte_unused)
1258 {
1259         struct pmd_internals *internals = dev->data->dev_private;
1260         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1261         int ret;
1262         uint32_t rx = 0;
1263         uint32_t tx = 1 << tx_queue_id;
1264
1265         txq->sze = szedata_open(internals->sze_dev);
1266         if (txq->sze == NULL)
1267                 return -EINVAL;
1268         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1269         if (ret != 0 || tx == 0) {
1270                 szedata_close(txq->sze);
1271                 txq->sze = NULL;
1272                 return -EINVAL;
1273         }
1274         txq->tx_channel = tx_queue_id;
1275         txq->tx_pkts = 0;
1276         txq->tx_bytes = 0;
1277         txq->err_pkts = 0;
1278
1279         dev->data->tx_queues[tx_queue_id] = txq;
1280         return 0;
1281 }
1282
1283 static void
1284 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1285                 struct ether_addr *mac_addr __rte_unused)
1286 {
1287 }
1288
1289 static void
1290 eth_promiscuous_enable(struct rte_eth_dev *dev)
1291 {
1292         struct pmd_internals *internals = (struct pmd_internals *)
1293                 dev->data->dev_private;
1294         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1295                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1296                         volatile struct szedata2_cgmii_ibuf *);
1297         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_PROMISC);
1298 }
1299
1300 static void
1301 eth_promiscuous_disable(struct rte_eth_dev *dev)
1302 {
1303         struct pmd_internals *internals = (struct pmd_internals *)
1304                 dev->data->dev_private;
1305         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1306                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1307                         volatile struct szedata2_cgmii_ibuf *);
1308         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1309 }
1310
1311 static void
1312 eth_allmulticast_enable(struct rte_eth_dev *dev)
1313 {
1314         struct pmd_internals *internals = (struct pmd_internals *)
1315                 dev->data->dev_private;
1316         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1317                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1318                         volatile struct szedata2_cgmii_ibuf *);
1319         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1320 }
1321
1322 static void
1323 eth_allmulticast_disable(struct rte_eth_dev *dev)
1324 {
1325         struct pmd_internals *internals = (struct pmd_internals *)
1326                 dev->data->dev_private;
1327         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1328                         internals->pci_rsc, SZEDATA2_CGMII_IBUF_BASE_OFF,
1329                         volatile struct szedata2_cgmii_ibuf *);
1330         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1331 }
1332
1333 static const struct eth_dev_ops ops = {
1334         .dev_start          = eth_dev_start,
1335         .dev_stop           = eth_dev_stop,
1336         .dev_set_link_up    = eth_dev_set_link_up,
1337         .dev_set_link_down  = eth_dev_set_link_down,
1338         .dev_close          = eth_dev_close,
1339         .dev_configure      = eth_dev_configure,
1340         .dev_infos_get      = eth_dev_info,
1341         .promiscuous_enable   = eth_promiscuous_enable,
1342         .promiscuous_disable  = eth_promiscuous_disable,
1343         .allmulticast_enable  = eth_allmulticast_enable,
1344         .allmulticast_disable = eth_allmulticast_disable,
1345         .rx_queue_start     = eth_rx_queue_start,
1346         .rx_queue_stop      = eth_rx_queue_stop,
1347         .tx_queue_start     = eth_tx_queue_start,
1348         .tx_queue_stop      = eth_tx_queue_stop,
1349         .rx_queue_setup     = eth_rx_queue_setup,
1350         .tx_queue_setup     = eth_tx_queue_setup,
1351         .rx_queue_release   = eth_rx_queue_release,
1352         .tx_queue_release   = eth_tx_queue_release,
1353         .link_update        = eth_link_update,
1354         .stats_get          = eth_stats_get,
1355         .stats_reset        = eth_stats_reset,
1356         .mac_addr_set       = eth_mac_addr_set,
1357 };
1358
1359 /*
1360  * This function goes through sysfs and looks for an index of szedata2
1361  * device file (/dev/szedataIIX, where X is the index).
1362  *
1363  * @return
1364  *           0 on success
1365  *          -1 on error
1366  */
1367 static int
1368 get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index)
1369 {
1370         DIR *dir;
1371         struct dirent *entry;
1372         int ret;
1373         uint32_t tmp_index;
1374         FILE *fd;
1375         char pcislot_path[PATH_MAX];
1376         uint32_t domain;
1377         uint32_t bus;
1378         uint32_t devid;
1379         uint32_t function;
1380
1381         dir = opendir("/sys/class/combo");
1382         if (dir == NULL)
1383                 return -1;
1384
1385         /*
1386          * Iterate through all combosixX directories.
1387          * When the value in /sys/class/combo/combosixX/device/pcislot
1388          * file is the location of the ethernet device dev, "X" is the
1389          * index of the device.
1390          */
1391         while ((entry = readdir(dir)) != NULL) {
1392                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1393                 if (ret != 1)
1394                         continue;
1395
1396                 snprintf(pcislot_path, PATH_MAX,
1397                         "/sys/class/combo/combosix%u/device/pcislot",
1398                         tmp_index);
1399
1400                 fd = fopen(pcislot_path, "r");
1401                 if (fd == NULL)
1402                         continue;
1403
1404                 ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8,
1405                                 &domain, &bus, &devid, &function);
1406                 fclose(fd);
1407                 if (ret != 4)
1408                         continue;
1409
1410                 if (pcislot_addr->domain == domain &&
1411                                 pcislot_addr->bus == bus &&
1412                                 pcislot_addr->devid == devid &&
1413                                 pcislot_addr->function == function) {
1414                         *index = tmp_index;
1415                         closedir(dir);
1416                         return 0;
1417                 }
1418         }
1419
1420         closedir(dir);
1421         return -1;
1422 }
1423
1424 static int
1425 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1426 {
1427         struct rte_eth_dev_data *data = dev->data;
1428         struct pmd_internals *internals = (struct pmd_internals *)
1429                 data->dev_private;
1430         struct szedata *szedata_temp;
1431         int ret;
1432         uint32_t szedata2_index;
1433         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1434         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1435         struct rte_mem_resource *pci_rsc =
1436                 &pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1437         char rsc_filename[PATH_MAX];
1438         void *pci_resource_ptr = NULL;
1439         int fd;
1440
1441         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1442                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1443                         pci_addr->function);
1444
1445         /* Get index of szedata2 device file and create path to device file */
1446         ret = get_szedata2_index(pci_addr, &szedata2_index);
1447         if (ret != 0) {
1448                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1449                 return -ENODEV;
1450         }
1451         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1452                         szedata2_index);
1453
1454         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1455
1456         /*
1457          * Get number of available DMA RX and TX channels, which is maximum
1458          * number of queues that can be created and store it in private device
1459          * data structure.
1460          */
1461         szedata_temp = szedata_open(internals->sze_dev);
1462         if (szedata_temp == NULL) {
1463                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1464                                 internals->sze_dev);
1465                 return -EINVAL;
1466         }
1467         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1468                         SZE2_DIR_RX);
1469         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1470                         SZE2_DIR_TX);
1471         szedata_close(szedata_temp);
1472
1473         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1474                         internals->max_rx_queues, internals->max_tx_queues);
1475
1476         /* Set rx, tx burst functions */
1477         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1478                 data->scattered_rx == 1) {
1479                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1480                 data->scattered_rx = 1;
1481         } else {
1482                 dev->rx_pkt_burst = eth_szedata2_rx;
1483                 data->scattered_rx = 0;
1484         }
1485         dev->tx_pkt_burst = eth_szedata2_tx;
1486
1487         /* Set function callbacks for Ethernet API */
1488         dev->dev_ops = &ops;
1489
1490         rte_eth_copy_pci_info(dev, pci_dev);
1491
1492         /* mmap pci resource0 file to rte_mem_resource structure */
1493         if (pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1494                         0) {
1495                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1496                                 PCI_RESOURCE_NUMBER);
1497                 return -EINVAL;
1498         }
1499         snprintf(rsc_filename, PATH_MAX,
1500                 "%s/" PCI_PRI_FMT "/resource%u", pci_get_sysfs_path(),
1501                 pci_addr->domain, pci_addr->bus,
1502                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1503         fd = open(rsc_filename, O_RDWR);
1504         if (fd < 0) {
1505                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1506                 return -EINVAL;
1507         }
1508
1509         pci_resource_ptr = mmap(0,
1510                         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1511                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1512         close(fd);
1513         if (pci_resource_ptr == NULL) {
1514                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1515                                 rsc_filename, fd);
1516                 return -EINVAL;
1517         }
1518         pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr = pci_resource_ptr;
1519         internals->pci_rsc = pci_rsc;
1520
1521         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1522                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1523                         (unsigned long long)pci_rsc->phys_addr,
1524                         (unsigned long long)pci_rsc->len,
1525                         (unsigned long long)pci_rsc->addr);
1526
1527         /* Get link state */
1528         eth_link_update(dev, 0);
1529
1530         /* Allocate space for one mac address */
1531         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1532                         RTE_CACHE_LINE_SIZE);
1533         if (data->mac_addrs == NULL) {
1534                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1535                 munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1536                        pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1537                 return -EINVAL;
1538         }
1539
1540         ether_addr_copy(&eth_addr, data->mac_addrs);
1541
1542         /* At initial state COMBO card is in promiscuous mode so disable it */
1543         eth_promiscuous_disable(dev);
1544
1545         RTE_LOG(INFO, PMD, "szedata2 device ("
1546                         PCI_PRI_FMT ") successfully initialized\n",
1547                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1548                         pci_addr->function);
1549
1550         return 0;
1551 }
1552
1553 static int
1554 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1555 {
1556         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1557         struct rte_pci_addr *pci_addr = &pci_dev->addr;
1558
1559         rte_free(dev->data->mac_addrs);
1560         dev->data->mac_addrs = NULL;
1561         munmap(pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1562                pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1563
1564         RTE_LOG(INFO, PMD, "szedata2 device ("
1565                         PCI_PRI_FMT ") successfully uninitialized\n",
1566                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1567                         pci_addr->function);
1568
1569         return 0;
1570 }
1571
1572 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1573         {
1574                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1575                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1576         },
1577         {
1578                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1579                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1580         },
1581         {
1582                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1583                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1584         },
1585         {
1586                 .vendor_id = 0,
1587         }
1588 };
1589
1590 static struct eth_driver szedata2_eth_driver = {
1591         .pci_drv = {
1592                 .id_table = rte_szedata2_pci_id_table,
1593                 .probe = rte_eth_dev_pci_probe,
1594                 .remove = rte_eth_dev_pci_remove,
1595         },
1596         .eth_dev_init     = rte_szedata2_eth_dev_init,
1597         .eth_dev_uninit   = rte_szedata2_eth_dev_uninit,
1598         .dev_private_size = sizeof(struct pmd_internals),
1599 };
1600
1601 RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver.pci_drv);
1602 RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1603 RTE_PMD_REGISTER_KMOD_DEP(RTE_SZEDATA2_DRIVER_NAME,
1604         "* combo6core & combov3 & szedata2 & szedata2_cv3");