drivers/net/mpipe/mpipe_tilegx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015 EZchip Semiconductor Ltd. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of EZchip Semiconductor nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <unistd.h>
  34
  35 #include <rte_eal.h>
  36 #include <rte_dev.h>
  37 #include <rte_eal_memconfig.h>
  38 #include <rte_ethdev.h>
  39 #include <rte_malloc.h>
  40 #include <rte_cycles.h>
  41
  42 #include <arch/mpipe_xaui_def.h>
  43 #include <arch/mpipe_gbe_def.h>
  44
  45 #include <gxio/mpipe.h>
  46
  47 #ifdef RTE_LIBRTE_MPIPE_PMD_DEBUG
  48 #define PMD_DEBUG_RX(...)       RTE_LOG(DEBUG, PMD, __VA_ARGS__)
  49 #define PMD_DEBUG_TX(...)       RTE_LOG(DEBUG, PMD, __VA_ARGS__)
  50 #else
  51 #define PMD_DEBUG_RX(...)
  52 #define PMD_DEBUG_TX(...)
  53 #endif
  54
  55 #define MPIPE_MAX_CHANNELS              128
  56 #define MPIPE_TX_MAX_QUEUES             128
  57 #define MPIPE_RX_MAX_QUEUES             16
  58 #define MPIPE_TX_DESCS                  512
  59 #define MPIPE_RX_BUCKETS                256
  60 #define MPIPE_RX_STACK_SIZE             65536
  61 #define MPIPE_RX_IP_ALIGN               2
  62 #define MPIPE_BSM_ALIGN                 128
  63
  64 #define MPIPE_LINK_UPDATE_TIMEOUT       10      /*  s */
  65 #define MPIPE_LINK_UPDATE_INTERVAL      100000  /* us */
  66
  67 struct mpipe_channel_config {
  68         int enable;
  69         int first_bucket;
  70         int num_buckets;
  71         int head_room;
  72         gxio_mpipe_rules_stacks_t stacks;
  73 };
  74
  75 struct mpipe_context {
  76         rte_spinlock_t        lock;
  77         gxio_mpipe_context_t  context;
  78         struct mpipe_channel_config channels[MPIPE_MAX_CHANNELS];
  79 };
  80
  81 /* Per-core local data. */
  82 struct mpipe_local {
  83         int mbuf_push_debt[RTE_MAX_ETHPORTS];   /* Buffer push debt. */
  84 } __rte_cache_aligned;
  85
  86 #define MPIPE_BUF_DEBT_THRESHOLD        32
  87 static __thread struct mpipe_local mpipe_local;
  88 static struct mpipe_context mpipe_contexts[GXIO_MPIPE_INSTANCE_MAX];
  89 static int mpipe_instances;
  90 static const char *drivername = "MPIPE PMD";
  91
  92 /* Per queue statistics. */
  93 struct mpipe_queue_stats {
  94         uint64_t packets, bytes, errors, nomem;
  95 };
  96
  97 /* Common tx/rx queue fields. */
  98 struct mpipe_queue {
  99         struct mpipe_dev_priv *priv;    /* "priv" data of its device. */
 100         uint16_t nb_desc;               /* Number of tx descriptors. */
 101         uint16_t port_id;               /* Device index. */
 102         uint16_t stat_idx;              /* Queue stats index. */
 103         uint8_t queue_idx;              /* Queue index. */
 104         uint8_t link_status;            /* 0 = link down. */
 105         struct mpipe_queue_stats stats; /* Stat data for the queue. */
 106 };
 107
 108 /* Transmit queue description. */
 109 struct mpipe_tx_queue {
 110         struct mpipe_queue q;           /* Common stuff. */
 111 };
 112
 113 /* Receive queue description. */
 114 struct mpipe_rx_queue {
 115         struct mpipe_queue q;           /* Common stuff. */
 116         gxio_mpipe_iqueue_t iqueue;     /* mPIPE iqueue. */
 117         gxio_mpipe_idesc_t *next_desc;  /* Next idesc to process. */
 118         int avail_descs;                /* Number of available descs. */
 119         void *rx_ring_mem;              /* DMA ring memory. */
 120 };
 121
 122 struct mpipe_dev_priv {
 123         gxio_mpipe_context_t *context;  /* mPIPE context. */
 124         gxio_mpipe_link_t link;         /* mPIPE link for the device. */
 125         gxio_mpipe_equeue_t equeue;     /* mPIPE equeue. */
 126         unsigned equeue_size;           /* mPIPE equeue desc count. */
 127         int instance;                   /* mPIPE instance. */
 128         int ering;                      /* mPIPE eDMA ring. */
 129         int stack;                      /* mPIPE buffer stack. */
 130         int channel;                    /* Device channel. */
 131         int port_id;                    /* DPDK port index. */
 132         struct rte_eth_dev *eth_dev;    /* DPDK device. */
 133         struct rte_mbuf **tx_comps;     /* TX completion array. */
 134         struct rte_mempool *rx_mpool;   /* mpool used by the rx queues. */
 135         unsigned rx_offset;             /* Receive head room. */
 136         unsigned rx_size_code;          /* mPIPE rx buffer size code. */
 137         unsigned rx_buffers;            /* receive buffers on stack. */
 138         int is_xaui:1,                  /* Is this an xgbe or gbe? */
 139             initialized:1,              /* Initialized port? */
 140             running:1;                  /* Running port? */
 141         struct ether_addr mac_addr;     /* MAC address. */
 142         unsigned nb_rx_queues;          /* Configured tx queues. */
 143         unsigned nb_tx_queues;          /* Configured rx queues. */
 144         int first_bucket;               /* mPIPE bucket start index. */
 145         int first_ring;                 /* mPIPE notif ring start index. */
 146         int notif_group;                /* mPIPE notif group. */
 147         rte_atomic32_t dp_count __rte_cache_aligned;    /* DP Entry count. */
 148         int tx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
 149         int rx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
 150 };
 151
 152 #define mpipe_priv(dev)                 \
 153         ((struct mpipe_dev_priv*)(dev)->data->dev_private)
 154
 155 #define mpipe_name(priv)                \
 156         ((priv)->eth_dev->data->name)
 157
 158 #define mpipe_rx_queue(priv, n)         \
 159         ((struct mpipe_rx_queue *)(priv)->eth_dev->data->rx_queues[n])
 160
 161 #define mpipe_tx_queue(priv, n)         \
 162         ((struct mpipe_tx_queue *)(priv)->eth_dev->data->tx_queues[n])
 163
 164 static void
 165 mpipe_xmit_flush(struct mpipe_dev_priv *priv);
 166
 167 static void
 168 mpipe_recv_flush(struct mpipe_dev_priv *priv);
 169
 170 static int mpipe_equeue_sizes[] = {
 171         [GXIO_MPIPE_EQUEUE_ENTRY_512]   = 512,
 172         [GXIO_MPIPE_EQUEUE_ENTRY_2K]    = 2048,
 173         [GXIO_MPIPE_EQUEUE_ENTRY_8K]    = 8192,
 174         [GXIO_MPIPE_EQUEUE_ENTRY_64K]   = 65536,
 175 };
 176
 177 static int mpipe_iqueue_sizes[] = {
 178         [GXIO_MPIPE_IQUEUE_ENTRY_128]   = 128,
 179         [GXIO_MPIPE_IQUEUE_ENTRY_512]   = 512,
 180         [GXIO_MPIPE_IQUEUE_ENTRY_2K]    = 2048,
 181         [GXIO_MPIPE_IQUEUE_ENTRY_64K]   = 65536,
 182 };
 183
 184 static int mpipe_buffer_sizes[] = {
 185         [GXIO_MPIPE_BUFFER_SIZE_128]    = 128,
 186         [GXIO_MPIPE_BUFFER_SIZE_256]    = 256,
 187         [GXIO_MPIPE_BUFFER_SIZE_512]    = 512,
 188         [GXIO_MPIPE_BUFFER_SIZE_1024]   = 1024,
 189         [GXIO_MPIPE_BUFFER_SIZE_1664]   = 1664,
 190         [GXIO_MPIPE_BUFFER_SIZE_4096]   = 4096,
 191         [GXIO_MPIPE_BUFFER_SIZE_10368]  = 10368,
 192         [GXIO_MPIPE_BUFFER_SIZE_16384]  = 16384,
 193 };
 194
 195 static gxio_mpipe_context_t *
 196 mpipe_context(int instance)
 197 {
 198         if (instance < 0 || instance >= mpipe_instances)
 199                 return NULL;
 200         return &mpipe_contexts[instance].context;
 201 }
 202
 203 static int mpipe_channel_config(int instance, int channel,
 204                                 struct mpipe_channel_config *config)
 205 {
 206         struct mpipe_channel_config *data;
 207         struct mpipe_context *context;
 208         gxio_mpipe_rules_t rules;
 209         int idx, rc = 0;
 210
 211         if (instance < 0 || instance >= mpipe_instances ||
 212             channel < 0 || channel >= MPIPE_MAX_CHANNELS)
 213                 return -EINVAL;
 214
 215         context = &mpipe_contexts[instance];
 216
 217         rte_spinlock_lock(&context->lock);
 218
 219         gxio_mpipe_rules_init(&rules, &context->context);
 220
 221         for (idx = 0; idx < MPIPE_MAX_CHANNELS; idx++) {
 222                 data = (channel == idx) ? config : &context->channels[idx];
 223
 224                 if (!data->enable)
 225                         continue;
 226
 227                 rc = gxio_mpipe_rules_begin(&rules, data->first_bucket,
 228                                             data->num_buckets, &data->stacks);
 229                 if (rc < 0) {
 230                         goto done;
 231                 }
 232
 233                 rc = gxio_mpipe_rules_add_channel(&rules, idx);
 234                 if (rc < 0) {
 235                         goto done;
 236                 }
 237
 238                 rc = gxio_mpipe_rules_set_headroom(&rules, data->head_room);
 239                 if (rc < 0) {
 240                         goto done;
 241                 }
 242         }
 243
 244         rc = gxio_mpipe_rules_commit(&rules);
 245         if (rc == 0) {
 246                 memcpy(&context->channels[channel], config, sizeof(*config));
 247         }
 248
 249 done:
 250         rte_spinlock_unlock(&context->lock);
 251
 252         return rc;
 253 }
 254
 255 static int
 256 mpipe_get_size_index(int *array, int count, int size,
 257                      bool roundup)
 258 {
 259         int i, last = -1;
 260
 261         for (i = 0; i < count && array[i] < size; i++) {
 262                 if (array[i])
 263                         last = i;
 264         }
 265
 266         if (roundup)
 267                 return i < count ? (int)i : -ENOENT;
 268         else
 269                 return last >= 0 ? last : -ENOENT;
 270 }
 271
 272 static int
 273 mpipe_calc_size(int *array, int count, int size)
 274 {
 275         int index = mpipe_get_size_index(array, count, size, 1);
 276         return index < 0 ? index : array[index];
 277 }
 278
 279 static int mpipe_equeue_size(int size)
 280 {
 281         int result;
 282         result = mpipe_calc_size(mpipe_equeue_sizes,
 283                                  RTE_DIM(mpipe_equeue_sizes), size);
 284         return result;
 285 }
 286
 287 static int mpipe_iqueue_size(int size)
 288 {
 289         int result;
 290         result = mpipe_calc_size(mpipe_iqueue_sizes,
 291                                  RTE_DIM(mpipe_iqueue_sizes), size);
 292         return result;
 293 }
 294
 295 static int mpipe_buffer_size_index(int size)
 296 {
 297         int result;
 298         result = mpipe_get_size_index(mpipe_buffer_sizes,
 299                                       RTE_DIM(mpipe_buffer_sizes), size, 0);
 300         return result;
 301 }
 302
 303 static inline int
 304 mpipe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
 305                                   struct rte_eth_link *link)
 306 {
 307         struct rte_eth_link *dst = link;
 308         struct rte_eth_link *src = &(dev->data->dev_link);
 309
 310         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 311                                 *(uint64_t *)src) == 0)
 312                 return -1;
 313
 314         return 0;
 315 }
 316
 317 static inline int
 318 mpipe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
 319                                    struct rte_eth_link *link)
 320 {
 321         struct rte_eth_link *dst = &(dev->data->dev_link);
 322         struct rte_eth_link *src = link;
 323
 324         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
 325                                 *(uint64_t *)src) == 0)
 326                 return -1;
 327
 328         return 0;
 329 }
 330
 331 static void
 332 mpipe_infos_get(struct rte_eth_dev *dev __rte_unused,
 333                 struct rte_eth_dev_info *dev_info)
 334 {
 335         dev_info->min_rx_bufsize  = 128;
 336         dev_info->max_rx_pktlen   = 1518;
 337         dev_info->max_tx_queues   = MPIPE_TX_MAX_QUEUES;
 338         dev_info->max_rx_queues   = MPIPE_RX_MAX_QUEUES;
 339         dev_info->max_mac_addrs   = 1;
 340         dev_info->rx_offload_capa = 0;
 341         dev_info->tx_offload_capa = 0;
 342 }
 343
 344 static int
 345 mpipe_configure(struct rte_eth_dev *dev)
 346 {
 347         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 348
 349         if (dev->data->nb_tx_queues > MPIPE_TX_MAX_QUEUES) {
 350                 RTE_LOG(ERR, PMD, "%s: Too many tx queues: %d > %d\n",
 351                         mpipe_name(priv), dev->data->nb_tx_queues,
 352                         MPIPE_TX_MAX_QUEUES);
 353                 return -EINVAL;
 354         }
 355         priv->nb_tx_queues = dev->data->nb_tx_queues;
 356
 357         if (dev->data->nb_rx_queues > MPIPE_RX_MAX_QUEUES) {
 358                 RTE_LOG(ERR, PMD, "%s: Too many rx queues: %d > %d\n",
 359                         mpipe_name(priv), dev->data->nb_rx_queues,
 360                         MPIPE_RX_MAX_QUEUES);
 361         }
 362         priv->nb_rx_queues = dev->data->nb_rx_queues;
 363
 364         return 0;
 365 }
 366
 367 static inline int
 368 mpipe_link_compare(struct rte_eth_link *link1,
 369                    struct rte_eth_link *link2)
 370 {
 371         return (*(uint64_t *)link1 == *(uint64_t *)link2)
 372                 ? -1 : 0;
 373 }
 374
 375 static int
 376 mpipe_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 377 {
 378         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 379         struct rte_eth_link old, new;
 380         int64_t state, speed;
 381         int count, rc;
 382
 383         memset(&old, 0, sizeof(old));
 384         memset(&new, 0, sizeof(new));
 385         mpipe_dev_atomic_read_link_status(dev, &old);
 386
 387         for (count = 0, rc = 0; count < MPIPE_LINK_UPDATE_TIMEOUT; count++) {
 388                 if (!priv->initialized)
 389                         break;
 390
 391                 state = gxio_mpipe_link_get_attr(&priv->link,
 392                                                  GXIO_MPIPE_LINK_CURRENT_STATE);
 393                 if (state < 0)
 394                         break;
 395
 396                 speed = state & GXIO_MPIPE_LINK_SPEED_MASK;
 397
 398                 if (speed == GXIO_MPIPE_LINK_1G) {
 399                         new.link_speed = ETH_LINK_SPEED_1000;
 400                         new.link_duplex = ETH_LINK_FULL_DUPLEX;
 401                         new.link_status = 1;
 402                 } else if (speed == GXIO_MPIPE_LINK_10G) {
 403                         new.link_speed = ETH_LINK_SPEED_10000;
 404                         new.link_duplex = ETH_LINK_FULL_DUPLEX;
 405                         new.link_status = 1;
 406                 }
 407
 408                 rc = mpipe_link_compare(&old, &new);
 409                 if (rc == 0 || !wait_to_complete)
 410                         break;
 411
 412                 rte_delay_us(MPIPE_LINK_UPDATE_INTERVAL);
 413         }
 414
 415         mpipe_dev_atomic_write_link_status(dev, &new);
 416         return rc;
 417 }
 418
 419 static int
 420 mpipe_set_link(struct rte_eth_dev *dev, int up)
 421 {
 422         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 423         int rc;
 424
 425         rc = gxio_mpipe_link_set_attr(&priv->link,
 426                                       GXIO_MPIPE_LINK_DESIRED_STATE,
 427                                       up ? GXIO_MPIPE_LINK_ANYSPEED : 0);
 428         if (rc < 0) {
 429                 RTE_LOG(ERR, PMD, "%s: Failed to set link %s.\n",
 430                         mpipe_name(priv), up ? "up" : "down");
 431         } else {
 432                 mpipe_link_update(dev, 0);
 433         }
 434
 435         return rc;
 436 }
 437
 438 static int
 439 mpipe_set_link_up(struct rte_eth_dev *dev)
 440 {
 441         return mpipe_set_link(dev, 1);
 442 }
 443
 444 static int
 445 mpipe_set_link_down(struct rte_eth_dev *dev)
 446 {
 447         return mpipe_set_link(dev, 0);
 448 }
 449
 450 static inline void
 451 mpipe_dp_enter(struct mpipe_dev_priv *priv)
 452 {
 453         __insn_mtspr(SPR_DSTREAM_PF, 0);
 454         rte_atomic32_inc(&priv->dp_count);
 455 }
 456
 457 static inline void
 458 mpipe_dp_exit(struct mpipe_dev_priv *priv)
 459 {
 460         rte_atomic32_dec(&priv->dp_count);
 461 }
 462
 463 static inline void
 464 mpipe_dp_wait(struct mpipe_dev_priv *priv)
 465 {
 466         while (rte_atomic32_read(&priv->dp_count) != 0) {
 467                 rte_pause();
 468         }
 469 }
 470
 471 static inline int
 472 mpipe_mbuf_stack_index(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
 473 {
 474         return (mbuf->port < RTE_MAX_ETHPORTS) ?
 475                 mpipe_priv(&rte_eth_devices[mbuf->port])->stack :
 476                 priv->stack;
 477 }
 478
 479 static inline struct rte_mbuf *
 480 mpipe_recv_mbuf(struct mpipe_dev_priv *priv, gxio_mpipe_idesc_t *idesc,
 481                 int in_port)
 482 {
 483         void *va = gxio_mpipe_idesc_get_va(idesc);
 484         uint16_t size = gxio_mpipe_idesc_get_xfer_size(idesc);
 485         struct rte_mbuf *mbuf = RTE_PTR_SUB(va, priv->rx_offset);
 486
 487         rte_pktmbuf_reset(mbuf);
 488         mbuf->data_off = (uintptr_t)va - (uintptr_t)mbuf->buf_addr;
 489         mbuf->port     = in_port;
 490         mbuf->data_len = size;
 491         mbuf->pkt_len  = size;
 492         mbuf->hash.rss = gxio_mpipe_idesc_get_flow_hash(idesc);
 493
 494         PMD_DEBUG_RX("%s: RX mbuf %p, buffer %p, buf_addr %p, size %d\n",
 495                      mpipe_name(priv), mbuf, va, mbuf->buf_addr, size);
 496
 497         return mbuf;
 498 }
 499
 500 static inline void
 501 mpipe_recv_push(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
 502 {
 503         const int offset = RTE_PKTMBUF_HEADROOM + MPIPE_RX_IP_ALIGN;
 504         void *buf_addr = RTE_PTR_ADD(mbuf->buf_addr, offset);
 505
 506         gxio_mpipe_push_buffer(priv->context, priv->stack, buf_addr);
 507         PMD_DEBUG_RX("%s: Pushed mbuf %p, buffer %p into stack %d\n",
 508                      mpipe_name(priv), mbuf, buf_addr, priv->stack);
 509 }
 510
 511 static inline void
 512 mpipe_recv_fill_stack(struct mpipe_dev_priv *priv, int count)
 513 {
 514         struct rte_mbuf *mbuf;
 515         int i;
 516
 517         for (i = 0; i < count; i++) {
 518                 mbuf = __rte_mbuf_raw_alloc(priv->rx_mpool);
 519                 if (!mbuf)
 520                         break;
 521                 mpipe_recv_push(priv, mbuf);
 522         }
 523
 524         priv->rx_buffers += count;
 525         PMD_DEBUG_RX("%s: Filled %d/%d buffers\n", mpipe_name(priv), i, count);
 526 }
 527
 528 static inline void
 529 mpipe_recv_flush_stack(struct mpipe_dev_priv *priv)
 530 {
 531         const int offset = priv->rx_offset & ~RTE_MEMPOOL_ALIGN_MASK;
 532         uint8_t in_port = priv->port_id;
 533         struct rte_mbuf *mbuf;
 534         unsigned count;
 535         void *va;
 536
 537         for (count = 0; count < priv->rx_buffers; count++) {
 538                 va = gxio_mpipe_pop_buffer(priv->context, priv->stack);
 539                 if (!va)
 540                         break;
 541                 mbuf = RTE_PTR_SUB(va, offset);
 542
 543                 PMD_DEBUG_RX("%s: Flushing mbuf %p, va %p\n",
 544                              mpipe_name(priv), mbuf, va);
 545
 546                 mbuf->data_off    = (uintptr_t)va - (uintptr_t)mbuf->buf_addr;
 547                 mbuf->refcnt      = 1;
 548                 mbuf->nb_segs     = 1;
 549                 mbuf->port        = in_port;
 550                 mbuf->packet_type = 0;
 551                 mbuf->data_len    = 0;
 552                 mbuf->pkt_len     = 0;
 553
 554                 __rte_mbuf_raw_free(mbuf);
 555         }
 556
 557         PMD_DEBUG_RX("%s: Returned %d/%d buffers\n",
 558                      mpipe_name(priv), count, priv->rx_buffers);
 559         priv->rx_buffers -= count;
 560 }
 561
 562 static void
 563 mpipe_register_segment(struct mpipe_dev_priv *priv, const struct rte_memseg *ms)
 564 {
 565         size_t size = ms->hugepage_sz;
 566         uint8_t *addr, *end;
 567         int rc;
 568
 569         for (addr = ms->addr, end = addr + ms->len; addr < end; addr += size) {
 570                 rc = gxio_mpipe_register_page(priv->context, priv->stack, addr,
 571                                               size, 0);
 572                 if (rc < 0)
 573                         break;
 574         }
 575
 576         if (rc < 0) {
 577                 RTE_LOG(ERR, PMD, "%s: Could not register memseg @%p, %d.\n",
 578                         mpipe_name(priv), ms->addr, rc);
 579         } else {
 580                 RTE_LOG(DEBUG, PMD, "%s: Registered segment %p - %p\n",
 581                         mpipe_name(priv), ms->addr,
 582                         RTE_PTR_ADD(ms->addr, ms->len - 1));
 583         }
 584 }
 585
 586 static int
 587 mpipe_recv_init(struct mpipe_dev_priv *priv)
 588 {
 589         const struct rte_memseg *seg = rte_eal_get_physmem_layout();
 590         size_t stack_size;
 591         void *stack_mem;
 592         int rc;
 593
 594         if (!priv->rx_mpool) {
 595                 RTE_LOG(ERR, PMD, "%s: No buffer pool.\n",
 596                         mpipe_name(priv));
 597                 return -ENODEV;
 598         }
 599
 600         /* Allocate one NotifRing for each queue. */
 601         rc = gxio_mpipe_alloc_notif_rings(priv->context, MPIPE_RX_MAX_QUEUES,
 602                                           0, 0);
 603         if (rc < 0) {
 604                 RTE_LOG(ERR, PMD, "%s: Failed to allocate notif rings.\n",
 605                         mpipe_name(priv));
 606                 return rc;
 607         }
 608         priv->first_ring = rc;
 609
 610         /* Allocate a NotifGroup. */
 611         rc = gxio_mpipe_alloc_notif_groups(priv->context, 1, 0, 0);
 612         if (rc < 0) {
 613                 RTE_LOG(ERR, PMD, "%s: Failed to allocate rx group.\n",
 614                         mpipe_name(priv));
 615                 return rc;
 616         }
 617         priv->notif_group = rc;
 618
 619         /* Allocate required buckets. */
 620         rc = gxio_mpipe_alloc_buckets(priv->context, MPIPE_RX_BUCKETS, 0, 0);
 621         if (rc < 0) {
 622                 RTE_LOG(ERR, PMD, "%s: Failed to allocate buckets.\n",
 623                         mpipe_name(priv));
 624                 return rc;
 625         }
 626         priv->first_bucket = rc;
 627
 628         rc = gxio_mpipe_alloc_buffer_stacks(priv->context, 1, 0, 0);
 629         if (rc < 0) {
 630                 RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer stack.\n",
 631                         mpipe_name(priv));
 632                 return rc;
 633         }
 634         priv->stack = rc;
 635
 636         while (seg && seg->addr)
 637                 mpipe_register_segment(priv, seg++);
 638
 639         stack_size = gxio_mpipe_calc_buffer_stack_bytes(MPIPE_RX_STACK_SIZE);
 640         stack_mem = rte_zmalloc(NULL, stack_size, 65536);
 641         if (!stack_mem) {
 642                 RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer memory.\n",
 643                         mpipe_name(priv));
 644                 return -ENOMEM;
 645         } else {
 646                 RTE_LOG(DEBUG, PMD, "%s: Buffer stack memory %p - %p.\n",
 647                         mpipe_name(priv), stack_mem,
 648                         RTE_PTR_ADD(stack_mem, stack_size - 1));
 649         }
 650
 651         rc = gxio_mpipe_init_buffer_stack(priv->context, priv->stack,
 652                                           priv->rx_size_code, stack_mem,
 653                                           stack_size, 0);
 654         if (rc < 0) {
 655                 RTE_LOG(ERR, PMD, "%s: Failed to initialize buffer stack.\n",
 656                         mpipe_name(priv));
 657                 return rc;
 658         }
 659
 660         return 0;
 661 }
 662
 663 static int
 664 mpipe_xmit_init(struct mpipe_dev_priv *priv)
 665 {
 666         size_t ring_size;
 667         void *ring_mem;
 668         int rc;
 669
 670         /* Allocate eDMA ring. */
 671         rc = gxio_mpipe_alloc_edma_rings(priv->context, 1, 0, 0);
 672         if (rc < 0) {
 673                 RTE_LOG(ERR, PMD, "%s: Failed to alloc tx ring.\n",
 674                         mpipe_name(priv));
 675                 return rc;
 676         }
 677         priv->ering = rc;
 678
 679         rc = mpipe_equeue_size(MPIPE_TX_DESCS);
 680         if (rc < 0) {
 681                 RTE_LOG(ERR, PMD, "%s: Cannot allocate %d equeue descs.\n",
 682                         mpipe_name(priv), (int)MPIPE_TX_DESCS);
 683                 return -ENOMEM;
 684         }
 685         priv->equeue_size = rc;
 686
 687         /* Initialize completion array. */
 688         ring_size = sizeof(priv->tx_comps[0]) * priv->equeue_size;
 689         priv->tx_comps = rte_zmalloc(NULL, ring_size, RTE_CACHE_LINE_SIZE);
 690         if (!priv->tx_comps) {
 691                 RTE_LOG(ERR, PMD, "%s: Failed to allocate egress comps.\n",
 692                         mpipe_name(priv));
 693                 return -ENOMEM;
 694         }
 695
 696         /* Allocate eDMA ring memory. */
 697         ring_size = sizeof(gxio_mpipe_edesc_t) * priv->equeue_size;
 698         ring_mem = rte_zmalloc(NULL, ring_size, ring_size);
 699         if (!ring_mem) {
 700                 RTE_LOG(ERR, PMD, "%s: Failed to allocate egress descs.\n",
 701                         mpipe_name(priv));
 702                 return -ENOMEM;
 703         } else {
 704                 RTE_LOG(DEBUG, PMD, "%s: eDMA ring memory %p - %p.\n",
 705                         mpipe_name(priv), ring_mem,
 706                         RTE_PTR_ADD(ring_mem, ring_size - 1));
 707         }
 708
 709         /* Initialize eDMA ring. */
 710         rc = gxio_mpipe_equeue_init(&priv->equeue, priv->context, priv->ering,
 711                                     priv->channel, ring_mem, ring_size, 0);
 712         if (rc < 0) {
 713                 RTE_LOG(ERR, PMD, "%s: Failed to init equeue\n",
 714                         mpipe_name(priv));
 715                 return rc;
 716         }
 717
 718         return 0;
 719 }
 720
 721 static int
 722 mpipe_link_init(struct mpipe_dev_priv *priv)
 723 {
 724         int rc;
 725
 726         /* Open the link. */
 727         rc = gxio_mpipe_link_open(&priv->link, priv->context,
 728                                   mpipe_name(priv), GXIO_MPIPE_LINK_AUTO_NONE);
 729         if (rc < 0) {
 730                 RTE_LOG(ERR, PMD, "%s: Failed to open link.\n",
 731                         mpipe_name(priv));
 732                 return rc;
 733         }
 734
 735         /* Get the channel index. */
 736         rc = gxio_mpipe_link_channel(&priv->link);
 737         if (rc < 0) {
 738                 RTE_LOG(ERR, PMD, "%s: Bad channel\n",
 739                         mpipe_name(priv));
 740                 return rc;
 741         }
 742         priv->channel = rc;
 743
 744         return 0;
 745 }
 746
 747 static int
 748 mpipe_init(struct mpipe_dev_priv *priv)
 749 {
 750         int rc;
 751
 752         if (priv->initialized)
 753                 return 0;
 754
 755         rc = mpipe_link_init(priv);
 756         if (rc < 0) {
 757                 RTE_LOG(ERR, PMD, "%s: Failed to init link.\n",
 758                         mpipe_name(priv));
 759                 return rc;
 760         }
 761
 762         rc = mpipe_recv_init(priv);
 763         if (rc < 0) {
 764                 RTE_LOG(ERR, PMD, "%s: Failed to init rx.\n",
 765                         mpipe_name(priv));
 766                 return rc;
 767         }
 768
 769         rc = mpipe_xmit_init(priv);
 770         if (rc < 0) {
 771                 RTE_LOG(ERR, PMD, "%s: Failed to init tx.\n",
 772                         mpipe_name(priv));
 773                 rte_free(priv);
 774                 return rc;
 775         }
 776
 777         priv->initialized = 1;
 778
 779         return 0;
 780 }
 781
 782 static int
 783 mpipe_start(struct rte_eth_dev *dev)
 784 {
 785         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 786         struct mpipe_channel_config config;
 787         struct mpipe_rx_queue *rx_queue;
 788         struct rte_eth_link eth_link;
 789         unsigned queue, buffers = 0;
 790         size_t ring_size;
 791         void *ring_mem;
 792         int rc;
 793
 794         memset(&eth_link, 0, sizeof(eth_link));
 795         mpipe_dev_atomic_write_link_status(dev, &eth_link);
 796
 797         rc = mpipe_init(priv);
 798         if (rc < 0)
 799                 return rc;
 800
 801         /* Initialize NotifRings. */
 802         for (queue = 0; queue < priv->nb_rx_queues; queue++) {
 803                 rx_queue = mpipe_rx_queue(priv, queue);
 804                 ring_size = rx_queue->q.nb_desc * sizeof(gxio_mpipe_idesc_t);
 805
 806                 ring_mem = rte_malloc(NULL, ring_size, ring_size);
 807                 if (!ring_mem) {
 808                         RTE_LOG(ERR, PMD, "%s: Failed to alloc rx descs.\n",
 809                                 mpipe_name(priv));
 810                         return -ENOMEM;
 811                 } else {
 812                         RTE_LOG(DEBUG, PMD, "%s: iDMA ring %d memory %p - %p.\n",
 813                                 mpipe_name(priv), queue, ring_mem,
 814                                 RTE_PTR_ADD(ring_mem, ring_size - 1));
 815                 }
 816
 817                 rc = gxio_mpipe_iqueue_init(&rx_queue->iqueue, priv->context,
 818                                             priv->first_ring + queue, ring_mem,
 819                                             ring_size, 0);
 820                 if (rc < 0) {
 821                         RTE_LOG(ERR, PMD, "%s: Failed to init rx queue.\n",
 822                                 mpipe_name(priv));
 823                         return rc;
 824                 }
 825
 826                 rx_queue->rx_ring_mem = ring_mem;
 827                 buffers += rx_queue->q.nb_desc;
 828         }
 829
 830         /* Initialize ingress NotifGroup and buckets. */
 831         rc = gxio_mpipe_init_notif_group_and_buckets(priv->context,
 832                         priv->notif_group, priv->first_ring, priv->nb_rx_queues,
 833                         priv->first_bucket, MPIPE_RX_BUCKETS,
 834                         GXIO_MPIPE_BUCKET_STATIC_FLOW_AFFINITY);
 835         if (rc < 0) {
 836                 RTE_LOG(ERR, PMD, "%s: Failed to init group and buckets.\n",
 837                         mpipe_name(priv));
 838                 return rc;
 839         }
 840
 841         /* Configure the classifier to deliver packets from this port. */
 842         config.enable = 1;
 843         config.first_bucket = priv->first_bucket;
 844         config.num_buckets = MPIPE_RX_BUCKETS;
 845         memset(&config.stacks, 0xff, sizeof(config.stacks));
 846         config.stacks.stacks[priv->rx_size_code] = priv->stack;
 847         config.head_room = priv->rx_offset & RTE_MEMPOOL_ALIGN_MASK;
 848
 849         rc = mpipe_channel_config(priv->instance, priv->channel,
 850                                   &config);
 851         if (rc < 0) {
 852                 RTE_LOG(ERR, PMD, "%s: Failed to setup classifier.\n",
 853                         mpipe_name(priv));
 854                 return rc;
 855         }
 856
 857         /* Fill empty buffers into the buffer stack. */
 858         mpipe_recv_fill_stack(priv, buffers);
 859
 860         /* Bring up the link. */
 861         mpipe_set_link_up(dev);
 862
 863         /* Start xmit/recv on queues. */
 864         for (queue = 0; queue < priv->nb_tx_queues; queue++)
 865                 mpipe_tx_queue(priv, queue)->q.link_status = 1;
 866         for (queue = 0; queue < priv->nb_rx_queues; queue++)
 867                 mpipe_rx_queue(priv, queue)->q.link_status = 1;
 868         priv->running = 1;
 869
 870         return 0;
 871 }
 872
 873 static void
 874 mpipe_stop(struct rte_eth_dev *dev)
 875 {
 876         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 877         struct mpipe_channel_config config;
 878         unsigned queue;
 879         int rc;
 880
 881         for (queue = 0; queue < priv->nb_tx_queues; queue++)
 882                 mpipe_tx_queue(priv, queue)->q.link_status = 0;
 883         for (queue = 0; queue < priv->nb_rx_queues; queue++)
 884                 mpipe_rx_queue(priv, queue)->q.link_status = 0;
 885
 886         /* Make sure the link_status writes land. */
 887         rte_wmb();
 888
 889         /*
 890          * Wait for link_status change to register with straggling datapath
 891          * threads.
 892          */
 893         mpipe_dp_wait(priv);
 894
 895         /* Bring down the link. */
 896         mpipe_set_link_down(dev);
 897
 898         /* Remove classifier rules. */
 899         memset(&config, 0, sizeof(config));
 900         rc = mpipe_channel_config(priv->instance, priv->channel,
 901                                   &config);
 902         if (rc < 0) {
 903                 RTE_LOG(ERR, PMD, "%s: Failed to stop classifier.\n",
 904                         mpipe_name(priv));
 905         }
 906
 907         /* Flush completed xmit packets. */
 908         mpipe_xmit_flush(priv);
 909
 910         /* Flush buffer stacks. */
 911         mpipe_recv_flush(priv);
 912
 913         priv->running = 0;
 914 }
 915
 916 static void
 917 mpipe_close(struct rte_eth_dev *dev)
 918 {
 919         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 920         if (priv->running)
 921                 mpipe_stop(dev);
 922 }
 923
 924 static void
 925 mpipe_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 926 {
 927         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 928         struct mpipe_tx_queue *tx_queue;
 929         struct mpipe_rx_queue *rx_queue;
 930         unsigned i;
 931         uint16_t idx;
 932
 933         memset(stats, 0, sizeof(*stats));
 934
 935         for (i = 0; i < priv->nb_tx_queues; i++) {
 936                 tx_queue = mpipe_tx_queue(priv, i);
 937
 938                 stats->opackets += tx_queue->q.stats.packets;
 939                 stats->obytes   += tx_queue->q.stats.bytes;
 940                 stats->oerrors  += tx_queue->q.stats.errors;
 941
 942                 idx = tx_queue->q.stat_idx;
 943                 if (idx != (uint16_t)-1) {
 944                         stats->q_opackets[idx] += tx_queue->q.stats.packets;
 945                         stats->q_obytes[idx]   += tx_queue->q.stats.bytes;
 946                         stats->q_errors[idx]   += tx_queue->q.stats.errors;
 947                 }
 948         }
 949
 950         for (i = 0; i < priv->nb_rx_queues; i++) {
 951                 rx_queue = mpipe_rx_queue(priv, i);
 952
 953                 stats->ipackets  += rx_queue->q.stats.packets;
 954                 stats->ibytes    += rx_queue->q.stats.bytes;
 955                 stats->ierrors   += rx_queue->q.stats.errors;
 956                 stats->rx_nombuf += rx_queue->q.stats.nomem;
 957
 958                 idx = rx_queue->q.stat_idx;
 959                 if (idx != (uint16_t)-1) {
 960                         stats->q_ipackets[idx] += rx_queue->q.stats.packets;
 961                         stats->q_ibytes[idx]   += rx_queue->q.stats.bytes;
 962                         stats->q_errors[idx]   += rx_queue->q.stats.errors;
 963                 }
 964         }
 965 }
 966
 967 static void
 968 mpipe_stats_reset(struct rte_eth_dev *dev)
 969 {
 970         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 971         struct mpipe_tx_queue *tx_queue;
 972         struct mpipe_rx_queue *rx_queue;
 973         unsigned i;
 974
 975         for (i = 0; i < priv->nb_tx_queues; i++) {
 976                 tx_queue = mpipe_tx_queue(priv, i);
 977                 memset(&tx_queue->q.stats, 0, sizeof(tx_queue->q.stats));
 978         }
 979
 980         for (i = 0; i < priv->nb_rx_queues; i++) {
 981                 rx_queue = mpipe_rx_queue(priv, i);
 982                 memset(&rx_queue->q.stats, 0, sizeof(rx_queue->q.stats));
 983         }
 984 }
 985
 986 static int
 987 mpipe_queue_stats_mapping_set(struct rte_eth_dev *dev, uint16_t queue_id,
 988                               uint8_t stat_idx, uint8_t is_rx)
 989 {
 990         struct mpipe_dev_priv *priv = mpipe_priv(dev);
 991
 992         if (is_rx) {
 993                 priv->rx_stat_mapping[stat_idx] = queue_id;
 994         } else {
 995                 priv->tx_stat_mapping[stat_idx] = queue_id;
 996         }
 997
 998         return 0;
 999 }
1000
1001 static int
1002 mpipe_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
1003                      uint16_t nb_desc, unsigned int socket_id __rte_unused,
1004                      const struct rte_eth_txconf *tx_conf __rte_unused)
1005 {
1006         struct mpipe_tx_queue *tx_queue = dev->data->tx_queues[queue_idx];
1007         struct mpipe_dev_priv *priv = mpipe_priv(dev);
1008         uint16_t idx;
1009
1010         tx_queue = rte_realloc(tx_queue, sizeof(*tx_queue),
1011                                RTE_CACHE_LINE_SIZE);
1012         if (!tx_queue) {
1013                 RTE_LOG(ERR, PMD, "%s: Failed to allocate TX queue.\n",
1014                         mpipe_name(priv));
1015                 return -ENOMEM;
1016         }
1017
1018         memset(&tx_queue->q, 0, sizeof(tx_queue->q));
1019         tx_queue->q.priv = priv;
1020         tx_queue->q.queue_idx = queue_idx;
1021         tx_queue->q.port_id = dev->data->port_id;
1022         tx_queue->q.nb_desc = nb_desc;
1023
1024         tx_queue->q.stat_idx = -1;
1025         for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) {
1026                 if (priv->tx_stat_mapping[idx] == queue_idx)
1027                         tx_queue->q.stat_idx = idx;
1028         }
1029
1030         dev->data->tx_queues[queue_idx] = tx_queue;
1031
1032         return 0;
1033 }
1034
1035 static void
1036 mpipe_tx_queue_release(void *_txq)
1037 {
1038         rte_free(_txq);
1039 }
1040
1041 static int
1042 mpipe_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
1043                      uint16_t nb_desc, unsigned int socket_id __rte_unused,
1044                      const struct rte_eth_rxconf *rx_conf __rte_unused,
1045                      struct rte_mempool *mp)
1046 {
1047         struct mpipe_rx_queue *rx_queue = dev->data->rx_queues[queue_idx];
1048         struct mpipe_dev_priv *priv = mpipe_priv(dev);
1049         uint16_t idx;
1050         int size, rc;
1051
1052         rc = mpipe_iqueue_size(nb_desc);
1053         if (rc < 0) {
1054                 RTE_LOG(ERR, PMD, "%s: Cannot allocate %d iqueue descs.\n",
1055                         mpipe_name(priv), (int)nb_desc);
1056                 return -ENOMEM;
1057         }
1058
1059         if (rc != nb_desc) {
1060                 RTE_LOG(WARNING, PMD, "%s: Extending RX descs from %d to %d.\n",
1061                         mpipe_name(priv), (int)nb_desc, rc);
1062                 nb_desc = rc;
1063         }
1064
1065         size = sizeof(*rx_queue);
1066         rx_queue = rte_realloc(rx_queue, size, RTE_CACHE_LINE_SIZE);
1067         if (!rx_queue) {
1068                 RTE_LOG(ERR, PMD, "%s: Failed to allocate RX queue.\n",
1069                         mpipe_name(priv));
1070                 return -ENOMEM;
1071         }
1072
1073         memset(&rx_queue->q, 0, sizeof(rx_queue->q));
1074         rx_queue->q.priv = priv;
1075         rx_queue->q.nb_desc = nb_desc;
1076         rx_queue->q.port_id = dev->data->port_id;
1077         rx_queue->q.queue_idx = queue_idx;
1078
1079         if (!priv->rx_mpool) {
1080                 int size = (rte_pktmbuf_data_room_size(mp) -
1081                             RTE_PKTMBUF_HEADROOM -
1082                             MPIPE_RX_IP_ALIGN);
1083
1084                 priv->rx_offset = (sizeof(struct rte_mbuf) +
1085                                    rte_pktmbuf_priv_size(mp) +
1086                                    RTE_PKTMBUF_HEADROOM +
1087                                    MPIPE_RX_IP_ALIGN);
1088                 if (size < 0) {
1089                         RTE_LOG(ERR, PMD, "%s: Bad buffer size %d.\n",
1090                                 mpipe_name(priv),
1091                                 rte_pktmbuf_data_room_size(mp));
1092                         return -ENOMEM;
1093                 }
1094
1095                 priv->rx_size_code = mpipe_buffer_size_index(size);
1096                 priv->rx_mpool = mp;
1097         }
1098
1099         if (priv->rx_mpool != mp) {
1100                 RTE_LOG(WARNING, PMD, "%s: Ignoring multiple buffer pools.\n",
1101                         mpipe_name(priv));
1102         }
1103
1104         rx_queue->q.stat_idx = -1;
1105         for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) {
1106                 if (priv->rx_stat_mapping[idx] == queue_idx)
1107                         rx_queue->q.stat_idx = idx;
1108         }
1109
1110         dev->data->rx_queues[queue_idx] = rx_queue;
1111
1112         return 0;
1113 }
1114
1115 static void
1116 mpipe_rx_queue_release(void *_rxq)
1117 {
1118         rte_free(_rxq);
1119 }
1120
1121 #define MPIPE_XGBE_ENA_HASH_MULTI       \
1122         (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_MULTI_SHIFT)
1123 #define MPIPE_XGBE_ENA_HASH_UNI         \
1124         (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_UNI_SHIFT)
1125 #define MPIPE_XGBE_COPY_ALL             \
1126         (1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__COPY_ALL_SHIFT)
1127 #define MPIPE_GBE_ENA_MULTI_HASH        \
1128         (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__MULTI_HASH_ENA_SHIFT)
1129 #define MPIPE_GBE_ENA_UNI_HASH          \
1130         (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__UNI_HASH_ENA_SHIFT)
1131 #define MPIPE_GBE_COPY_ALL              \
1132         (1UL << MPIPE_GBE_NETWORK_CONFIGURATION__COPY_ALL_SHIFT)
1133
1134 static void
1135 mpipe_promiscuous_enable(struct rte_eth_dev *dev)
1136 {
1137         struct mpipe_dev_priv *priv = mpipe_priv(dev);
1138         int64_t reg;
1139         int addr;
1140
1141         if (priv->is_xaui) {
1142                 addr = MPIPE_XAUI_RECEIVE_CONFIGURATION;
1143                 reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
1144                 reg &= ~MPIPE_XGBE_ENA_HASH_MULTI;
1145                 reg &= ~MPIPE_XGBE_ENA_HASH_UNI;
1146                 reg |=  MPIPE_XGBE_COPY_ALL;
1147                 gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1148         } else {
1149                 addr = MPIPE_GBE_NETWORK_CONFIGURATION;
1150                 reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
1151                 reg &= ~MPIPE_GBE_ENA_MULTI_HASH;
1152                 reg &= ~MPIPE_GBE_ENA_UNI_HASH;
1153                 reg |=  MPIPE_GBE_COPY_ALL;
1154                 gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1155         }
1156 }
1157
1158 static void
1159 mpipe_promiscuous_disable(struct rte_eth_dev *dev)
1160 {
1161         struct mpipe_dev_priv *priv = mpipe_priv(dev);
1162         int64_t reg;
1163         int addr;
1164
1165         if (priv->is_xaui) {
1166                 addr = MPIPE_XAUI_RECEIVE_CONFIGURATION;
1167                 reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
1168                 reg |=  MPIPE_XGBE_ENA_HASH_MULTI;
1169                 reg |=  MPIPE_XGBE_ENA_HASH_UNI;
1170                 reg &= ~MPIPE_XGBE_COPY_ALL;
1171                 gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1172         } else {
1173                 addr = MPIPE_GBE_NETWORK_CONFIGURATION;
1174                 reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
1175                 reg |=  MPIPE_GBE_ENA_MULTI_HASH;
1176                 reg |=  MPIPE_GBE_ENA_UNI_HASH;
1177                 reg &= ~MPIPE_GBE_COPY_ALL;
1178                 gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1179         }
1180 }
1181
1182 static struct eth_dev_ops mpipe_dev_ops = {
1183         .dev_infos_get           = mpipe_infos_get,
1184         .dev_configure           = mpipe_configure,
1185         .dev_start               = mpipe_start,
1186         .dev_stop                = mpipe_stop,
1187         .dev_close               = mpipe_close,
1188         .stats_get               = mpipe_stats_get,
1189         .stats_reset             = mpipe_stats_reset,
1190         .queue_stats_mapping_set = mpipe_queue_stats_mapping_set,
1191         .tx_queue_setup          = mpipe_tx_queue_setup,
1192         .rx_queue_setup          = mpipe_rx_queue_setup,
1193         .tx_queue_release        = mpipe_tx_queue_release,
1194         .rx_queue_release        = mpipe_rx_queue_release,
1195         .link_update             = mpipe_link_update,
1196         .dev_set_link_up         = mpipe_set_link_up,
1197         .dev_set_link_down       = mpipe_set_link_down,
1198         .promiscuous_enable      = mpipe_promiscuous_enable,
1199         .promiscuous_disable     = mpipe_promiscuous_disable,
1200 };
1201
1202 static inline void
1203 mpipe_xmit_null(struct mpipe_dev_priv *priv, int64_t start, int64_t end)
1204 {
1205         gxio_mpipe_edesc_t null_desc = { { .bound = 1, .ns = 1 } };
1206         gxio_mpipe_equeue_t *equeue = &priv->equeue;
1207         int64_t slot;
1208
1209         for (slot = start; slot < end; slot++) {
1210                 gxio_mpipe_equeue_put_at(equeue, null_desc, slot);
1211         }
1212 }
1213
1214 static void
1215 mpipe_xmit_flush(struct mpipe_dev_priv *priv)
1216 {
1217         gxio_mpipe_equeue_t *equeue = &priv->equeue;
1218         int64_t slot;
1219
1220         /* Post a dummy descriptor and wait for its return. */
1221         slot = gxio_mpipe_equeue_reserve(equeue, 1);
1222         if (slot < 0) {
1223                 RTE_LOG(ERR, PMD, "%s: Failed to reserve stop slot.\n",
1224                         mpipe_name(priv));
1225                 return;
1226         }
1227
1228         mpipe_xmit_null(priv, slot, slot + 1);
1229
1230         while (!gxio_mpipe_equeue_is_complete(equeue, slot, 1)) {
1231                 rte_pause();
1232         }
1233
1234         for (slot = 0; slot < priv->equeue_size; slot++) {
1235                 if (priv->tx_comps[slot])
1236                         rte_pktmbuf_free_seg(priv->tx_comps[slot]);
1237         }
1238 }
1239
1240 static void
1241 mpipe_recv_flush(struct mpipe_dev_priv *priv)
1242 {
1243         uint8_t in_port = priv->port_id;
1244         struct mpipe_rx_queue *rx_queue;
1245         gxio_mpipe_iqueue_t *iqueue;
1246         gxio_mpipe_idesc_t idesc;
1247         struct rte_mbuf *mbuf;
1248         int retries = 0;
1249         unsigned queue;
1250
1251         do {
1252                 mpipe_recv_flush_stack(priv);
1253
1254                 /* Flush packets sitting in recv queues. */
1255                 for (queue = 0; queue < priv->nb_rx_queues; queue++) {
1256                         rx_queue = mpipe_rx_queue(priv, queue);
1257                         iqueue = &rx_queue->iqueue;
1258                         while (gxio_mpipe_iqueue_try_get(iqueue, &idesc) >= 0) {
1259                                 mbuf = mpipe_recv_mbuf(priv, &idesc, in_port);
1260                                 rte_pktmbuf_free(mbuf);
1261                                 priv->rx_buffers--;
1262                         }
1263                         rte_free(rx_queue->rx_ring_mem);
1264                 }
1265         } while (retries++ < 10 && priv->rx_buffers);
1266
1267         if (priv->rx_buffers) {
1268                 RTE_LOG(ERR, PMD, "%s: Leaked %d receive buffers.\n",
1269                         mpipe_name(priv), priv->rx_buffers);
1270         } else {
1271                 PMD_DEBUG_RX("%s: Returned all receive buffers.\n",
1272                              mpipe_name(priv));
1273         }
1274 }
1275
1276 static inline uint16_t
1277 mpipe_do_xmit(struct mpipe_tx_queue *tx_queue, struct rte_mbuf **tx_pkts,
1278               uint16_t nb_pkts)
1279 {
1280         struct mpipe_dev_priv *priv = tx_queue->q.priv;
1281         gxio_mpipe_equeue_t *equeue = &priv->equeue;
1282         unsigned nb_bytes = 0;
1283         unsigned nb_sent = 0;
1284         int nb_slots, i;
1285         uint8_t port_id;
1286
1287         PMD_DEBUG_TX("Trying to transmit %d packets on %s:%d.\n",
1288                      nb_pkts, mpipe_name(tx_queue->q.priv),
1289                      tx_queue->q.queue_idx);
1290
1291         /* Optimistic assumption that we need exactly one slot per packet. */
1292         nb_slots = RTE_MIN(nb_pkts, MPIPE_TX_DESCS / 2);
1293
1294         do {
1295                 struct rte_mbuf *mbuf = NULL, *pkt = NULL;
1296                 int64_t slot;
1297
1298                 /* Reserve eDMA ring slots. */
1299                 slot = gxio_mpipe_equeue_try_reserve_fast(equeue, nb_slots);
1300                 if (unlikely(slot < 0)) {
1301                         break;
1302                 }
1303
1304                 for (i = 0; i < nb_slots; i++) {
1305                         unsigned idx = (slot + i) & (priv->equeue_size - 1);
1306                         rte_prefetch0(priv->tx_comps[idx]);
1307                 }
1308
1309                 /* Fill up slots with descriptor and completion info. */
1310                 for (i = 0; i < nb_slots; i++) {
1311                         unsigned idx = (slot + i) & (priv->equeue_size - 1);
1312                         gxio_mpipe_edesc_t desc;
1313                         struct rte_mbuf *next;
1314
1315                         /* Starting on a new packet? */
1316                         if (likely(!mbuf)) {
1317                                 int room = nb_slots - i;
1318
1319                                 pkt = mbuf = tx_pkts[nb_sent];
1320
1321                                 /* Bail out if we run out of descs. */
1322                                 if (unlikely(pkt->nb_segs > room))
1323                                         break;
1324
1325                                 nb_sent++;
1326                         }
1327
1328                         /* We have a segment to send. */
1329                         next = mbuf->next;
1330
1331                         if (priv->tx_comps[idx])
1332                                 rte_pktmbuf_free_seg(priv->tx_comps[idx]);
1333
1334                         port_id = (mbuf->port < RTE_MAX_ETHPORTS) ?
1335                                                 mbuf->port : priv->port_id;
1336                         desc = (gxio_mpipe_edesc_t) { {
1337                                 .va        = rte_pktmbuf_mtod(mbuf, uintptr_t),
1338                                 .xfer_size = rte_pktmbuf_data_len(mbuf),
1339                                 .bound     = next ? 0 : 1,
1340                                 .stack_idx = mpipe_mbuf_stack_index(priv, mbuf),
1341                         } };
1342                         if (mpipe_local.mbuf_push_debt[port_id] > 0) {
1343                                 mpipe_local.mbuf_push_debt[port_id]--;
1344                                 desc.hwb = 1;
1345                                 priv->tx_comps[idx] = NULL;
1346                         } else
1347                                 priv->tx_comps[idx] = mbuf;
1348
1349                         nb_bytes += mbuf->data_len;
1350                         gxio_mpipe_equeue_put_at(equeue, desc, slot + i);
1351
1352                         PMD_DEBUG_TX("%s:%d: Sending packet %p, len %d\n",
1353                                      mpipe_name(priv),
1354                                      tx_queue->q.queue_idx,
1355                                      rte_pktmbuf_mtod(mbuf, void *),
1356                                      rte_pktmbuf_data_len(mbuf));
1357
1358                         mbuf = next;
1359                 }
1360
1361                 if (unlikely(nb_sent < nb_pkts)) {
1362
1363                         /* Fill remaining slots with null descriptors. */
1364                         mpipe_xmit_null(priv, slot + i, slot + nb_slots);
1365
1366                         /*
1367                          * Calculate exact number of descriptors needed for
1368                          * the next go around.
1369                          */
1370                         nb_slots = 0;
1371                         for (i = nb_sent; i < nb_pkts; i++) {
1372                                 nb_slots += tx_pkts[i]->nb_segs;
1373                         }
1374
1375                         nb_slots = RTE_MIN(nb_slots, MPIPE_TX_DESCS / 2);
1376                 }
1377         } while (nb_sent < nb_pkts);
1378
1379         tx_queue->q.stats.packets += nb_sent;
1380         tx_queue->q.stats.bytes   += nb_bytes;
1381
1382         return nb_sent;
1383 }
1384
1385 static inline uint16_t
1386 mpipe_do_recv(struct mpipe_rx_queue *rx_queue, struct rte_mbuf **rx_pkts,
1387               uint16_t nb_pkts)
1388 {
1389         struct mpipe_dev_priv *priv = rx_queue->q.priv;
1390         gxio_mpipe_iqueue_t *iqueue = &rx_queue->iqueue;
1391         gxio_mpipe_idesc_t *first_idesc, *idesc, *last_idesc;
1392         uint8_t in_port = rx_queue->q.port_id;
1393         const unsigned look_ahead = 8;
1394         int room = nb_pkts, rc = 0;
1395         unsigned nb_packets = 0;
1396         unsigned nb_dropped = 0;
1397         unsigned nb_nomem = 0;
1398         unsigned nb_bytes = 0;
1399         unsigned nb_descs, i;
1400
1401         while (room && !rc) {
1402                 if (rx_queue->avail_descs < room) {
1403                         rc = gxio_mpipe_iqueue_try_peek(iqueue,
1404                                                         &rx_queue->next_desc);
1405                         rx_queue->avail_descs = rc < 0 ? 0 : rc;
1406                 }
1407
1408                 if (unlikely(!rx_queue->avail_descs)) {
1409                         break;
1410                 }
1411
1412                 nb_descs = RTE_MIN(room, rx_queue->avail_descs);
1413
1414                 first_idesc = rx_queue->next_desc;
1415                 last_idesc  = first_idesc + nb_descs;
1416
1417                 rx_queue->next_desc   += nb_descs;
1418                 rx_queue->avail_descs -= nb_descs;
1419
1420                 for (i = 1; i < look_ahead; i++) {
1421                         rte_prefetch0(first_idesc + i);
1422                 }
1423
1424                 PMD_DEBUG_RX("%s:%d: Trying to receive %d packets\n",
1425                              mpipe_name(rx_queue->q.priv),
1426                              rx_queue->q.queue_idx,
1427                              nb_descs);
1428
1429                 for (idesc = first_idesc; idesc < last_idesc; idesc++) {
1430                         struct rte_mbuf *mbuf;
1431
1432                         PMD_DEBUG_RX("%s:%d: processing idesc %d/%d\n",
1433                                      mpipe_name(priv),
1434                                      rx_queue->q.queue_idx,
1435                                      nb_packets, nb_descs);
1436
1437                         rte_prefetch0(idesc + look_ahead);
1438
1439                         PMD_DEBUG_RX("%s:%d: idesc %p, %s%s%s%s%s%s%s%s%s%s"
1440                                      "size: %d, bkt: %d, chan: %d, ring: %d, sqn: %lu, va: %lu\n",
1441                                      mpipe_name(priv),
1442                                      rx_queue->q.queue_idx,
1443                                      idesc,
1444                                      idesc->me ? "me, " : "",
1445                                      idesc->tr ? "tr, " : "",
1446                                      idesc->ce ? "ce, " : "",
1447                                      idesc->ct ? "ct, " : "",
1448                                      idesc->cs ? "cs, " : "",
1449                                      idesc->nr ? "nr, " : "",
1450                                      idesc->sq ? "sq, " : "",
1451                                      idesc->ts ? "ts, " : "",
1452                                      idesc->ps ? "ps, " : "",
1453                                      idesc->be ? "be, " : "",
1454                                      idesc->l2_size,
1455                                      idesc->bucket_id,
1456                                      idesc->channel,
1457                                      idesc->notif_ring,
1458                                      (unsigned long)idesc->packet_sqn,
1459                                      (unsigned long)idesc->va);
1460
1461                         if (unlikely(gxio_mpipe_idesc_has_error(idesc))) {
1462                                 nb_dropped++;
1463                                 gxio_mpipe_iqueue_drop(iqueue, idesc);
1464                                 PMD_DEBUG_RX("%s:%d: Descriptor error\n",
1465                                              mpipe_name(rx_queue->q.priv),
1466                                              rx_queue->q.queue_idx);
1467                                 continue;
1468                         }
1469
1470                         if (mpipe_local.mbuf_push_debt[in_port] <
1471                                         MPIPE_BUF_DEBT_THRESHOLD)
1472                                 mpipe_local.mbuf_push_debt[in_port]++;
1473                         else {
1474                                 mbuf = __rte_mbuf_raw_alloc(priv->rx_mpool);
1475                                 if (unlikely(!mbuf)) {
1476                                         nb_nomem++;
1477                                         gxio_mpipe_iqueue_drop(iqueue, idesc);
1478                                         PMD_DEBUG_RX("%s:%d: alloc failure\n",
1479                                              mpipe_name(rx_queue->q.priv),
1480                                              rx_queue->q.queue_idx);
1481                                         continue;
1482                                 }
1483
1484                                 mpipe_recv_push(priv, mbuf);
1485                         }
1486
1487                         /* Get and setup the mbuf for the received packet. */
1488                         mbuf = mpipe_recv_mbuf(priv, idesc, in_port);
1489
1490                         /* Update results and statistics counters. */
1491                         rx_pkts[nb_packets] = mbuf;
1492                         nb_bytes += mbuf->pkt_len;
1493                         nb_packets++;
1494                 }
1495
1496                 /*
1497                  * We release the ring in bursts, but do not track and release
1498                  * buckets.  This therefore breaks dynamic flow affinity, but
1499                  * we always operate in static affinity mode, and so we're OK
1500                  * with this optimization.
1501                  */
1502                 gxio_mpipe_iqueue_advance(iqueue, nb_descs);
1503                 gxio_mpipe_credit(iqueue->context, iqueue->ring, -1, nb_descs);
1504
1505                 /*
1506                  * Go around once more if we haven't yet peeked the queue, and
1507                  * if we have more room to receive.
1508                  */
1509                 room = nb_pkts - nb_packets;
1510         }
1511
1512         rx_queue->q.stats.packets += nb_packets;
1513         rx_queue->q.stats.bytes   += nb_bytes;
1514         rx_queue->q.stats.errors  += nb_dropped;
1515         rx_queue->q.stats.nomem   += nb_nomem;
1516
1517         PMD_DEBUG_RX("%s:%d: RX: %d/%d pkts/bytes, %d/%d drops/nomem\n",
1518                      mpipe_name(rx_queue->q.priv), rx_queue->q.queue_idx,
1519                      nb_packets, nb_bytes, nb_dropped, nb_nomem);
1520
1521         return nb_packets;
1522 }
1523
1524 static uint16_t
1525 mpipe_recv_pkts(void *_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1526 {
1527         struct mpipe_rx_queue *rx_queue = _rxq;
1528         uint16_t result = 0;
1529
1530         if (rx_queue) {
1531                 mpipe_dp_enter(rx_queue->q.priv);
1532                 if (likely(rx_queue->q.link_status))
1533                         result = mpipe_do_recv(rx_queue, rx_pkts, nb_pkts);
1534                 mpipe_dp_exit(rx_queue->q.priv);
1535         }
1536
1537         return result;
1538 }
1539
1540 static uint16_t
1541 mpipe_xmit_pkts(void *_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1542 {
1543         struct mpipe_tx_queue *tx_queue = _txq;
1544         uint16_t result = 0;
1545
1546         if (tx_queue) {
1547                 mpipe_dp_enter(tx_queue->q.priv);
1548                 if (likely(tx_queue->q.link_status))
1549                         result = mpipe_do_xmit(tx_queue, tx_pkts, nb_pkts);
1550                 mpipe_dp_exit(tx_queue->q.priv);
1551         }
1552
1553         return result;
1554 }
1555
1556 static int
1557 mpipe_link_mac(const char *ifname, uint8_t *mac)
1558 {
1559         int rc, idx;
1560         char name[GXIO_MPIPE_LINK_NAME_LEN];
1561
1562         for (idx = 0, rc = 0; !rc; idx++) {
1563                 rc = gxio_mpipe_link_enumerate_mac(idx, name, mac);
1564                 if (!rc && !strncmp(name, ifname, GXIO_MPIPE_LINK_NAME_LEN))
1565                         return 0;
1566         }
1567         return -ENODEV;
1568 }
1569
1570 static int
1571 rte_pmd_mpipe_devinit(const char *ifname,
1572                       const char *params __rte_unused)
1573 {
1574         gxio_mpipe_context_t *context;
1575         struct rte_eth_dev *eth_dev;
1576         struct mpipe_dev_priv *priv;
1577         int instance, rc;
1578         uint8_t *mac;
1579
1580         /* Get the mPIPE instance that the device belongs to. */
1581         instance = gxio_mpipe_link_instance(ifname);
1582         context = mpipe_context(instance);
1583         if (!context) {
1584                 RTE_LOG(ERR, PMD, "%s: No device for link.\n", ifname);
1585                 return -ENODEV;
1586         }
1587
1588         priv = rte_zmalloc(NULL, sizeof(*priv), 0);
1589         if (!priv) {
1590                 RTE_LOG(ERR, PMD, "%s: Failed to allocate priv.\n", ifname);
1591                 return -ENOMEM;
1592         }
1593
1594         memset(&priv->tx_stat_mapping, 0xff, sizeof(priv->tx_stat_mapping));
1595         memset(&priv->rx_stat_mapping, 0xff, sizeof(priv->rx_stat_mapping));
1596         priv->context = context;
1597         priv->instance = instance;
1598         priv->is_xaui = (strncmp(ifname, "xgbe", 4) == 0);
1599         priv->channel = -1;
1600
1601         mac = priv->mac_addr.addr_bytes;
1602         rc = mpipe_link_mac(ifname, mac);
1603         if (rc < 0) {
1604                 RTE_LOG(ERR, PMD, "%s: Failed to enumerate link.\n", ifname);
1605                 rte_free(priv);
1606                 return -ENODEV;
1607         }
1608
1609         eth_dev = rte_eth_dev_allocate(ifname, RTE_ETH_DEV_VIRTUAL);
1610         if (!eth_dev) {
1611                 RTE_LOG(ERR, PMD, "%s: Failed to allocate device.\n", ifname);
1612                 rte_free(priv);
1613                 return -ENOMEM;
1614         }
1615
1616         RTE_LOG(INFO, PMD, "%s: Initialized mpipe device"
1617                 "(mac %02x:%02x:%02x:%02x:%02x:%02x).\n",
1618                 ifname, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
1619
1620         priv->eth_dev = eth_dev;
1621         priv->port_id = eth_dev->data->port_id;
1622         eth_dev->data->dev_private = priv;
1623         eth_dev->data->mac_addrs = &priv->mac_addr;
1624
1625         eth_dev->data->dev_flags = 0;
1626         eth_dev->data->kdrv = RTE_KDRV_NONE;
1627         eth_dev->driver = NULL;
1628         eth_dev->data->drv_name = drivername;
1629         eth_dev->data->numa_node = instance;
1630
1631         eth_dev->dev_ops      = &mpipe_dev_ops;
1632         eth_dev->rx_pkt_burst = &mpipe_recv_pkts;
1633         eth_dev->tx_pkt_burst = &mpipe_xmit_pkts;
1634
1635         return 0;
1636 }
1637
1638 static struct rte_driver pmd_mpipe_xgbe_drv = {
1639         .name = "xgbe",
1640         .type = PMD_VDEV,
1641         .init = rte_pmd_mpipe_devinit,
1642 };
1643
1644 static struct rte_driver pmd_mpipe_gbe_drv = {
1645         .name = "gbe",
1646         .type = PMD_VDEV,
1647         .init = rte_pmd_mpipe_devinit,
1648 };
1649
1650 PMD_REGISTER_DRIVER(pmd_mpipe_xgbe_drv);
1651 PMD_REGISTER_DRIVER(pmd_mpipe_gbe_drv);
1652
1653 static void __attribute__((constructor, used))
1654 mpipe_init_contexts(void)
1655 {
1656         struct mpipe_context *context;
1657         int rc, instance;
1658
1659         for (instance = 0; instance < GXIO_MPIPE_INSTANCE_MAX; instance++) {
1660                 context = &mpipe_contexts[instance];
1661
1662                 rte_spinlock_init(&context->lock);
1663                 rc = gxio_mpipe_init(&context->context, instance);
1664                 if (rc < 0)
1665                         break;
1666         }
1667
1668         mpipe_instances = instance;
1669 }