lib/librte_pmd_fm10k/fm10k_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <inttypes.h>
  35
  36 #include <rte_ethdev.h>
  37 #include <rte_common.h>
  38 #include "fm10k.h"
  39 #include "base/fm10k_type.h"
  40
  41 #ifdef RTE_PMD_PACKET_PREFETCH
  42 #define rte_packet_prefetch(p)  rte_prefetch1(p)
  43 #else
  44 #define rte_packet_prefetch(p)  do {} while (0)
  45 #endif
  46
  47 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
  48 static inline void dump_rxd(union fm10k_rx_desc *rxd)
  49 {
  50         PMD_RX_LOG(DEBUG, "+----------------|----------------+");
  51         PMD_RX_LOG(DEBUG, "|     GLORT      | PKT HDR & TYPE |");
  52         PMD_RX_LOG(DEBUG, "|   0x%08x   |   0x%08x   |", rxd->d.glort,
  53                         rxd->d.data);
  54         PMD_RX_LOG(DEBUG, "+----------------|----------------+");
  55         PMD_RX_LOG(DEBUG, "|   VLAN & LEN   |     STATUS     |");
  56         PMD_RX_LOG(DEBUG, "|   0x%08x   |   0x%08x   |", rxd->d.vlan_len,
  57                         rxd->d.staterr);
  58         PMD_RX_LOG(DEBUG, "+----------------|----------------+");
  59         PMD_RX_LOG(DEBUG, "|    RESERVED    |    RSS_HASH    |");
  60         PMD_RX_LOG(DEBUG, "|   0x%08x   |   0x%08x   |", 0, rxd->d.rss);
  61         PMD_RX_LOG(DEBUG, "+----------------|----------------+");
  62         PMD_RX_LOG(DEBUG, "|            TIME TAG             |");
  63         PMD_RX_LOG(DEBUG, "|       0x%016"PRIx64"        |", rxd->q.timestamp);
  64         PMD_RX_LOG(DEBUG, "+----------------|----------------+");
  65 }
  66 #endif
  67
  68 static inline void
  69 rx_desc_to_ol_flags(struct rte_mbuf *m, const union fm10k_rx_desc *d)
  70 {
  71         uint16_t ptype;
  72         static const uint16_t pt_lut[] = { 0,
  73                 PKT_RX_IPV4_HDR, PKT_RX_IPV4_HDR_EXT,
  74                 PKT_RX_IPV6_HDR, PKT_RX_IPV6_HDR_EXT,
  75                 0, 0, 0
  76         };
  77
  78         if (d->w.pkt_info & FM10K_RXD_RSSTYPE_MASK)
  79                 m->ol_flags |= PKT_RX_RSS_HASH;
  80
  81         if (unlikely((d->d.staterr &
  82                 (FM10K_RXD_STATUS_IPCS | FM10K_RXD_STATUS_IPE)) ==
  83                 (FM10K_RXD_STATUS_IPCS | FM10K_RXD_STATUS_IPE)))
  84                 m->ol_flags |= PKT_RX_IP_CKSUM_BAD;
  85
  86         if (unlikely((d->d.staterr &
  87                 (FM10K_RXD_STATUS_L4CS | FM10K_RXD_STATUS_L4E)) ==
  88                 (FM10K_RXD_STATUS_L4CS | FM10K_RXD_STATUS_L4E)))
  89                 m->ol_flags |= PKT_RX_L4_CKSUM_BAD;
  90
  91         if (d->d.staterr & FM10K_RXD_STATUS_VEXT)
  92                 m->ol_flags |= PKT_RX_VLAN_PKT;
  93
  94         if (unlikely(d->d.staterr & FM10K_RXD_STATUS_HBO))
  95                 m->ol_flags |= PKT_RX_HBUF_OVERFLOW;
  96
  97         if (unlikely(d->d.staterr & FM10K_RXD_STATUS_RXE))
  98                 m->ol_flags |= PKT_RX_RECIP_ERR;
  99
 100         ptype = (d->d.data & FM10K_RXD_PKTTYPE_MASK_L3) >>
 101                                                 FM10K_RXD_PKTTYPE_SHIFT;
 102         m->ol_flags |= pt_lut[(uint8_t)ptype];
 103 }
 104
 105 uint16_t
 106 fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 107         uint16_t nb_pkts)
 108 {
 109         struct rte_mbuf *mbuf;
 110         union fm10k_rx_desc desc;
 111         struct fm10k_rx_queue *q = rx_queue;
 112         uint16_t count = 0;
 113         int alloc = 0;
 114         uint16_t next_dd;
 115         int ret;
 116
 117         next_dd = q->next_dd;
 118
 119         nb_pkts = RTE_MIN(nb_pkts, q->alloc_thresh);
 120         for (count = 0; count < nb_pkts; ++count) {
 121                 mbuf = q->sw_ring[next_dd];
 122                 desc = q->hw_ring[next_dd];
 123                 if (!(desc.d.staterr & FM10K_RXD_STATUS_DD))
 124                         break;
 125 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
 126                 dump_rxd(&desc);
 127 #endif
 128                 rte_pktmbuf_pkt_len(mbuf) = desc.w.length;
 129                 rte_pktmbuf_data_len(mbuf) = desc.w.length;
 130
 131                 mbuf->ol_flags = 0;
 132 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
 133                 rx_desc_to_ol_flags(mbuf, &desc);
 134 #endif
 135
 136                 mbuf->hash.rss = desc.d.rss;
 137
 138                 rx_pkts[count] = mbuf;
 139                 if (++next_dd == q->nb_desc) {
 140                         next_dd = 0;
 141                         alloc = 1;
 142                 }
 143
 144                 /* Prefetch next mbuf while processing current one. */
 145                 rte_prefetch0(q->sw_ring[next_dd]);
 146
 147                 /*
 148                  * When next RX descriptor is on a cache-line boundary,
 149                  * prefetch the next 4 RX descriptors and the next 8 pointers
 150                  * to mbufs.
 151                  */
 152                 if ((next_dd & 0x3) == 0) {
 153                         rte_prefetch0(&q->hw_ring[next_dd]);
 154                         rte_prefetch0(&q->sw_ring[next_dd]);
 155                 }
 156         }
 157
 158         q->next_dd = next_dd;
 159
 160         if ((q->next_dd > q->next_trigger) || (alloc == 1)) {
 161                 ret = rte_mempool_get_bulk(q->mp,
 162                                         (void **)&q->sw_ring[q->next_alloc],
 163                                         q->alloc_thresh);
 164
 165                 if (unlikely(ret != 0)) {
 166                         uint8_t port = q->port_id;
 167                         PMD_RX_LOG(ERR, "Failed to alloc mbuf");
 168                         /*
 169                          * Need to restore next_dd if we cannot allocate new
 170                          * buffers to replenish the old ones.
 171                          */
 172                         q->next_dd = (q->next_dd + q->nb_desc - count) %
 173                                                                 q->nb_desc;
 174                         rte_eth_devices[port].data->rx_mbuf_alloc_failed++;
 175                         return 0;
 176                 }
 177
 178                 for (; q->next_alloc <= q->next_trigger; ++q->next_alloc) {
 179                         mbuf = q->sw_ring[q->next_alloc];
 180
 181                         /* setup static mbuf fields */
 182                         fm10k_pktmbuf_reset(mbuf, q->port_id);
 183
 184                         /* write descriptor */
 185                         desc.q.pkt_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
 186                         desc.q.hdr_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
 187                         q->hw_ring[q->next_alloc] = desc;
 188                 }
 189                 FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_trigger);
 190                 q->next_trigger += q->alloc_thresh;
 191                 if (q->next_trigger >= q->nb_desc) {
 192                         q->next_trigger = q->alloc_thresh - 1;
 193                         q->next_alloc = 0;
 194                 }
 195         }
 196
 197         return count;
 198 }
 199
 200 uint16_t
 201 fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 202                                 uint16_t nb_pkts)
 203 {
 204         struct rte_mbuf *mbuf;
 205         union fm10k_rx_desc desc;
 206         struct fm10k_rx_queue *q = rx_queue;
 207         uint16_t count = 0;
 208         uint16_t nb_rcv, nb_seg;
 209         int alloc = 0;
 210         uint16_t next_dd;
 211         struct rte_mbuf *first_seg = q->pkt_first_seg;
 212         struct rte_mbuf *last_seg = q->pkt_last_seg;
 213         int ret;
 214
 215         next_dd = q->next_dd;
 216         nb_rcv = 0;
 217
 218         nb_seg = RTE_MIN(nb_pkts, q->alloc_thresh);
 219         for (count = 0; count < nb_seg; count++) {
 220                 mbuf = q->sw_ring[next_dd];
 221                 desc = q->hw_ring[next_dd];
 222                 if (!(desc.d.staterr & FM10K_RXD_STATUS_DD))
 223                         break;
 224 #ifdef RTE_LIBRTE_FM10K_DEBUG_RX
 225                 dump_rxd(&desc);
 226 #endif
 227
 228                 if (++next_dd == q->nb_desc) {
 229                         next_dd = 0;
 230                         alloc = 1;
 231                 }
 232
 233                 /* Prefetch next mbuf while processing current one. */
 234                 rte_prefetch0(q->sw_ring[next_dd]);
 235
 236                 /*
 237                  * When next RX descriptor is on a cache-line boundary,
 238                  * prefetch the next 4 RX descriptors and the next 8 pointers
 239                  * to mbufs.
 240                  */
 241                 if ((next_dd & 0x3) == 0) {
 242                         rte_prefetch0(&q->hw_ring[next_dd]);
 243                         rte_prefetch0(&q->sw_ring[next_dd]);
 244                 }
 245
 246                 /* Fill data length */
 247                 rte_pktmbuf_data_len(mbuf) = desc.w.length;
 248
 249                 /*
 250                  * If this is the first buffer of the received packet,
 251                  * set the pointer to the first mbuf of the packet and
 252                  * initialize its context.
 253                  * Otherwise, update the total length and the number of segments
 254                  * of the current scattered packet, and update the pointer to
 255                  * the last mbuf of the current packet.
 256                  */
 257                 if (!first_seg) {
 258                         first_seg = mbuf;
 259                         first_seg->pkt_len = desc.w.length;
 260                 } else {
 261                         first_seg->pkt_len =
 262                                         (uint16_t)(first_seg->pkt_len +
 263                                         rte_pktmbuf_data_len(mbuf));
 264                         first_seg->nb_segs++;
 265                         last_seg->next = mbuf;
 266                 }
 267
 268                 /*
 269                  * If this is not the last buffer of the received packet,
 270                  * update the pointer to the last mbuf of the current scattered
 271                  * packet and continue to parse the RX ring.
 272                  */
 273                 if (!(desc.d.staterr & FM10K_RXD_STATUS_EOP)) {
 274                         last_seg = mbuf;
 275                         continue;
 276                 }
 277
 278                 first_seg->ol_flags = 0;
 279 #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
 280                 rx_desc_to_ol_flags(first_seg, &desc);
 281 #endif
 282                 first_seg->hash.rss = desc.d.rss;
 283
 284                 /* Prefetch data of first segment, if configured to do so. */
 285                 rte_packet_prefetch((char *)first_seg->buf_addr +
 286                         first_seg->data_off);
 287
 288                 /*
 289                  * Store the mbuf address into the next entry of the array
 290                  * of returned packets.
 291                  */
 292                 rx_pkts[nb_rcv++] = first_seg;
 293
 294                 /*
 295                  * Setup receipt context for a new packet.
 296                  */
 297                 first_seg = NULL;
 298         }
 299
 300         q->next_dd = next_dd;
 301
 302         if ((q->next_dd > q->next_trigger) || (alloc == 1)) {
 303                 ret = rte_mempool_get_bulk(q->mp,
 304                                         (void **)&q->sw_ring[q->next_alloc],
 305                                         q->alloc_thresh);
 306
 307                 if (unlikely(ret != 0)) {
 308                         uint8_t port = q->port_id;
 309                         PMD_RX_LOG(ERR, "Failed to alloc mbuf");
 310                         /*
 311                          * Need to restore next_dd if we cannot allocate new
 312                          * buffers to replenish the old ones.
 313                          */
 314                         q->next_dd = (q->next_dd + q->nb_desc - count) %
 315                                                                 q->nb_desc;
 316                         rte_eth_devices[port].data->rx_mbuf_alloc_failed++;
 317                         return 0;
 318                 }
 319
 320                 for (; q->next_alloc <= q->next_trigger; ++q->next_alloc) {
 321                         mbuf = q->sw_ring[q->next_alloc];
 322
 323                         /* setup static mbuf fields */
 324                         fm10k_pktmbuf_reset(mbuf, q->port_id);
 325
 326                         /* write descriptor */
 327                         desc.q.pkt_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
 328                         desc.q.hdr_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
 329                         q->hw_ring[q->next_alloc] = desc;
 330                 }
 331                 FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_trigger);
 332                 q->next_trigger += q->alloc_thresh;
 333                 if (q->next_trigger >= q->nb_desc) {
 334                         q->next_trigger = q->alloc_thresh - 1;
 335                         q->next_alloc = 0;
 336                 }
 337         }
 338
 339         q->pkt_first_seg = first_seg;
 340         q->pkt_last_seg = last_seg;
 341
 342         return nb_rcv;
 343 }
 344
 345 static inline void tx_free_descriptors(struct fm10k_tx_queue *q)
 346 {
 347         uint16_t next_rs, count = 0;
 348
 349         next_rs = fifo_peek(&q->rs_tracker);
 350         if (!(q->hw_ring[next_rs].flags & FM10K_TXD_FLAG_DONE))
 351                 return;
 352
 353         /* the DONE flag is set on this descriptor so remove the ID
 354          * from the RS bit tracker and free the buffers */
 355         fifo_remove(&q->rs_tracker);
 356
 357         /* wrap around? if so, free buffers from last_free up to but NOT
 358          * including nb_desc */
 359         if (q->last_free > next_rs) {
 360                 count = q->nb_desc - q->last_free;
 361                 while (q->last_free < q->nb_desc) {
 362                         rte_pktmbuf_free_seg(q->sw_ring[q->last_free]);
 363                         q->sw_ring[q->last_free] = NULL;
 364                         ++q->last_free;
 365                 }
 366                 q->last_free = 0;
 367         }
 368
 369         /* adjust free descriptor count before the next loop */
 370         q->nb_free += count + (next_rs + 1 - q->last_free);
 371
 372         /* free buffers from last_free, up to and including next_rs */
 373         while (q->last_free <= next_rs) {
 374                 rte_pktmbuf_free_seg(q->sw_ring[q->last_free]);
 375                 q->sw_ring[q->last_free] = NULL;
 376                 ++q->last_free;
 377         }
 378
 379         if (q->last_free == q->nb_desc)
 380                 q->last_free = 0;
 381 }
 382
 383 static inline void tx_xmit_pkt(struct fm10k_tx_queue *q, struct rte_mbuf *mb)
 384 {
 385         uint16_t last_id;
 386         uint8_t flags;
 387
 388         /* always set the LAST flag on the last descriptor used to
 389          * transmit the packet */
 390         flags = FM10K_TXD_FLAG_LAST;
 391         last_id = q->next_free + mb->nb_segs - 1;
 392         if (last_id >= q->nb_desc)
 393                 last_id = last_id - q->nb_desc;
 394
 395         /* but only set the RS flag on the last descriptor if rs_thresh
 396          * descriptors will be used since the RS flag was last set */
 397         if ((q->nb_used + mb->nb_segs) >= q->rs_thresh) {
 398                 flags |= FM10K_TXD_FLAG_RS;
 399                 fifo_insert(&q->rs_tracker, last_id);
 400                 q->nb_used = 0;
 401         } else {
 402                 q->nb_used = q->nb_used + mb->nb_segs;
 403         }
 404
 405         q->hw_ring[last_id].flags = flags;
 406         q->nb_free -= mb->nb_segs;
 407
 408         /* set checksum flags on first descriptor of packet. SCTP checksum
 409          * offload is not supported, but we do not explicitly check for this
 410          * case in favor of greatly simplified processing. */
 411         if (mb->ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK))
 412                 q->hw_ring[q->next_free].flags |= FM10K_TXD_FLAG_CSUM;
 413
 414         /* set vlan if requested */
 415         if (mb->ol_flags & PKT_TX_VLAN_PKT)
 416                 q->hw_ring[q->next_free].vlan = mb->vlan_tci;
 417
 418         /* fill up the rings */
 419         for (; mb != NULL; mb = mb->next) {
 420                 q->sw_ring[q->next_free] = mb;
 421                 q->hw_ring[q->next_free].buffer_addr =
 422                                 rte_cpu_to_le_64(MBUF_DMA_ADDR(mb));
 423                 q->hw_ring[q->next_free].buflen =
 424                                 rte_cpu_to_le_16(rte_pktmbuf_data_len(mb));
 425                 if (++q->next_free == q->nb_desc)
 426                         q->next_free = 0;
 427         }
 428 }
 429
 430 uint16_t
 431 fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 432         uint16_t nb_pkts)
 433 {
 434         struct fm10k_tx_queue *q = tx_queue;
 435         struct rte_mbuf *mb;
 436         uint16_t count;
 437
 438         for (count = 0; count < nb_pkts; ++count) {
 439                 mb = tx_pkts[count];
 440
 441                 /* running low on descriptors? try to free some... */
 442                 if (q->nb_free < q->free_trigger)
 443                         tx_free_descriptors(q);
 444
 445                 /* make sure there are enough free descriptors to transmit the
 446                  * entire packet before doing anything */
 447                 if (q->nb_free < mb->nb_segs)
 448                         break;
 449
 450                 /* sanity check to make sure the mbuf is valid */
 451                 if ((mb->nb_segs == 0) ||
 452                     ((mb->nb_segs > 1) && (mb->next == NULL)))
 453                         break;
 454
 455                 /* process the packet */
 456                 tx_xmit_pkt(q, mb);
 457         }
 458
 459         /* update the tail pointer if any packets were processed */
 460         if (likely(count > 0))
 461                 FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_free);
 462
 463         return count;
 464 }