drivers/net/sfc/sfc_rx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  * Copyright (c) 2016-2017 Solarflare Communications Inc.
   5  * All rights reserved.
   6  *
   7  * This software was jointly developed between OKTET Labs (under contract
   8  * for Solarflare) and Solarflare Communications, Inc.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions are met:
  12  *
  13  * 1. Redistributions of source code must retain the above copyright notice,
  14  *    this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright notice,
  16  *    this list of conditions and the following disclaimer in the documentation
  17  *    and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  21  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  22  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  23  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  24  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  25  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  26  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  27  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  28  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
  29  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30  */
  31
  32 #include <rte_mempool.h>
  33
  34 #include "efx.h"
  35
  36 #include "sfc.h"
  37 #include "sfc_debug.h"
  38 #include "sfc_log.h"
  39 #include "sfc_ev.h"
  40 #include "sfc_rx.h"
  41 #include "sfc_kvargs.h"
  42 #include "sfc_tweak.h"
  43
  44 /*
  45  * Maximum number of Rx queue flush attempt in the case of failure or
  46  * flush timeout
  47  */
  48 #define SFC_RX_QFLUSH_ATTEMPTS          (3)
  49
  50 /*
  51  * Time to wait between event queue polling attempts when waiting for Rx
  52  * queue flush done or failed events.
  53  */
  54 #define SFC_RX_QFLUSH_POLL_WAIT_MS      (1)
  55
  56 /*
  57  * Maximum number of event queue polling attempts when waiting for Rx queue
  58  * flush done or failed events. It defines Rx queue flush attempt timeout
  59  * together with SFC_RX_QFLUSH_POLL_WAIT_MS.
  60  */
  61 #define SFC_RX_QFLUSH_POLL_ATTEMPTS     (2000)
  62
  63 void
  64 sfc_rx_qflush_done(struct sfc_rxq *rxq)
  65 {
  66         rxq->state |= SFC_RXQ_FLUSHED;
  67         rxq->state &= ~SFC_RXQ_FLUSHING;
  68 }
  69
  70 void
  71 sfc_rx_qflush_failed(struct sfc_rxq *rxq)
  72 {
  73         rxq->state |= SFC_RXQ_FLUSH_FAILED;
  74         rxq->state &= ~SFC_RXQ_FLUSHING;
  75 }
  76
  77 static void
  78 sfc_efx_rx_qrefill(struct sfc_efx_rxq *rxq)
  79 {
  80         unsigned int free_space;
  81         unsigned int bulks;
  82         void *objs[SFC_RX_REFILL_BULK];
  83         efsys_dma_addr_t addr[RTE_DIM(objs)];
  84         unsigned int added = rxq->added;
  85         unsigned int id;
  86         unsigned int i;
  87         struct sfc_efx_rx_sw_desc *rxd;
  88         struct rte_mbuf *m;
  89         uint16_t port_id = rxq->dp.dpq.port_id;
  90
  91         free_space = EFX_RXQ_LIMIT(rxq->ptr_mask + 1) -
  92                 (added - rxq->completed);
  93
  94         if (free_space < rxq->refill_threshold)
  95                 return;
  96
  97         bulks = free_space / RTE_DIM(objs);
  98
  99         id = added & rxq->ptr_mask;
 100         while (bulks-- > 0) {
 101                 if (rte_mempool_get_bulk(rxq->refill_mb_pool, objs,
 102                                          RTE_DIM(objs)) < 0) {
 103                         /*
 104                          * It is hardly a safe way to increment counter
 105                          * from different contexts, but all PMDs do it.
 106                          */
 107                         rxq->evq->sa->eth_dev->data->rx_mbuf_alloc_failed +=
 108                                 RTE_DIM(objs);
 109                         break;
 110                 }
 111
 112                 for (i = 0; i < RTE_DIM(objs);
 113                      ++i, id = (id + 1) & rxq->ptr_mask) {
 114                         m = objs[i];
 115
 116                         rxd = &rxq->sw_desc[id];
 117                         rxd->mbuf = m;
 118
 119                         rte_mbuf_refcnt_set(m, 1);
 120                         m->data_off = RTE_PKTMBUF_HEADROOM;
 121                         m->next = NULL;
 122                         m->nb_segs = 1;
 123                         m->port = port_id;
 124
 125                         addr[i] = rte_pktmbuf_mtophys(m);
 126                 }
 127
 128                 efx_rx_qpost(rxq->common, addr, rxq->buf_size,
 129                              RTE_DIM(objs), rxq->completed, added);
 130                 added += RTE_DIM(objs);
 131         }
 132
 133         /* Push doorbell if something is posted */
 134         if (rxq->added != added) {
 135                 rxq->added = added;
 136                 efx_rx_qpush(rxq->common, added, &rxq->pushed);
 137         }
 138 }
 139
 140 static uint64_t
 141 sfc_efx_rx_desc_flags_to_offload_flags(const unsigned int desc_flags)
 142 {
 143         uint64_t mbuf_flags = 0;
 144
 145         switch (desc_flags & (EFX_PKT_IPV4 | EFX_CKSUM_IPV4)) {
 146         case (EFX_PKT_IPV4 | EFX_CKSUM_IPV4):
 147                 mbuf_flags |= PKT_RX_IP_CKSUM_GOOD;
 148                 break;
 149         case EFX_PKT_IPV4:
 150                 mbuf_flags |= PKT_RX_IP_CKSUM_BAD;
 151                 break;
 152         default:
 153                 RTE_BUILD_BUG_ON(PKT_RX_IP_CKSUM_UNKNOWN != 0);
 154                 SFC_ASSERT((mbuf_flags & PKT_RX_IP_CKSUM_MASK) ==
 155                            PKT_RX_IP_CKSUM_UNKNOWN);
 156                 break;
 157         }
 158
 159         switch ((desc_flags &
 160                  (EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP))) {
 161         case (EFX_PKT_TCP | EFX_CKSUM_TCPUDP):
 162         case (EFX_PKT_UDP | EFX_CKSUM_TCPUDP):
 163                 mbuf_flags |= PKT_RX_L4_CKSUM_GOOD;
 164                 break;
 165         case EFX_PKT_TCP:
 166         case EFX_PKT_UDP:
 167                 mbuf_flags |= PKT_RX_L4_CKSUM_BAD;
 168                 break;
 169         default:
 170                 RTE_BUILD_BUG_ON(PKT_RX_L4_CKSUM_UNKNOWN != 0);
 171                 SFC_ASSERT((mbuf_flags & PKT_RX_L4_CKSUM_MASK) ==
 172                            PKT_RX_L4_CKSUM_UNKNOWN);
 173                 break;
 174         }
 175
 176         return mbuf_flags;
 177 }
 178
 179 static uint32_t
 180 sfc_efx_rx_desc_flags_to_packet_type(const unsigned int desc_flags)
 181 {
 182         return RTE_PTYPE_L2_ETHER |
 183                 ((desc_flags & EFX_PKT_IPV4) ?
 184                         RTE_PTYPE_L3_IPV4_EXT_UNKNOWN : 0) |
 185                 ((desc_flags & EFX_PKT_IPV6) ?
 186                         RTE_PTYPE_L3_IPV6_EXT_UNKNOWN : 0) |
 187                 ((desc_flags & EFX_PKT_TCP) ? RTE_PTYPE_L4_TCP : 0) |
 188                 ((desc_flags & EFX_PKT_UDP) ? RTE_PTYPE_L4_UDP : 0);
 189 }
 190
 191 static const uint32_t *
 192 sfc_efx_supported_ptypes_get(void)
 193 {
 194         static const uint32_t ptypes[] = {
 195                 RTE_PTYPE_L2_ETHER,
 196                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 197                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 198                 RTE_PTYPE_L4_TCP,
 199                 RTE_PTYPE_L4_UDP,
 200                 RTE_PTYPE_UNKNOWN
 201         };
 202
 203         return ptypes;
 204 }
 205
 206 static void
 207 sfc_efx_rx_set_rss_hash(struct sfc_efx_rxq *rxq, unsigned int flags,
 208                         struct rte_mbuf *m)
 209 {
 210 #if EFSYS_OPT_RX_SCALE
 211         uint8_t *mbuf_data;
 212
 213
 214         if ((rxq->flags & SFC_EFX_RXQ_FLAG_RSS_HASH) == 0)
 215                 return;
 216
 217         mbuf_data = rte_pktmbuf_mtod(m, uint8_t *);
 218
 219         if (flags & (EFX_PKT_IPV4 | EFX_PKT_IPV6)) {
 220                 m->hash.rss = efx_pseudo_hdr_hash_get(rxq->common,
 221                                                       EFX_RX_HASHALG_TOEPLITZ,
 222                                                       mbuf_data);
 223
 224                 m->ol_flags |= PKT_RX_RSS_HASH;
 225         }
 226 #endif
 227 }
 228
 229 static uint16_t
 230 sfc_efx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 231 {
 232         struct sfc_dp_rxq *dp_rxq = rx_queue;
 233         struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 234         unsigned int completed;
 235         unsigned int prefix_size = rxq->prefix_size;
 236         unsigned int done_pkts = 0;
 237         boolean_t discard_next = B_FALSE;
 238         struct rte_mbuf *scatter_pkt = NULL;
 239
 240         if (unlikely((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0))
 241                 return 0;
 242
 243         sfc_ev_qpoll(rxq->evq);
 244
 245         completed = rxq->completed;
 246         while (completed != rxq->pending && done_pkts < nb_pkts) {
 247                 unsigned int id;
 248                 struct sfc_efx_rx_sw_desc *rxd;
 249                 struct rte_mbuf *m;
 250                 unsigned int seg_len;
 251                 unsigned int desc_flags;
 252
 253                 id = completed++ & rxq->ptr_mask;
 254                 rxd = &rxq->sw_desc[id];
 255                 m = rxd->mbuf;
 256                 desc_flags = rxd->flags;
 257
 258                 if (discard_next)
 259                         goto discard;
 260
 261                 if (desc_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
 262                         goto discard;
 263
 264                 if (desc_flags & EFX_PKT_PREFIX_LEN) {
 265                         uint16_t tmp_size;
 266                         int rc __rte_unused;
 267
 268                         rc = efx_pseudo_hdr_pkt_length_get(rxq->common,
 269                                 rte_pktmbuf_mtod(m, uint8_t *), &tmp_size);
 270                         SFC_ASSERT(rc == 0);
 271                         seg_len = tmp_size;
 272                 } else {
 273                         seg_len = rxd->size - prefix_size;
 274                 }
 275
 276                 rte_pktmbuf_data_len(m) = seg_len;
 277                 rte_pktmbuf_pkt_len(m) = seg_len;
 278
 279                 if (scatter_pkt != NULL) {
 280                         if (rte_pktmbuf_chain(scatter_pkt, m) != 0) {
 281                                 rte_mempool_put(rxq->refill_mb_pool,
 282                                                 scatter_pkt);
 283                                 goto discard;
 284                         }
 285                         /* The packet to deliver */
 286                         m = scatter_pkt;
 287                 }
 288
 289                 if (desc_flags & EFX_PKT_CONT) {
 290                         /* The packet is scattered, more fragments to come */
 291                         scatter_pkt = m;
 292                         /* Futher fragments have no prefix */
 293                         prefix_size = 0;
 294                         continue;
 295                 }
 296
 297                 /* Scattered packet is done */
 298                 scatter_pkt = NULL;
 299                 /* The first fragment of the packet has prefix */
 300                 prefix_size = rxq->prefix_size;
 301
 302                 m->ol_flags =
 303                         sfc_efx_rx_desc_flags_to_offload_flags(desc_flags);
 304                 m->packet_type =
 305                         sfc_efx_rx_desc_flags_to_packet_type(desc_flags);
 306
 307                 /*
 308                  * Extract RSS hash from the packet prefix and
 309                  * set the corresponding field (if needed and possible)
 310                  */
 311                 sfc_efx_rx_set_rss_hash(rxq, desc_flags, m);
 312
 313                 m->data_off += prefix_size;
 314
 315                 *rx_pkts++ = m;
 316                 done_pkts++;
 317                 continue;
 318
 319 discard:
 320                 discard_next = ((desc_flags & EFX_PKT_CONT) != 0);
 321                 rte_mempool_put(rxq->refill_mb_pool, m);
 322                 rxd->mbuf = NULL;
 323         }
 324
 325         /* pending is only moved when entire packet is received */
 326         SFC_ASSERT(scatter_pkt == NULL);
 327
 328         rxq->completed = completed;
 329
 330         sfc_efx_rx_qrefill(rxq);
 331
 332         return done_pkts;
 333 }
 334
 335 static sfc_dp_rx_qdesc_npending_t sfc_efx_rx_qdesc_npending;
 336 static unsigned int
 337 sfc_efx_rx_qdesc_npending(struct sfc_dp_rxq *dp_rxq)
 338 {
 339         struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 340
 341         if ((rxq->flags & SFC_EFX_RXQ_FLAG_RUNNING) == 0)
 342                 return 0;
 343
 344         sfc_ev_qpoll(rxq->evq);
 345
 346         return rxq->pending - rxq->completed;
 347 }
 348
 349 struct sfc_rxq *
 350 sfc_rxq_by_dp_rxq(const struct sfc_dp_rxq *dp_rxq)
 351 {
 352         const struct sfc_dp_queue *dpq = &dp_rxq->dpq;
 353         struct rte_eth_dev *eth_dev;
 354         struct sfc_adapter *sa;
 355         struct sfc_rxq *rxq;
 356
 357         SFC_ASSERT(rte_eth_dev_is_valid_port(dpq->port_id));
 358         eth_dev = &rte_eth_devices[dpq->port_id];
 359
 360         sa = eth_dev->data->dev_private;
 361
 362         SFC_ASSERT(dpq->queue_id < sa->rxq_count);
 363         rxq = sa->rxq_info[dpq->queue_id].rxq;
 364
 365         SFC_ASSERT(rxq != NULL);
 366         return rxq;
 367 }
 368
 369 static sfc_dp_rx_qcreate_t sfc_efx_rx_qcreate;
 370 static int
 371 sfc_efx_rx_qcreate(uint16_t port_id, uint16_t queue_id,
 372                    const struct rte_pci_addr *pci_addr, int socket_id,
 373                    const struct sfc_dp_rx_qcreate_info *info,
 374                    struct sfc_dp_rxq **dp_rxqp)
 375 {
 376         struct sfc_efx_rxq *rxq;
 377         int rc;
 378
 379         rc = ENOMEM;
 380         rxq = rte_zmalloc_socket("sfc-efx-rxq", sizeof(*rxq),
 381                                  RTE_CACHE_LINE_SIZE, socket_id);
 382         if (rxq == NULL)
 383                 goto fail_rxq_alloc;
 384
 385         sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr);
 386
 387         rc = ENOMEM;
 388         rxq->sw_desc = rte_calloc_socket("sfc-efx-rxq-sw_desc",
 389                                          info->rxq_entries,
 390                                          sizeof(*rxq->sw_desc),
 391                                          RTE_CACHE_LINE_SIZE, socket_id);
 392         if (rxq->sw_desc == NULL)
 393                 goto fail_desc_alloc;
 394
 395         /* efx datapath is bound to efx control path */
 396         rxq->evq = sfc_rxq_by_dp_rxq(&rxq->dp)->evq;
 397         if (info->flags & SFC_RXQ_FLAG_RSS_HASH)
 398                 rxq->flags |= SFC_EFX_RXQ_FLAG_RSS_HASH;
 399         rxq->ptr_mask = info->rxq_entries - 1;
 400         rxq->batch_max = info->batch_max;
 401         rxq->prefix_size = info->prefix_size;
 402         rxq->refill_threshold = info->refill_threshold;
 403         rxq->buf_size = info->buf_size;
 404         rxq->refill_mb_pool = info->refill_mb_pool;
 405
 406         *dp_rxqp = &rxq->dp;
 407         return 0;
 408
 409 fail_desc_alloc:
 410         rte_free(rxq);
 411
 412 fail_rxq_alloc:
 413         return rc;
 414 }
 415
 416 static sfc_dp_rx_qdestroy_t sfc_efx_rx_qdestroy;
 417 static void
 418 sfc_efx_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
 419 {
 420         struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 421
 422         rte_free(rxq->sw_desc);
 423         rte_free(rxq);
 424 }
 425
 426 static sfc_dp_rx_qstart_t sfc_efx_rx_qstart;
 427 static int
 428 sfc_efx_rx_qstart(struct sfc_dp_rxq *dp_rxq,
 429                   __rte_unused unsigned int evq_read_ptr)
 430 {
 431         /* libefx-based datapath is specific to libefx-based PMD */
 432         struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 433         struct sfc_rxq *crxq = sfc_rxq_by_dp_rxq(dp_rxq);
 434
 435         rxq->common = crxq->common;
 436
 437         rxq->pending = rxq->completed = rxq->added = rxq->pushed = 0;
 438
 439         sfc_efx_rx_qrefill(rxq);
 440
 441         rxq->flags |= (SFC_EFX_RXQ_FLAG_STARTED | SFC_EFX_RXQ_FLAG_RUNNING);
 442
 443         return 0;
 444 }
 445
 446 static sfc_dp_rx_qstop_t sfc_efx_rx_qstop;
 447 static void
 448 sfc_efx_rx_qstop(struct sfc_dp_rxq *dp_rxq,
 449                  __rte_unused unsigned int *evq_read_ptr)
 450 {
 451         struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 452
 453         rxq->flags &= ~SFC_EFX_RXQ_FLAG_RUNNING;
 454
 455         /* libefx-based datapath is bound to libefx-based PMD and uses
 456          * event queue structure directly. So, there is no necessity to
 457          * return EvQ read pointer.
 458          */
 459 }
 460
 461 static sfc_dp_rx_qpurge_t sfc_efx_rx_qpurge;
 462 static void
 463 sfc_efx_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 464 {
 465         struct sfc_efx_rxq *rxq = sfc_efx_rxq_by_dp_rxq(dp_rxq);
 466         unsigned int i;
 467         struct sfc_efx_rx_sw_desc *rxd;
 468
 469         for (i = rxq->completed; i != rxq->added; ++i) {
 470                 rxd = &rxq->sw_desc[i & rxq->ptr_mask];
 471                 rte_mempool_put(rxq->refill_mb_pool, rxd->mbuf);
 472                 rxd->mbuf = NULL;
 473                 /* Packed stream relies on 0 in inactive SW desc.
 474                  * Rx queue stop is not performance critical, so
 475                  * there is no harm to do it always.
 476                  */
 477                 rxd->flags = 0;
 478                 rxd->size = 0;
 479         }
 480
 481         rxq->flags &= ~SFC_EFX_RXQ_FLAG_STARTED;
 482 }
 483
 484 struct sfc_dp_rx sfc_efx_rx = {
 485         .dp = {
 486                 .name           = SFC_KVARG_DATAPATH_EFX,
 487                 .type           = SFC_DP_RX,
 488                 .hw_fw_caps     = 0,
 489         },
 490         .features               = SFC_DP_RX_FEAT_SCATTER,
 491         .qcreate                = sfc_efx_rx_qcreate,
 492         .qdestroy               = sfc_efx_rx_qdestroy,
 493         .qstart                 = sfc_efx_rx_qstart,
 494         .qstop                  = sfc_efx_rx_qstop,
 495         .qpurge                 = sfc_efx_rx_qpurge,
 496         .supported_ptypes_get   = sfc_efx_supported_ptypes_get,
 497         .qdesc_npending         = sfc_efx_rx_qdesc_npending,
 498         .pkt_burst              = sfc_efx_recv_pkts,
 499 };
 500
 501 unsigned int
 502 sfc_rx_qdesc_npending(struct sfc_adapter *sa, unsigned int sw_index)
 503 {
 504         struct sfc_rxq *rxq;
 505
 506         SFC_ASSERT(sw_index < sa->rxq_count);
 507         rxq = sa->rxq_info[sw_index].rxq;
 508
 509         if (rxq == NULL || (rxq->state & SFC_RXQ_STARTED) == 0)
 510                 return 0;
 511
 512         return sa->dp_rx->qdesc_npending(rxq->dp);
 513 }
 514
 515 int
 516 sfc_rx_qdesc_done(struct sfc_dp_rxq *dp_rxq, unsigned int offset)
 517 {
 518         struct sfc_rxq *rxq = sfc_rxq_by_dp_rxq(dp_rxq);
 519
 520         return offset < rxq->evq->sa->dp_rx->qdesc_npending(dp_rxq);
 521 }
 522
 523 static void
 524 sfc_rx_qflush(struct sfc_adapter *sa, unsigned int sw_index)
 525 {
 526         struct sfc_rxq *rxq;
 527         unsigned int retry_count;
 528         unsigned int wait_count;
 529
 530         rxq = sa->rxq_info[sw_index].rxq;
 531         SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
 532
 533         /*
 534          * Retry Rx queue flushing in the case of flush failed or
 535          * timeout. In the worst case it can delay for 6 seconds.
 536          */
 537         for (retry_count = 0;
 538              ((rxq->state & SFC_RXQ_FLUSHED) == 0) &&
 539              (retry_count < SFC_RX_QFLUSH_ATTEMPTS);
 540              ++retry_count) {
 541                 if (efx_rx_qflush(rxq->common) != 0) {
 542                         rxq->state |= SFC_RXQ_FLUSH_FAILED;
 543                         break;
 544                 }
 545                 rxq->state &= ~SFC_RXQ_FLUSH_FAILED;
 546                 rxq->state |= SFC_RXQ_FLUSHING;
 547
 548                 /*
 549                  * Wait for Rx queue flush done or failed event at least
 550                  * SFC_RX_QFLUSH_POLL_WAIT_MS milliseconds and not more
 551                  * than 2 seconds (SFC_RX_QFLUSH_POLL_WAIT_MS multiplied
 552                  * by SFC_RX_QFLUSH_POLL_ATTEMPTS).
 553                  */
 554                 wait_count = 0;
 555                 do {
 556                         rte_delay_ms(SFC_RX_QFLUSH_POLL_WAIT_MS);
 557                         sfc_ev_qpoll(rxq->evq);
 558                 } while ((rxq->state & SFC_RXQ_FLUSHING) &&
 559                          (wait_count++ < SFC_RX_QFLUSH_POLL_ATTEMPTS));
 560
 561                 if (rxq->state & SFC_RXQ_FLUSHING)
 562                         sfc_err(sa, "RxQ %u flush timed out", sw_index);
 563
 564                 if (rxq->state & SFC_RXQ_FLUSH_FAILED)
 565                         sfc_err(sa, "RxQ %u flush failed", sw_index);
 566
 567                 if (rxq->state & SFC_RXQ_FLUSHED)
 568                         sfc_info(sa, "RxQ %u flushed", sw_index);
 569         }
 570
 571         sa->dp_rx->qpurge(rxq->dp);
 572 }
 573
 574 static int
 575 sfc_rx_default_rxq_set_filter(struct sfc_adapter *sa, struct sfc_rxq *rxq)
 576 {
 577         boolean_t rss = (sa->rss_channels > 1) ? B_TRUE : B_FALSE;
 578         struct sfc_port *port = &sa->port;
 579         int rc;
 580
 581         /*
 582          * If promiscuous or all-multicast mode has been requested, setting
 583          * filter for the default Rx queue might fail, in particular, while
 584          * running over PCI function which is not a member of corresponding
 585          * privilege groups; if this occurs, few iterations will be made to
 586          * repeat this step without promiscuous and all-multicast flags set
 587          */
 588 retry:
 589         rc = efx_mac_filter_default_rxq_set(sa->nic, rxq->common, rss);
 590         if (rc == 0)
 591                 return 0;
 592         else if (rc != EOPNOTSUPP)
 593                 return rc;
 594
 595         if (port->promisc) {
 596                 sfc_warn(sa, "promiscuous mode has been requested, "
 597                              "but the HW rejects it");
 598                 sfc_warn(sa, "promiscuous mode will be disabled");
 599
 600                 port->promisc = B_FALSE;
 601                 rc = sfc_set_rx_mode(sa);
 602                 if (rc != 0)
 603                         return rc;
 604
 605                 goto retry;
 606         }
 607
 608         if (port->allmulti) {
 609                 sfc_warn(sa, "all-multicast mode has been requested, "
 610                              "but the HW rejects it");
 611                 sfc_warn(sa, "all-multicast mode will be disabled");
 612
 613                 port->allmulti = B_FALSE;
 614                 rc = sfc_set_rx_mode(sa);
 615                 if (rc != 0)
 616                         return rc;
 617
 618                 goto retry;
 619         }
 620
 621         return rc;
 622 }
 623
 624 int
 625 sfc_rx_qstart(struct sfc_adapter *sa, unsigned int sw_index)
 626 {
 627         struct sfc_rxq_info *rxq_info;
 628         struct sfc_rxq *rxq;
 629         struct sfc_evq *evq;
 630         int rc;
 631
 632         sfc_log_init(sa, "sw_index=%u", sw_index);
 633
 634         SFC_ASSERT(sw_index < sa->rxq_count);
 635
 636         rxq_info = &sa->rxq_info[sw_index];
 637         rxq = rxq_info->rxq;
 638         SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED);
 639
 640         evq = rxq->evq;
 641
 642         rc = sfc_ev_qstart(sa, evq->evq_index);
 643         if (rc != 0)
 644                 goto fail_ev_qstart;
 645
 646         rc = efx_rx_qcreate(sa->nic, rxq->hw_index, 0, rxq_info->type,
 647                             &rxq->mem, rxq_info->entries,
 648                             0 /* not used on EF10 */, evq->common,
 649                             &rxq->common);
 650         if (rc != 0)
 651                 goto fail_rx_qcreate;
 652
 653         efx_rx_qenable(rxq->common);
 654
 655         rc = sa->dp_rx->qstart(rxq->dp, evq->read_ptr);
 656         if (rc != 0)
 657                 goto fail_dp_qstart;
 658
 659         rxq->state |= SFC_RXQ_STARTED;
 660
 661         if (sw_index == 0) {
 662                 rc = sfc_rx_default_rxq_set_filter(sa, rxq);
 663                 if (rc != 0)
 664                         goto fail_mac_filter_default_rxq_set;
 665         }
 666
 667         /* It seems to be used by DPDK for debug purposes only ('rte_ether') */
 668         sa->eth_dev->data->rx_queue_state[sw_index] =
 669                 RTE_ETH_QUEUE_STATE_STARTED;
 670
 671         return 0;
 672
 673 fail_mac_filter_default_rxq_set:
 674         sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr);
 675
 676 fail_dp_qstart:
 677         sfc_rx_qflush(sa, sw_index);
 678
 679 fail_rx_qcreate:
 680         sfc_ev_qstop(sa, evq->evq_index);
 681
 682 fail_ev_qstart:
 683         return rc;
 684 }
 685
 686 void
 687 sfc_rx_qstop(struct sfc_adapter *sa, unsigned int sw_index)
 688 {
 689         struct sfc_rxq_info *rxq_info;
 690         struct sfc_rxq *rxq;
 691
 692         sfc_log_init(sa, "sw_index=%u", sw_index);
 693
 694         SFC_ASSERT(sw_index < sa->rxq_count);
 695
 696         rxq_info = &sa->rxq_info[sw_index];
 697         rxq = rxq_info->rxq;
 698
 699         if (rxq->state == SFC_RXQ_INITIALIZED)
 700                 return;
 701         SFC_ASSERT(rxq->state & SFC_RXQ_STARTED);
 702
 703         /* It seems to be used by DPDK for debug purposes only ('rte_ether') */
 704         sa->eth_dev->data->rx_queue_state[sw_index] =
 705                 RTE_ETH_QUEUE_STATE_STOPPED;
 706
 707         sa->dp_rx->qstop(rxq->dp, &rxq->evq->read_ptr);
 708
 709         if (sw_index == 0)
 710                 efx_mac_filter_default_rxq_clear(sa->nic);
 711
 712         sfc_rx_qflush(sa, sw_index);
 713
 714         rxq->state = SFC_RXQ_INITIALIZED;
 715
 716         efx_rx_qdestroy(rxq->common);
 717
 718         sfc_ev_qstop(sa, rxq->evq->evq_index);
 719 }
 720
 721 static int
 722 sfc_rx_qcheck_conf(struct sfc_adapter *sa, uint16_t nb_rx_desc,
 723                    const struct rte_eth_rxconf *rx_conf)
 724 {
 725         const uint16_t rx_free_thresh_max = EFX_RXQ_LIMIT(nb_rx_desc);
 726         int rc = 0;
 727
 728         if (rx_conf->rx_thresh.pthresh != 0 ||
 729             rx_conf->rx_thresh.hthresh != 0 ||
 730             rx_conf->rx_thresh.wthresh != 0) {
 731                 sfc_err(sa,
 732                         "RxQ prefetch/host/writeback thresholds are not supported");
 733                 rc = EINVAL;
 734         }
 735
 736         if (rx_conf->rx_free_thresh > rx_free_thresh_max) {
 737                 sfc_err(sa,
 738                         "RxQ free threshold too large: %u vs maximum %u",
 739                         rx_conf->rx_free_thresh, rx_free_thresh_max);
 740                 rc = EINVAL;
 741         }
 742
 743         if (rx_conf->rx_drop_en == 0) {
 744                 sfc_err(sa, "RxQ drop disable is not supported");
 745                 rc = EINVAL;
 746         }
 747
 748         return rc;
 749 }
 750
 751 static unsigned int
 752 sfc_rx_mbuf_data_alignment(struct rte_mempool *mb_pool)
 753 {
 754         uint32_t data_off;
 755         uint32_t order;
 756
 757         /* The mbuf object itself is always cache line aligned */
 758         order = rte_bsf32(RTE_CACHE_LINE_SIZE);
 759
 760         /* Data offset from mbuf object start */
 761         data_off = sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mb_pool) +
 762                 RTE_PKTMBUF_HEADROOM;
 763
 764         order = MIN(order, rte_bsf32(data_off));
 765
 766         return 1u << (order - 1);
 767 }
 768
 769 static uint16_t
 770 sfc_rx_mb_pool_buf_size(struct sfc_adapter *sa, struct rte_mempool *mb_pool)
 771 {
 772         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
 773         const uint32_t nic_align_start = MAX(1, encp->enc_rx_buf_align_start);
 774         const uint32_t nic_align_end = MAX(1, encp->enc_rx_buf_align_end);
 775         uint16_t buf_size;
 776         unsigned int buf_aligned;
 777         unsigned int start_alignment;
 778         unsigned int end_padding_alignment;
 779
 780         /* Below it is assumed that both alignments are power of 2 */
 781         SFC_ASSERT(rte_is_power_of_2(nic_align_start));
 782         SFC_ASSERT(rte_is_power_of_2(nic_align_end));
 783
 784         /*
 785          * mbuf is always cache line aligned, double-check
 786          * that it meets rx buffer start alignment requirements.
 787          */
 788
 789         /* Start from mbuf pool data room size */
 790         buf_size = rte_pktmbuf_data_room_size(mb_pool);
 791
 792         /* Remove headroom */
 793         if (buf_size <= RTE_PKTMBUF_HEADROOM) {
 794                 sfc_err(sa,
 795                         "RxQ mbuf pool %s object data room size %u is smaller than headroom %u",
 796                         mb_pool->name, buf_size, RTE_PKTMBUF_HEADROOM);
 797                 return 0;
 798         }
 799         buf_size -= RTE_PKTMBUF_HEADROOM;
 800
 801         /* Calculate guaranteed data start alignment */
 802         buf_aligned = sfc_rx_mbuf_data_alignment(mb_pool);
 803
 804         /* Reserve space for start alignment */
 805         if (buf_aligned < nic_align_start) {
 806                 start_alignment = nic_align_start - buf_aligned;
 807                 if (buf_size <= start_alignment) {
 808                         sfc_err(sa,
 809                                 "RxQ mbuf pool %s object data room size %u is insufficient for headroom %u and buffer start alignment %u required by NIC",
 810                                 mb_pool->name,
 811                                 rte_pktmbuf_data_room_size(mb_pool),
 812                                 RTE_PKTMBUF_HEADROOM, start_alignment);
 813                         return 0;
 814                 }
 815                 buf_aligned = nic_align_start;
 816                 buf_size -= start_alignment;
 817         } else {
 818                 start_alignment = 0;
 819         }
 820
 821         /* Make sure that end padding does not write beyond the buffer */
 822         if (buf_aligned < nic_align_end) {
 823                 /*
 824                  * Estimate space which can be lost. If guarnteed buffer
 825                  * size is odd, lost space is (nic_align_end - 1). More
 826                  * accurate formula is below.
 827                  */
 828                 end_padding_alignment = nic_align_end -
 829                         MIN(buf_aligned, 1u << (rte_bsf32(buf_size) - 1));
 830                 if (buf_size <= end_padding_alignment) {
 831                         sfc_err(sa,
 832                                 "RxQ mbuf pool %s object data room size %u is insufficient for headroom %u, buffer start alignment %u and end padding alignment %u required by NIC",
 833                                 mb_pool->name,
 834                                 rte_pktmbuf_data_room_size(mb_pool),
 835                                 RTE_PKTMBUF_HEADROOM, start_alignment,
 836                                 end_padding_alignment);
 837                         return 0;
 838                 }
 839                 buf_size -= end_padding_alignment;
 840         } else {
 841                 /*
 842                  * Start is aligned the same or better than end,
 843                  * just align length.
 844                  */
 845                 buf_size = P2ALIGN(buf_size, nic_align_end);
 846         }
 847
 848         return buf_size;
 849 }
 850
 851 int
 852 sfc_rx_qinit(struct sfc_adapter *sa, unsigned int sw_index,
 853              uint16_t nb_rx_desc, unsigned int socket_id,
 854              const struct rte_eth_rxconf *rx_conf,
 855              struct rte_mempool *mb_pool)
 856 {
 857         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
 858         int rc;
 859         uint16_t buf_size;
 860         struct sfc_rxq_info *rxq_info;
 861         unsigned int evq_index;
 862         struct sfc_evq *evq;
 863         struct sfc_rxq *rxq;
 864         struct sfc_dp_rx_qcreate_info info;
 865
 866         rc = sfc_rx_qcheck_conf(sa, nb_rx_desc, rx_conf);
 867         if (rc != 0)
 868                 goto fail_bad_conf;
 869
 870         buf_size = sfc_rx_mb_pool_buf_size(sa, mb_pool);
 871         if (buf_size == 0) {
 872                 sfc_err(sa, "RxQ %u mbuf pool object size is too small",
 873                         sw_index);
 874                 rc = EINVAL;
 875                 goto fail_bad_conf;
 876         }
 877
 878         if ((buf_size < sa->port.pdu + encp->enc_rx_prefix_size) &&
 879             !sa->eth_dev->data->dev_conf.rxmode.enable_scatter) {
 880                 sfc_err(sa, "Rx scatter is disabled and RxQ %u mbuf pool "
 881                         "object size is too small", sw_index);
 882                 sfc_err(sa, "RxQ %u calculated Rx buffer size is %u vs "
 883                         "PDU size %u plus Rx prefix %u bytes",
 884                         sw_index, buf_size, (unsigned int)sa->port.pdu,
 885                         encp->enc_rx_prefix_size);
 886                 rc = EINVAL;
 887                 goto fail_bad_conf;
 888         }
 889
 890         SFC_ASSERT(sw_index < sa->rxq_count);
 891         rxq_info = &sa->rxq_info[sw_index];
 892
 893         SFC_ASSERT(nb_rx_desc <= rxq_info->max_entries);
 894         rxq_info->entries = nb_rx_desc;
 895         rxq_info->type =
 896                 sa->eth_dev->data->dev_conf.rxmode.enable_scatter ?
 897                 EFX_RXQ_TYPE_SCATTER : EFX_RXQ_TYPE_DEFAULT;
 898
 899         evq_index = sfc_evq_index_by_rxq_sw_index(sa, sw_index);
 900
 901         rc = sfc_ev_qinit(sa, evq_index, rxq_info->entries, socket_id);
 902         if (rc != 0)
 903                 goto fail_ev_qinit;
 904
 905         evq = sa->evq_info[evq_index].evq;
 906
 907         rc = ENOMEM;
 908         rxq = rte_zmalloc_socket("sfc-rxq", sizeof(*rxq), RTE_CACHE_LINE_SIZE,
 909                                  socket_id);
 910         if (rxq == NULL)
 911                 goto fail_rxq_alloc;
 912
 913         rxq_info->rxq = rxq;
 914
 915         rxq->evq = evq;
 916         rxq->hw_index = sw_index;
 917         rxq->refill_threshold = rx_conf->rx_free_thresh;
 918         rxq->refill_mb_pool = mb_pool;
 919
 920         rc = sfc_dma_alloc(sa, "rxq", sw_index, EFX_RXQ_SIZE(rxq_info->entries),
 921                            socket_id, &rxq->mem);
 922         if (rc != 0)
 923                 goto fail_dma_alloc;
 924
 925         memset(&info, 0, sizeof(info));
 926         info.refill_mb_pool = rxq->refill_mb_pool;
 927         info.refill_threshold = rxq->refill_threshold;
 928         info.buf_size = buf_size;
 929         info.batch_max = encp->enc_rx_batch_max;
 930         info.prefix_size = encp->enc_rx_prefix_size;
 931
 932 #if EFSYS_OPT_RX_SCALE
 933         if (sa->hash_support == EFX_RX_HASH_AVAILABLE)
 934                 info.flags |= SFC_RXQ_FLAG_RSS_HASH;
 935 #endif
 936
 937         info.rxq_entries = rxq_info->entries;
 938
 939         rc = sa->dp_rx->qcreate(sa->eth_dev->data->port_id, sw_index,
 940                                 &SFC_DEV_TO_PCI(sa->eth_dev)->addr,
 941                                 socket_id, &info, &rxq->dp);
 942         if (rc != 0)
 943                 goto fail_dp_rx_qcreate;
 944
 945         evq->dp_rxq = rxq->dp;
 946
 947         rxq->state = SFC_RXQ_INITIALIZED;
 948
 949         rxq_info->deferred_start = (rx_conf->rx_deferred_start != 0);
 950
 951         return 0;
 952
 953 fail_dp_rx_qcreate:
 954         sfc_dma_free(sa, &rxq->mem);
 955
 956 fail_dma_alloc:
 957         rxq_info->rxq = NULL;
 958         rte_free(rxq);
 959
 960 fail_rxq_alloc:
 961         sfc_ev_qfini(sa, evq_index);
 962
 963 fail_ev_qinit:
 964         rxq_info->entries = 0;
 965
 966 fail_bad_conf:
 967         sfc_log_init(sa, "failed %d", rc);
 968         return rc;
 969 }
 970
 971 void
 972 sfc_rx_qfini(struct sfc_adapter *sa, unsigned int sw_index)
 973 {
 974         struct sfc_rxq_info *rxq_info;
 975         struct sfc_rxq *rxq;
 976
 977         SFC_ASSERT(sw_index < sa->rxq_count);
 978
 979         rxq_info = &sa->rxq_info[sw_index];
 980
 981         rxq = rxq_info->rxq;
 982         SFC_ASSERT(rxq->state == SFC_RXQ_INITIALIZED);
 983
 984         sa->dp_rx->qdestroy(rxq->dp);
 985         rxq->dp = NULL;
 986
 987         rxq_info->rxq = NULL;
 988         rxq_info->entries = 0;
 989
 990         sfc_dma_free(sa, &rxq->mem);
 991         rte_free(rxq);
 992 }
 993
 994 #if EFSYS_OPT_RX_SCALE
 995 efx_rx_hash_type_t
 996 sfc_rte_to_efx_hash_type(uint64_t rss_hf)
 997 {
 998         efx_rx_hash_type_t efx_hash_types = 0;
 999
1000         if ((rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1001                        ETH_RSS_NONFRAG_IPV4_OTHER)) != 0)
1002                 efx_hash_types |= EFX_RX_HASH_IPV4;
1003
1004         if ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != 0)
1005                 efx_hash_types |= EFX_RX_HASH_TCPIPV4;
1006
1007         if ((rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
1008                         ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX)) != 0)
1009                 efx_hash_types |= EFX_RX_HASH_IPV6;
1010
1011         if ((rss_hf & (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX)) != 0)
1012                 efx_hash_types |= EFX_RX_HASH_TCPIPV6;
1013
1014         return efx_hash_types;
1015 }
1016
1017 uint64_t
1018 sfc_efx_to_rte_hash_type(efx_rx_hash_type_t efx_hash_types)
1019 {
1020         uint64_t rss_hf = 0;
1021
1022         if ((efx_hash_types & EFX_RX_HASH_IPV4) != 0)
1023                 rss_hf |= (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
1024                            ETH_RSS_NONFRAG_IPV4_OTHER);
1025
1026         if ((efx_hash_types & EFX_RX_HASH_TCPIPV4) != 0)
1027                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
1028
1029         if ((efx_hash_types & EFX_RX_HASH_IPV6) != 0)
1030                 rss_hf |= (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
1031                            ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX);
1032
1033         if ((efx_hash_types & EFX_RX_HASH_TCPIPV6) != 0)
1034                 rss_hf |= (ETH_RSS_NONFRAG_IPV6_TCP | ETH_RSS_IPV6_TCP_EX);
1035
1036         return rss_hf;
1037 }
1038 #endif
1039
1040 static int
1041 sfc_rx_rss_config(struct sfc_adapter *sa)
1042 {
1043         int rc = 0;
1044
1045 #if EFSYS_OPT_RX_SCALE
1046         if (sa->rss_channels > 1) {
1047                 rc = efx_rx_scale_mode_set(sa->nic, EFX_RX_HASHALG_TOEPLITZ,
1048                                            sa->rss_hash_types, B_TRUE);
1049                 if (rc != 0)
1050                         goto finish;
1051
1052                 rc = efx_rx_scale_key_set(sa->nic, sa->rss_key,
1053                                           sizeof(sa->rss_key));
1054                 if (rc != 0)
1055                         goto finish;
1056
1057                 rc = efx_rx_scale_tbl_set(sa->nic, sa->rss_tbl,
1058                                           sizeof(sa->rss_tbl));
1059         }
1060
1061 finish:
1062 #endif
1063         return rc;
1064 }
1065
1066 int
1067 sfc_rx_start(struct sfc_adapter *sa)
1068 {
1069         unsigned int sw_index;
1070         int rc;
1071
1072         sfc_log_init(sa, "rxq_count=%u", sa->rxq_count);
1073
1074         rc = efx_rx_init(sa->nic);
1075         if (rc != 0)
1076                 goto fail_rx_init;
1077
1078         rc = sfc_rx_rss_config(sa);
1079         if (rc != 0)
1080                 goto fail_rss_config;
1081
1082         for (sw_index = 0; sw_index < sa->rxq_count; ++sw_index) {
1083                 if ((!sa->rxq_info[sw_index].deferred_start ||
1084                      sa->rxq_info[sw_index].deferred_started)) {
1085                         rc = sfc_rx_qstart(sa, sw_index);
1086                         if (rc != 0)
1087                                 goto fail_rx_qstart;
1088                 }
1089         }
1090
1091         return 0;
1092
1093 fail_rx_qstart:
1094         while (sw_index-- > 0)
1095                 sfc_rx_qstop(sa, sw_index);
1096
1097 fail_rss_config:
1098         efx_rx_fini(sa->nic);
1099
1100 fail_rx_init:
1101         sfc_log_init(sa, "failed %d", rc);
1102         return rc;
1103 }
1104
1105 void
1106 sfc_rx_stop(struct sfc_adapter *sa)
1107 {
1108         unsigned int sw_index;
1109
1110         sfc_log_init(sa, "rxq_count=%u", sa->rxq_count);
1111
1112         sw_index = sa->rxq_count;
1113         while (sw_index-- > 0) {
1114                 if (sa->rxq_info[sw_index].rxq != NULL)
1115                         sfc_rx_qstop(sa, sw_index);
1116         }
1117
1118         efx_rx_fini(sa->nic);
1119 }
1120
1121 static int
1122 sfc_rx_qinit_info(struct sfc_adapter *sa, unsigned int sw_index)
1123 {
1124         struct sfc_rxq_info *rxq_info = &sa->rxq_info[sw_index];
1125         unsigned int max_entries;
1126
1127         max_entries = EFX_RXQ_MAXNDESCS;
1128         SFC_ASSERT(rte_is_power_of_2(max_entries));
1129
1130         rxq_info->max_entries = max_entries;
1131
1132         return 0;
1133 }
1134
1135 static int
1136 sfc_rx_check_mode(struct sfc_adapter *sa, struct rte_eth_rxmode *rxmode)
1137 {
1138         int rc = 0;
1139
1140         switch (rxmode->mq_mode) {
1141         case ETH_MQ_RX_NONE:
1142                 /* No special checks are required */
1143                 break;
1144 #if EFSYS_OPT_RX_SCALE
1145         case ETH_MQ_RX_RSS:
1146                 if (sa->rss_support == EFX_RX_SCALE_UNAVAILABLE) {
1147                         sfc_err(sa, "RSS is not available");
1148                         rc = EINVAL;
1149                 }
1150                 break;
1151 #endif
1152         default:
1153                 sfc_err(sa, "Rx multi-queue mode %u not supported",
1154                         rxmode->mq_mode);
1155                 rc = EINVAL;
1156         }
1157
1158         if (rxmode->header_split) {
1159                 sfc_err(sa, "Header split on Rx not supported");
1160                 rc = EINVAL;
1161         }
1162
1163         if (rxmode->hw_vlan_filter) {
1164                 sfc_err(sa, "HW VLAN filtering not supported");
1165                 rc = EINVAL;
1166         }
1167
1168         if (rxmode->hw_vlan_strip) {
1169                 sfc_err(sa, "HW VLAN stripping not supported");
1170                 rc = EINVAL;
1171         }
1172
1173         if (rxmode->hw_vlan_extend) {
1174                 sfc_err(sa,
1175                         "Q-in-Q HW VLAN stripping not supported");
1176                 rc = EINVAL;
1177         }
1178
1179         if (!rxmode->hw_strip_crc) {
1180                 sfc_warn(sa,
1181                          "FCS stripping control not supported - always stripped");
1182                 rxmode->hw_strip_crc = 1;
1183         }
1184
1185         if (rxmode->enable_scatter &&
1186             (~sa->dp_rx->features & SFC_DP_RX_FEAT_SCATTER)) {
1187                 sfc_err(sa, "Rx scatter not supported by %s datapath",
1188                         sa->dp_rx->dp.name);
1189                 rc = EINVAL;
1190         }
1191
1192         if (rxmode->enable_lro) {
1193                 sfc_err(sa, "LRO not supported");
1194                 rc = EINVAL;
1195         }
1196
1197         return rc;
1198 }
1199
1200 /**
1201  * Initialize Rx subsystem.
1202  *
1203  * Called at device configuration stage when number of receive queues is
1204  * specified together with other device level receive configuration.
1205  *
1206  * It should be used to allocate NUMA-unaware resources.
1207  */
1208 int
1209 sfc_rx_init(struct sfc_adapter *sa)
1210 {
1211         struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf;
1212         unsigned int sw_index;
1213         int rc;
1214
1215         rc = sfc_rx_check_mode(sa, &dev_conf->rxmode);
1216         if (rc != 0)
1217                 goto fail_check_mode;
1218
1219         sa->rxq_count = sa->eth_dev->data->nb_rx_queues;
1220
1221         rc = ENOMEM;
1222         sa->rxq_info = rte_calloc_socket("sfc-rxqs", sa->rxq_count,
1223                                          sizeof(struct sfc_rxq_info), 0,
1224                                          sa->socket_id);
1225         if (sa->rxq_info == NULL)
1226                 goto fail_rxqs_alloc;
1227
1228         for (sw_index = 0; sw_index < sa->rxq_count; ++sw_index) {
1229                 rc = sfc_rx_qinit_info(sa, sw_index);
1230                 if (rc != 0)
1231                         goto fail_rx_qinit_info;
1232         }
1233
1234 #if EFSYS_OPT_RX_SCALE
1235         sa->rss_channels = (dev_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) ?
1236                            MIN(sa->rxq_count, EFX_MAXRSS) : 1;
1237
1238         if (sa->rss_channels > 1) {
1239                 for (sw_index = 0; sw_index < EFX_RSS_TBL_SIZE; ++sw_index)
1240                         sa->rss_tbl[sw_index] = sw_index % sa->rss_channels;
1241         }
1242 #endif
1243
1244         return 0;
1245
1246 fail_rx_qinit_info:
1247         rte_free(sa->rxq_info);
1248         sa->rxq_info = NULL;
1249
1250 fail_rxqs_alloc:
1251         sa->rxq_count = 0;
1252 fail_check_mode:
1253         sfc_log_init(sa, "failed %d", rc);
1254         return rc;
1255 }
1256
1257 /**
1258  * Shutdown Rx subsystem.
1259  *
1260  * Called at device close stage, for example, before device
1261  * reconfiguration or shutdown.
1262  */
1263 void
1264 sfc_rx_fini(struct sfc_adapter *sa)
1265 {
1266         unsigned int sw_index;
1267
1268         sw_index = sa->rxq_count;
1269         while (sw_index-- > 0) {
1270                 if (sa->rxq_info[sw_index].rxq != NULL)
1271                         sfc_rx_qfini(sa, sw_index);
1272         }
1273
1274         rte_free(sa->rxq_info);
1275         sa->rxq_info = NULL;
1276         sa->rxq_count = 0;
1277 }