drivers/net/enic/enic_rxtx.c

   1 /* Copyright 2008-2016 Cisco Systems, Inc.  All rights reserved.
   2  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   3  *
   4  * Copyright (c) 2014, Cisco Systems, Inc.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  * notice, this list of conditions and the following disclaimer.
  13  *
  14  * 2. Redistributions in binary form must reproduce the above copyright
  15  * notice, this list of conditions and the following disclaimer in
  16  * the documentation and/or other materials provided with the
  17  * distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  22  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
  23  * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  25  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  26  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  27  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  29  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_mbuf.h>
  34 #include <rte_ethdev.h>
  35 #include <rte_prefetch.h>
  36
  37 #include "enic_compat.h"
  38 #include "rq_enet_desc.h"
  39 #include "enic.h"
  40 #include "enic_vnic_wq.h"
  41
  42 #define RTE_PMD_USE_PREFETCH
  43
  44 #ifdef RTE_PMD_USE_PREFETCH
  45 /*Prefetch a cache line into all cache levels. */
  46 #define rte_enic_prefetch(p) rte_prefetch0(p)
  47 #else
  48 #define rte_enic_prefetch(p) do {} while (0)
  49 #endif
  50
  51 #ifdef RTE_PMD_PACKET_PREFETCH
  52 #define rte_packet_prefetch(p) rte_prefetch1(p)
  53 #else
  54 #define rte_packet_prefetch(p) do {} while (0)
  55 #endif
  56
  57 static inline uint16_t
  58 enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
  59 {
  60         return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
  61 }
  62
  63 static inline uint16_t
  64 enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
  65 {
  66         return le16_to_cpu(crd->bytes_written_flags) &
  67                            ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
  68 }
  69
  70 static inline uint8_t
  71 enic_cq_rx_desc_packet_error(uint16_t bwflags)
  72 {
  73         return (bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
  74                 CQ_ENET_RQ_DESC_FLAGS_TRUNCATED;
  75 }
  76
  77 static inline uint8_t
  78 enic_cq_rx_desc_eop(uint16_t ciflags)
  79 {
  80         return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
  81                 == CQ_ENET_RQ_DESC_FLAGS_EOP;
  82 }
  83
  84 static inline uint8_t
  85 enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
  86 {
  87         return (le16_to_cpu(cqrd->q_number_rss_type_flags) &
  88                 CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
  89                 CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC;
  90 }
  91
  92 static inline uint8_t
  93 enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
  94 {
  95         return (cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
  96                 CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK;
  97 }
  98
  99 static inline uint8_t
 100 enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
 101 {
 102         return (cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
 103                 CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK;
 104 }
 105
 106 static inline uint8_t
 107 enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
 108 {
 109         return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
 110                 CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
 111 }
 112
 113 static inline uint32_t
 114 enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
 115 {
 116         return le32_to_cpu(cqrd->rss_hash);
 117 }
 118
 119 static inline uint16_t
 120 enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
 121 {
 122         return le16_to_cpu(cqrd->vlan);
 123 }
 124
 125 static inline uint16_t
 126 enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
 127 {
 128         struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
 129         return le16_to_cpu(cqrd->bytes_written_flags) &
 130                 CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
 131 }
 132
 133 static inline uint8_t
 134 enic_cq_rx_check_err(struct cq_desc *cqd)
 135 {
 136         struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
 137         uint16_t bwflags;
 138
 139         bwflags = enic_cq_rx_desc_bwflags(cqrd);
 140         if (unlikely(enic_cq_rx_desc_packet_error(bwflags)))
 141                 return 1;
 142         return 0;
 143 }
 144
 145 /* Lookup table to translate RX CQ flags to mbuf flags. */
 146 static inline uint32_t
 147 enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
 148 {
 149         struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
 150         uint8_t cqrd_flags = cqrd->flags;
 151         static const uint32_t cq_type_table[128] __rte_cache_aligned = {
 152                 [32] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
 153                 [34] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
 154                         | RTE_PTYPE_L4_UDP,
 155                 [36] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
 156                         | RTE_PTYPE_L4_TCP,
 157                 [96] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
 158                         | RTE_PTYPE_L4_FRAG,
 159                 [16] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
 160                 [18] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
 161                         | RTE_PTYPE_L4_UDP,
 162                 [20] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
 163                         | RTE_PTYPE_L4_TCP,
 164                 [80] =  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
 165                         | RTE_PTYPE_L4_FRAG,
 166                 /* All others reserved */
 167         };
 168         cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
 169                 | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
 170                 | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
 171         return cq_type_table[cqrd_flags];
 172 }
 173
 174 static inline void
 175 enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
 176 {
 177         struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
 178         uint16_t ciflags, bwflags, pkt_flags = 0;
 179         ciflags = enic_cq_rx_desc_ciflags(cqrd);
 180         bwflags = enic_cq_rx_desc_bwflags(cqrd);
 181
 182         mbuf->ol_flags = 0;
 183
 184         /* flags are meaningless if !EOP */
 185         if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
 186                 goto mbuf_flags_done;
 187
 188         /* VLAN stripping */
 189         if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
 190                 pkt_flags |= PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED;
 191                 mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
 192         } else {
 193                 mbuf->vlan_tci = 0;
 194         }
 195
 196         /* RSS flag */
 197         if (enic_cq_rx_desc_rss_type(cqrd)) {
 198                 pkt_flags |= PKT_RX_RSS_HASH;
 199                 mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
 200         }
 201
 202         /* checksum flags */
 203         if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
 204                 (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
 205                 if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
 206                         pkt_flags |= PKT_RX_IP_CKSUM_BAD;
 207                 if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
 208                         if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
 209                                 pkt_flags |= PKT_RX_L4_CKSUM_BAD;
 210                 }
 211         }
 212
 213  mbuf_flags_done:
 214         mbuf->ol_flags = pkt_flags;
 215 }
 216
 217 static inline uint32_t
 218 enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
 219 {
 220         uint32_t d = i0 + i1;
 221         RTE_ASSERT(i0 < n_descriptors);
 222         RTE_ASSERT(i1 < n_descriptors);
 223         d -= (d >= n_descriptors) ? n_descriptors : 0;
 224         return d;
 225 }
 226
 227
 228 uint16_t
 229 enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 230                uint16_t nb_pkts)
 231 {
 232         struct vnic_rq *rq = rx_queue;
 233         struct enic *enic = vnic_dev_priv(rq->vdev);
 234         unsigned int rx_id;
 235         struct rte_mbuf *nmb, *rxmb;
 236         uint16_t nb_rx = 0, nb_err = 0;
 237         uint16_t nb_hold;
 238         struct vnic_cq *cq;
 239         volatile struct cq_desc *cqd_ptr;
 240         uint8_t color;
 241
 242         cq = &enic->cq[enic_cq_rq(enic, rq->index)];
 243         rx_id = cq->to_clean;           /* index of cqd, rqd, mbuf_table */
 244         cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
 245
 246         nb_hold = rq->rx_nb_hold;       /* mbufs held by software */
 247
 248         while (nb_rx < nb_pkts) {
 249                 volatile struct rq_enet_desc *rqd_ptr;
 250                 dma_addr_t dma_addr;
 251                 struct cq_desc cqd;
 252                 uint8_t packet_error;
 253
 254                 /* Check for pkts available */
 255                 color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
 256                         & CQ_DESC_COLOR_MASK;
 257                 if (color == cq->last_color)
 258                         break;
 259
 260                 /* Get the cq descriptor and rq pointer */
 261                 cqd = *cqd_ptr;
 262                 rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
 263
 264                 /* allocate a new mbuf */
 265                 nmb = rte_mbuf_raw_alloc(rq->mp);
 266                 if (nmb == NULL) {
 267                         rte_atomic64_inc(&enic->soft_stats.rx_nombuf);
 268                         break;
 269                 }
 270
 271                 /* A packet error means descriptor and data are untrusted */
 272                 packet_error = enic_cq_rx_check_err(&cqd);
 273
 274                 /* Get the mbuf to return and replace with one just allocated */
 275                 rxmb = rq->mbuf_ring[rx_id];
 276                 rq->mbuf_ring[rx_id] = nmb;
 277
 278                 /* Increment cqd, rqd, mbuf_table index */
 279                 rx_id++;
 280                 if (unlikely(rx_id == rq->ring.desc_count)) {
 281                         rx_id = 0;
 282                         cq->last_color = cq->last_color ? 0 : 1;
 283                 }
 284
 285                 /* Prefetch next mbuf & desc while processing current one */
 286                 cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
 287                 rte_enic_prefetch(cqd_ptr);
 288                 rte_enic_prefetch(rq->mbuf_ring[rx_id]);
 289                 rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
 290                                  + rx_id);
 291
 292                 /* Push descriptor for newly allocated mbuf */
 293                 dma_addr = (dma_addr_t)(nmb->buf_physaddr
 294                            + RTE_PKTMBUF_HEADROOM);
 295                 rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
 296                 rqd_ptr->length_type = cpu_to_le16(nmb->buf_len
 297                                        - RTE_PKTMBUF_HEADROOM);
 298
 299                 /* Drop incoming bad packet */
 300                 if (unlikely(packet_error)) {
 301                         rte_pktmbuf_free(rxmb);
 302                         nb_err++;
 303                         continue;
 304                 }
 305
 306                 /* Fill in the rest of the mbuf */
 307                 rxmb->data_off = RTE_PKTMBUF_HEADROOM;
 308                 rxmb->nb_segs = 1;
 309                 rxmb->next = NULL;
 310                 rxmb->port = enic->port_id;
 311                 rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
 312                 rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
 313                 enic_cq_rx_to_pkt_flags(&cqd, rxmb);
 314                 rxmb->data_len = rxmb->pkt_len;
 315
 316                 /* prefetch mbuf data for caller */
 317                 rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
 318                                     RTE_PKTMBUF_HEADROOM));
 319
 320                 /* store the mbuf address into the next entry of the array */
 321                 rx_pkts[nb_rx++] = rxmb;
 322         }
 323
 324         nb_hold += nb_rx + nb_err;
 325         cq->to_clean = rx_id;
 326
 327         if (nb_hold > rq->rx_free_thresh) {
 328                 rq->posted_index = enic_ring_add(rq->ring.desc_count,
 329                                 rq->posted_index, nb_hold);
 330                 nb_hold = 0;
 331                 rte_mb();
 332                 iowrite32(rq->posted_index, &rq->ctrl->posted_index);
 333         }
 334
 335         rq->rx_nb_hold = nb_hold;
 336
 337         return nb_rx;
 338 }
 339
 340 static void enic_wq_free_buf(struct vnic_wq *wq,
 341         __rte_unused struct cq_desc *cq_desc,
 342         struct vnic_wq_buf *buf,
 343         __rte_unused void *opaque)
 344 {
 345         enic_free_wq_buf(wq, buf);
 346 }
 347
 348 static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
 349         __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
 350 {
 351         struct enic *enic = vnic_dev_priv(vdev);
 352
 353         vnic_wq_service(&enic->wq[q_number], cq_desc,
 354                 completed_index, enic_wq_free_buf,
 355                 opaque);
 356
 357         return 0;
 358 }
 359
 360 unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq)
 361 {
 362         unsigned int cq = enic_cq_wq(enic, wq->index);
 363
 364         /* Return the work done */
 365         return vnic_cq_service(&enic->cq[cq],
 366                 -1 /*wq_work_to_do*/, enic_wq_service, NULL);
 367 }
 368
 369 void enic_post_wq_index(struct vnic_wq *wq)
 370 {
 371         enic_vnic_post_wq_index(wq);
 372 }
 373
 374 void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
 375                    struct rte_mbuf *tx_pkt, unsigned short len,
 376                    uint8_t sop, uint8_t eop, uint8_t cq_entry,
 377                    uint16_t ol_flags, uint16_t vlan_tag)
 378 {
 379         struct wq_enet_desc *desc = vnic_wq_next_desc(wq);
 380         uint16_t mss = 0;
 381         uint8_t vlan_tag_insert = 0;
 382         uint64_t bus_addr = (dma_addr_t)
 383             (tx_pkt->buf_physaddr + tx_pkt->data_off);
 384
 385         if (sop) {
 386                 if (ol_flags & PKT_TX_VLAN_PKT)
 387                         vlan_tag_insert = 1;
 388
 389                 if (enic->hw_ip_checksum) {
 390                         if (ol_flags & PKT_TX_IP_CKSUM)
 391                                 mss |= ENIC_CALC_IP_CKSUM;
 392
 393                         if (ol_flags & PKT_TX_TCP_UDP_CKSUM)
 394                                 mss |= ENIC_CALC_TCP_UDP_CKSUM;
 395                 }
 396         }
 397
 398         wq_enet_desc_enc(desc,
 399                 bus_addr,
 400                 len,
 401                 mss,
 402                 0 /* header_length */,
 403                 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */,
 404                 eop,
 405                 cq_entry,
 406                 0 /* fcoe_encap */,
 407                 vlan_tag_insert,
 408                 vlan_tag,
 409                 0 /* loopback */);
 410
 411         enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len,
 412                           sop,
 413                           1 /*desc_skip_cnt*/,
 414                           cq_entry,
 415                           0 /*compressed send*/,
 416                           0 /*wrid*/);
 417 }
 418
 419 uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 420         uint16_t nb_pkts)
 421 {
 422         uint16_t index;
 423         unsigned int frags;
 424         unsigned int pkt_len;
 425         unsigned int seg_len;
 426         unsigned int inc_len;
 427         unsigned int nb_segs;
 428         struct rte_mbuf *tx_pkt, *next_tx_pkt;
 429         struct vnic_wq *wq = (struct vnic_wq *)tx_queue;
 430         struct enic *enic = vnic_dev_priv(wq->vdev);
 431         unsigned short vlan_id;
 432         unsigned short ol_flags;
 433         uint8_t last_seg, eop;
 434         unsigned int host_tx_descs = 0;
 435
 436         for (index = 0; index < nb_pkts; index++) {
 437                 tx_pkt = *tx_pkts++;
 438                 inc_len = 0;
 439                 nb_segs = tx_pkt->nb_segs;
 440                 if (nb_segs > vnic_wq_desc_avail(wq)) {
 441                         if (index > 0)
 442                                 enic_post_wq_index(wq);
 443
 444                         /* wq cleanup and try again */
 445                         if (!enic_cleanup_wq(enic, wq) ||
 446                                 (nb_segs > vnic_wq_desc_avail(wq))) {
 447                                 return index;
 448                         }
 449                 }
 450
 451                 pkt_len = tx_pkt->pkt_len;
 452                 vlan_id = tx_pkt->vlan_tci;
 453                 ol_flags = tx_pkt->ol_flags;
 454                 for (frags = 0; inc_len < pkt_len; frags++) {
 455                         if (!tx_pkt)
 456                                 break;
 457                         next_tx_pkt = tx_pkt->next;
 458                         seg_len = tx_pkt->data_len;
 459                         inc_len += seg_len;
 460
 461                         host_tx_descs++;
 462                         last_seg = 0;
 463                         eop = 0;
 464                         if ((pkt_len == inc_len) || !next_tx_pkt) {
 465                                 eop = 1;
 466                                 /* post if last packet in batch or > thresh */
 467                                 if ((index == (nb_pkts - 1)) ||
 468                                    (host_tx_descs > ENIC_TX_POST_THRESH)) {
 469                                         last_seg = 1;
 470                                         host_tx_descs = 0;
 471                                 }
 472                         }
 473                         enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len,
 474                                       !frags, eop, last_seg, ol_flags, vlan_id);
 475                         tx_pkt = next_tx_pkt;
 476                 }
 477         }
 478
 479         enic_cleanup_wq(enic, wq);
 480         return index;
 481 }