X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rxtx.h;h=30dfaf3590ea88da24756799790fc61f1bbcf9d5;hb=3669a1af893088064778f14fba0c2ca88819e095;hp=c072169f2041384e4bd4d08f3b601e1a412a0976;hpb=2eefbec531c7abf4c9c668f7941b285858c06733;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index c072169f20..30dfaf3590 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -114,8 +114,7 @@ struct mlx5_rxq_data { unsigned int elts_n:4; /* Log 2 of Mbufs. */ unsigned int rss_hash:1; /* RSS hash result is enabled. */ unsigned int mark:1; /* Marked flow available on the queue. */ - unsigned int pending_err:1; /* CQE error needs to be handled. */ - unsigned int :14; /* Remaining bits. */ + unsigned int :15; /* Remaining bits. */ volatile uint32_t *rq_db; volatile uint32_t *cq_db; uint16_t port_id; @@ -201,12 +200,12 @@ struct mlx5_txq_data { uint16_t inline_max_packet_sz; /* Max packet size for inlining. */ uint16_t mr_cache_idx; /* Index of last hit entry. */ uint32_t qp_num_8s; /* QP number shifted by 8. */ - uint32_t flags; /* Flags for Tx Queue. */ + uint64_t offloads; /* Offloads for Tx Queue. */ volatile struct mlx5_cqe (*cqes)[]; /* Completion queue. */ volatile void *wqes; /* Work queue (use volatile to write into). */ volatile uint32_t *qp_db; /* Work queue doorbell. */ volatile uint32_t *cq_db; /* Completion queue doorbell. */ - volatile void *bf_reg; /* Blueflame register. */ + volatile void *bf_reg; /* Blueflame register remapped. */ struct mlx5_mr *mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MR translation table. */ struct rte_mbuf *(*elts)[]; /* TX elements. */ struct mlx5_txq_stats stats; /* TX queue counters. */ @@ -231,6 +230,7 @@ struct mlx5_txq_ctrl { struct mlx5_txq_ibv *ibv; /* Verbs queue object. */ struct mlx5_txq_data txq; /* Data path structure. */ off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */ + volatile void *bf_reg_orig; /* Blueflame register from verbs. */ }; /* mlx5_rxq.c */ @@ -253,6 +253,7 @@ int mlx5_priv_rxq_ibv_releasable(struct priv *, struct mlx5_rxq_ibv *); int mlx5_priv_rxq_ibv_verify(struct priv *); struct mlx5_rxq_ctrl *mlx5_priv_rxq_new(struct priv *, uint16_t, uint16_t, unsigned int, + const struct rte_eth_rxconf *, struct rte_mempool *); struct mlx5_rxq_ctrl *mlx5_priv_rxq_get(struct priv *, uint16_t); int mlx5_priv_rxq_release(struct priv *, uint16_t); @@ -273,6 +274,8 @@ struct mlx5_hrxq *mlx5_priv_hrxq_get(struct priv *, uint8_t *, uint8_t, uint64_t, uint16_t [], uint16_t); int mlx5_priv_hrxq_release(struct priv *, struct mlx5_hrxq *); int mlx5_priv_hrxq_ibv_verify(struct priv *); +uint64_t mlx5_priv_get_rx_port_offloads(struct priv *); +uint64_t mlx5_priv_get_rx_queue_offloads(struct priv *); /* mlx5_txq.c */ @@ -293,6 +296,7 @@ int mlx5_priv_txq_release(struct priv *, uint16_t); int mlx5_priv_txq_releasable(struct priv *, uint16_t); int mlx5_priv_txq_verify(struct priv *); void txq_alloc_elts(struct mlx5_txq_ctrl *); +uint64_t mlx5_priv_get_tx_port_offloads(struct priv *); /* mlx5_rxtx.c */ @@ -310,8 +314,8 @@ int mlx5_rx_descriptor_status(void *, uint16_t); int mlx5_tx_descriptor_status(void *, uint16_t); /* Vectorized version of mlx5_rxtx.c */ -int priv_check_raw_vec_tx_support(struct priv *); -int priv_check_vec_tx_support(struct priv *); +int priv_check_raw_vec_tx_support(struct priv *, struct rte_eth_dev *); +int priv_check_vec_tx_support(struct priv *, struct rte_eth_dev *); int rxq_check_vec_support(struct mlx5_rxq_data *); int priv_check_vec_rx_support(struct priv *); uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t); @@ -545,27 +549,25 @@ static __rte_always_inline uint32_t mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) { uint16_t i = txq->mr_cache_idx; - uintptr_t addr = rte_pktmbuf_mtod_offset(mb, uintptr_t, DATA_LEN(mb)); + uintptr_t addr = rte_pktmbuf_mtod(mb, uintptr_t); struct mlx5_mr *mr; assert(i < RTE_DIM(txq->mp2mr)); - if (likely(txq->mp2mr[i]->start <= addr && txq->mp2mr[i]->end >= addr)) + if (likely(txq->mp2mr[i]->start <= addr && txq->mp2mr[i]->end > addr)) return txq->mp2mr[i]->lkey; for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) { - if (unlikely(txq->mp2mr[i]->mr == NULL)) { + if (unlikely(txq->mp2mr[i] == NULL || + txq->mp2mr[i]->mr == NULL)) { /* Unknown MP, add a new MR for it. */ break; } if (txq->mp2mr[i]->start <= addr && - txq->mp2mr[i]->end >= addr) { + txq->mp2mr[i]->end > addr) { assert(txq->mp2mr[i]->lkey != (uint32_t)-1); - assert(rte_cpu_to_be_32(txq->mp2mr[i]->mr->lkey) == - txq->mp2mr[i]->lkey); txq->mr_cache_idx = i; return txq->mp2mr[i]->lkey; } } - txq->mr_cache_idx = 0; mr = mlx5_txq_mp2mr_reg(txq, mlx5_tx_mb2mp(mb), i); /* * Request the reference to use in this queue, the original one is @@ -573,7 +575,13 @@ mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) */ if (mr) { rte_atomic32_inc(&mr->refcnt); + txq->mr_cache_idx = i >= RTE_DIM(txq->mp2mr) ? i - 1 : i; return mr->lkey; + } else { + struct rte_mempool *mp = mlx5_tx_mb2mp(mb); + + WARN("Failed to register mempool 0x%p(%s)", + (void *)mp, mp->name); } return (uint32_t)-1; } @@ -595,7 +603,7 @@ mlx5_tx_dbrec_cond_wmb(struct mlx5_txq_data *txq, volatile struct mlx5_wqe *wqe, uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg); volatile uint64_t *src = ((volatile uint64_t *)wqe); - rte_io_wmb(); + rte_cio_wmb(); *txq->qp_db = rte_cpu_to_be_32(txq->wqe_ci); /* Ensure ordering between DB record and BF copy. */ rte_wmb(); @@ -636,7 +644,8 @@ txq_ol_cksum_to_cs(struct mlx5_txq_data *txq_data, struct rte_mbuf *buf) /* Should we enable HW CKSUM offload */ if (buf->ol_flags & - (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { + (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | + PKT_TX_OUTER_IP_CKSUM)) { if (txq_data->tunnel_en && (buf->ol_flags & (PKT_TX_TUNNEL_GRE | PKT_TX_TUNNEL_VXLAN))) { @@ -652,4 +661,54 @@ txq_ol_cksum_to_cs(struct mlx5_txq_data *txq_data, struct rte_mbuf *buf) return cs_flags; } +/** + * Count the number of contiguous single segment packets. + * + * @param pkts + * Pointer to array of packets. + * @param pkts_n + * Number of packets. + * + * @return + * Number of contiguous single segment packets. + */ +static __rte_always_inline unsigned int +txq_count_contig_single_seg(struct rte_mbuf **pkts, uint16_t pkts_n) +{ + unsigned int pos; + + if (!pkts_n) + return 0; + /* Count the number of contiguous single segment packets. */ + for (pos = 0; pos < pkts_n; ++pos) + if (NB_SEGS(pkts[pos]) > 1) + break; + return pos; +} + +/** + * Count the number of contiguous multi-segment packets. + * + * @param pkts + * Pointer to array of packets. + * @param pkts_n + * Number of packets. + * + * @return + * Number of contiguous multi-segment packets. + */ +static __rte_always_inline unsigned int +txq_count_contig_multi_seg(struct rte_mbuf **pkts, uint16_t pkts_n) +{ + unsigned int pos; + + if (!pkts_n) + return 0; + /* Count the number of contiguous multi-segment packets. */ + for (pos = 0; pos < pkts_n; ++pos) + if (NB_SEGS(pkts[pos]) == 1) + break; + return pos; +} + #endif /* RTE_PMD_MLX5_RXTX_H_ */