X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fcxgbe%2Fsge.c;h=d088065e53140ac405a4af8cac8659d319341f20;hb=740feaf349b10ca166aaaa77aa903368e1897f16;hp=1a10278f24e454dc8252b4907daf15590f709c8c;hpb=e7df2fda1fec3fa598d77cff6f33b3f64c48a40f;p=dpdk.git diff --git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c index 1a10278f24..d088065e53 100644 --- a/drivers/net/cxgbe/sge.c +++ b/drivers/net/cxgbe/sge.c @@ -31,7 +31,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include #include #include #include @@ -98,7 +97,8 @@ static inline unsigned int fl_mtu_bufsize(struct adapter *adapter, { struct sge *s = &adapter->sge; - return ALIGN(s->pktshift + ETH_HLEN + VLAN_HLEN + mtu, s->fl_align); + return CXGBE_ALIGN(s->pktshift + ETHER_HDR_LEN + VLAN_HLEN + mtu, + s->fl_align); } #define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL) @@ -199,11 +199,20 @@ static void free_tx_desc(struct sge_txq *q, unsigned int n) static void reclaim_tx_desc(struct sge_txq *q, unsigned int n) { + struct tx_sw_desc *d; unsigned int cidx = q->cidx; + d = &q->sdesc[cidx]; while (n--) { - if (++cidx == q->size) + if (d->mbuf) { /* an SGL is present */ + rte_pktmbuf_free(d->mbuf); + d->mbuf = NULL; + } + ++d; + if (++cidx == q->size) { cidx = 0; + d = q->sdesc; + } } q->cidx = cidx; } @@ -238,6 +247,29 @@ static inline bool fl_starving(const struct adapter *adapter, return fl->avail - fl->pend_cred <= s->fl_starve_thres; } +static inline unsigned int get_buf_size(struct adapter *adapter, + const struct rx_sw_desc *d) +{ + unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE; + unsigned int buf_size = 0; + + switch (rx_buf_size_idx) { + case RX_SMALL_MTU_BUF: + buf_size = FL_MTU_SMALL_BUFSIZE(adapter); + break; + + case RX_LARGE_MTU_BUF: + buf_size = FL_MTU_LARGE_BUFSIZE(adapter); + break; + + default: + BUG_ON(1); + /* NOT REACHED */ + } + + return buf_size; +} + /** * free_rx_bufs - free the Rx buffers on an SGE free list * @q: the SGE free list to free buffers from @@ -286,8 +318,7 @@ static void unmap_rx_buf(struct sge_fl *q) static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) { - /* see if we have exceeded q->size / 4 */ - if (q->pend_cred >= (q->size / 4)) { + if (q->pend_cred >= 64) { u32 val = adap->params.arch.sge_fl_db; if (is_t4(adap->params.chip)) @@ -307,12 +338,12 @@ static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) * mechanism. */ if (unlikely(!q->bar2_addr)) { - t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL), - val | V_QID(q->cntxt_id)); + t4_write_reg_relaxed(adap, MYPF_REG(A_SGE_PF_KDOORBELL), + val | V_QID(q->cntxt_id)); } else { - writel(val | V_QID(q->bar2_qid), - (void *)((uintptr_t)q->bar2_addr + - SGE_UDB_KDOORBELL)); + writel_relaxed(val | V_QID(q->bar2_qid), + (void *)((uintptr_t)q->bar2_addr + + SGE_UDB_KDOORBELL)); /* * This Write memory Barrier will force the write to @@ -354,6 +385,14 @@ static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, unsigned int buf_size_idx = RX_SMALL_MTU_BUF; struct rte_mbuf *buf_bulk[n]; int ret, i; + struct rte_pktmbuf_pool_private *mbp_priv; + u8 jumbo_en = rxq->rspq.eth_dev->data->dev_conf.rxmode.jumbo_frame; + + /* Use jumbo mtu buffers iff mbuf data room size can fit jumbo data. */ + mbp_priv = rte_mempool_get_priv(rxq->rspq.mb_pool); + if (jumbo_en && + ((mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM) >= 9000)) + buf_size_idx = RX_LARGE_MTU_BUF; ret = rte_mempool_get_bulk(rxq->rspq.mb_pool, (void *)buf_bulk, n); if (unlikely(ret != 0)) { @@ -381,7 +420,9 @@ static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, mbuf->nb_segs = 1; mbuf->port = rxq->rspq.port_id; - mapping = (dma_addr_t)(mbuf->buf_physaddr + mbuf->data_off); + mapping = (dma_addr_t)RTE_ALIGN(mbuf->buf_physaddr + + mbuf->data_off, + adap->sge.fl_align); mapping |= buf_size_idx; *d++ = cpu_to_be64(mapping); set_rx_sw_desc(sd, mbuf, mapping); @@ -642,6 +683,10 @@ static void write_sgl(struct rte_mbuf *mbuf, struct sge_txq *q, #define Q_IDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->size) #define R_IDXDIFF(q, idx) IDXDIFF((q)->cidx, (q)->idx, (q)->size) +#define PIDXDIFF(head, tail, wrap) \ + ((tail) >= (head) ? (tail) - (head) : (wrap) - (head) + (tail)) +#define P_IDXDIFF(q, idx) PIDXDIFF((q)->cidx, idx, (q)->size) + /** * ring_tx_db - ring a Tx queue's doorbell * @adap: the adapter @@ -732,7 +777,7 @@ static u64 hwcsum(enum chip_type chip, const struct rte_mbuf *m) } if (likely(csum_type >= TX_CSUM_TCPIP)) { - int hdr_len = V_TXPKT_IPHDR_LEN(m->l3_len); + u64 hdr_len = V_TXPKT_IPHDR_LEN(m->l3_len); int eth_hdr_len = m->l2_len; if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) @@ -769,20 +814,23 @@ static void tx_timer_cb(void *data) struct adapter *adap = (struct adapter *)data; struct sge_eth_txq *txq = &adap->sge.ethtxq[0]; int i; + unsigned int coal_idx; /* monitor any pending tx */ for (i = 0; i < adap->sge.max_ethqsets; i++, txq++) { - t4_os_lock(&txq->txq_lock); - if (txq->q.coalesce.idx) { - if (txq->q.coalesce.idx == txq->q.last_coal_idx && - txq->q.pidx == txq->q.last_pidx) { - ship_tx_pkt_coalesce_wr(adap, txq); - } else { - txq->q.last_coal_idx = txq->q.coalesce.idx; - txq->q.last_pidx = txq->q.pidx; + if (t4_os_trylock(&txq->txq_lock)) { + coal_idx = txq->q.coalesce.idx; + if (coal_idx) { + if (coal_idx == txq->q.last_coal_idx && + txq->q.pidx == txq->q.last_pidx) { + ship_tx_pkt_coalesce_wr(adap, txq); + } else { + txq->q.last_coal_idx = coal_idx; + txq->q.last_pidx = txq->q.pidx; + } } + t4_os_unlock(&txq->txq_lock); } - t4_os_unlock(&txq->txq_lock); } rte_eal_alarm_set(50, tx_timer_cb, (void *)adap); } @@ -804,7 +852,7 @@ static inline void ship_tx_pkt_coalesce_wr(struct adapter *adap, /* fill the pkts WR header */ wr = (void *)&q->desc[q->pidx]; - wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS_WR)); + wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR)); wr_mid = V_FW_WR_LEN16(DIV_ROUND_UP(q->coalesce.flits, 2)); ndesc = flits_to_desc(q->coalesce.flits); @@ -848,15 +896,11 @@ static inline int should_tx_packet_coalesce(struct sge_eth_txq *txq, struct sge_txq *q = &txq->q; unsigned int flits, ndesc; unsigned char type = 0; - int credits, hw_cidx = ntohs(q->stat->cidx); - int in_use = q->pidx - hw_cidx + flits_to_desc(q->coalesce.flits); + int credits; /* use coal WR type 1 when no frags are present */ type = (mbuf->nb_segs == 1) ? 1 : 0; - if (in_use < 0) - in_use += q->size; - if (unlikely(type != q->coalesce.type && q->coalesce.idx)) ship_tx_pkt_coalesce_wr(adap, txq); @@ -931,7 +975,7 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, struct rte_mbuf *mbuf, int flits, struct adapter *adap, const struct port_info *pi, - dma_addr_t *addr) + dma_addr_t *addr, uint16_t nb_pkts) { u64 cntrl, *end; struct sge_txq *q = &txq->q; @@ -941,6 +985,10 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, struct tx_sw_desc *sd; unsigned int idx = q->coalesce.idx, len = mbuf->pkt_len; +#ifdef RTE_LIBRTE_CXGBE_TPUT + RTE_SET_USED(nb_pkts); +#endif + if (q->coalesce.type == 0) { mc = (struct ulp_txpkt *)q->coalesce.ptr; mc->cmd_dest = htonl(V_ULPTX_CMD(4) | V_ULP_TXPKT_DEST(0) | @@ -1010,7 +1058,11 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, sd->coalesce.idx = (idx & 1) + 1; /* send the coaelsced work request if max reached */ - if (++q->coalesce.idx == ETH_COALESCE_PKT_NUM) + if (++q->coalesce.idx == ETH_COALESCE_PKT_NUM +#ifndef RTE_LIBRTE_CXGBE_TPUT + || q->coalesce.idx >= nb_pkts +#endif + ) ship_tx_pkt_coalesce_wr(adap, txq); return 0; } @@ -1022,7 +1074,8 @@ static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq, * * Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled. */ -int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf) +int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf, + uint16_t nb_pkts) { const struct port_info *pi; struct cpl_tx_pkt_lso_core *lso; @@ -1039,6 +1092,7 @@ int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf) u32 wr_mid; u64 cntrl, *end; bool v6; + u32 max_pkt_len = txq->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; /* Reject xmit if queue is stopped */ if (unlikely(txq->flags & EQ_STOPPED)) @@ -1054,7 +1108,10 @@ out_free: return 0; } - rte_prefetch0(&((&txq->q)->sdesc->mbuf->pool)); + if ((!(m->ol_flags & PKT_TX_TCP_SEG)) && + (unlikely(m->pkt_len > max_pkt_len))) + goto out_free; + pi = (struct port_info *)txq->eth_dev->data->dev_private; adap = pi->adapter; @@ -1062,7 +1119,7 @@ out_free: /* align the end of coalesce WR to a 512 byte boundary */ txq->q.coalesce.max = (8 - (txq->q.pidx & 7)) * 8; - if (!(m->ol_flags & PKT_TX_TCP_SEG)) { + if (!((m->ol_flags & PKT_TX_TCP_SEG) || (m->pkt_len > ETHER_MAX_LEN))) { if (should_tx_packet_coalesce(txq, mbuf, &cflits, adap)) { if (unlikely(map_mbuf(mbuf, addr) < 0)) { dev_warn(adap, "%s: mapping err for coalesce\n", @@ -1070,8 +1127,9 @@ out_free: txq->stats.mapping_err++; goto out_free; } + rte_prefetch0((volatile void *)addr); return tx_do_packet_coalesce(txq, mbuf, cflits, adap, - pi, addr); + pi, addr, nb_pkts); } else { return -EBUSY; } @@ -1108,33 +1166,52 @@ out_free: len = 0; len += sizeof(*cpl); - lso = (void *)(wr + 1); - v6 = (m->ol_flags & PKT_TX_IPV6) != 0; - l3hdr_len = m->l3_len; - l4hdr_len = m->l4_len; - eth_xtra_len = m->l2_len - ETHER_HDR_LEN; - len += sizeof(*lso); - wr->op_immdlen = htonl(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | - V_FW_WR_IMMDLEN(len)); - lso->lso_ctrl = htonl(V_LSO_OPCODE(CPL_TX_PKT_LSO) | - F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | - V_LSO_IPV6(v6) | - V_LSO_ETHHDR_LEN(eth_xtra_len / 4) | - V_LSO_IPHDR_LEN(l3hdr_len / 4) | - V_LSO_TCPHDR_LEN(l4hdr_len / 4)); - lso->ipid_ofst = htons(0); - lso->mss = htons(m->tso_segsz); - lso->seqno_offset = htonl(0); - if (is_t4(adap->params.chip)) - lso->len = htonl(m->pkt_len); - else - lso->len = htonl(V_LSO_T5_XFER_SIZE(m->pkt_len)); - cpl = (void *)(lso + 1); - cntrl = V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | - V_TXPKT_IPHDR_LEN(l3hdr_len) | - V_TXPKT_ETHHDR_LEN(eth_xtra_len); - txq->stats.tso++; - txq->stats.tx_cso += m->tso_segsz; + + /* Coalescing skipped and we send through normal path */ + if (!(m->ol_flags & PKT_TX_TCP_SEG)) { + wr->op_immdlen = htonl(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | + V_FW_WR_IMMDLEN(len)); + cpl = (void *)(wr + 1); + if (m->ol_flags & PKT_TX_IP_CKSUM) { + cntrl = hwcsum(adap->params.chip, m) | + F_TXPKT_IPCSUM_DIS; + txq->stats.tx_cso++; + } + } else { + lso = (void *)(wr + 1); + v6 = (m->ol_flags & PKT_TX_IPV6) != 0; + l3hdr_len = m->l3_len; + l4hdr_len = m->l4_len; + eth_xtra_len = m->l2_len - ETHER_HDR_LEN; + len += sizeof(*lso); + wr->op_immdlen = htonl(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | + V_FW_WR_IMMDLEN(len)); + lso->lso_ctrl = htonl(V_LSO_OPCODE(CPL_TX_PKT_LSO) | + F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | + V_LSO_IPV6(v6) | + V_LSO_ETHHDR_LEN(eth_xtra_len / 4) | + V_LSO_IPHDR_LEN(l3hdr_len / 4) | + V_LSO_TCPHDR_LEN(l4hdr_len / 4)); + lso->ipid_ofst = htons(0); + lso->mss = htons(m->tso_segsz); + lso->seqno_offset = htonl(0); + if (is_t4(adap->params.chip)) + lso->len = htonl(m->pkt_len); + else + lso->len = htonl(V_LSO_T5_XFER_SIZE(m->pkt_len)); + cpl = (void *)(lso + 1); + + if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) + cntrl = V_TXPKT_ETHHDR_LEN(eth_xtra_len); + else + cntrl = V_T6_TXPKT_ETHHDR_LEN(eth_xtra_len); + + cntrl |= V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : + TX_CSUM_TCPIP) | + V_TXPKT_IPHDR_LEN(l3hdr_len); + txq->stats.tso++; + txq->stats.tx_cso += m->tso_segsz; + } if (m->ol_flags & PKT_TX_VLAN_PKT) { txq->stats.vlan_ins++; @@ -1155,9 +1232,14 @@ out_free: last_desc -= txq->q.size; d = &txq->q.sdesc[last_desc]; - if (d->mbuf) { - rte_pktmbuf_free(d->mbuf); - d->mbuf = NULL; + if (d->coalesce.idx) { + int i; + + for (i = 0; i < d->coalesce.idx; i++) { + rte_pktmbuf_free(d->coalesce.mbuf[i]); + d->coalesce.mbuf[i] = NULL; + } + d->coalesce.idx = 0; } write_sgl(m, &txq->q, (struct ulptx_sgl *)(cpl + 1), end, 0, addr); @@ -1267,9 +1349,6 @@ static struct rte_mbuf *t4_pktgl_to_mbuf(const struct pkt_gl *gl) return t4_pktgl_to_mbuf_usembufs(gl); } -#define RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb) \ - ((dma_addr_t) ((mb)->buf_physaddr + (mb)->data_off)) - /** * t4_ethrx_handler - process an ingress ethernet packet * @q: the response queue that received the packet @@ -1286,10 +1365,16 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, const struct rss_header *rss_hdr; bool csum_ok; struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + u16 err_vec; rss_hdr = (const void *)rsp; pkt = (const void *)&rsp[1]; - csum_ok = pkt->csum_calc && !pkt->err_vec; + /* Compressed error vector is enabled for T6 only */ + if (q->adapter->params.tp.rx_pkt_encap) + err_vec = G_T6_COMPR_RXERR_VEC(ntohs(pkt->err_vec)); + else + err_vec = ntohs(pkt->err_vec); + csum_ok = pkt->csum_calc && !err_vec; mbuf = t4_pktgl_to_mbuf(si); if (unlikely(!mbuf)) { @@ -1299,22 +1384,14 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, mbuf->port = pkt->iff; if (pkt->l2info & htonl(F_RXF_IP)) { -#ifdef RTE_NEXT_ABI mbuf->packet_type = RTE_PTYPE_L3_IPV4; -#else - mbuf->ol_flags |= PKT_RX_IPV4_HDR; -#endif if (unlikely(!csum_ok)) mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; if ((pkt->l2info & htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok) mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; } else if (pkt->l2info & htonl(F_RXF_IP6)) { -#ifdef RTE_NEXT_ABI mbuf->packet_type = RTE_PTYPE_L3_IPV6; -#else - mbuf->ol_flags |= PKT_RX_IPV6_HDR; -#endif } mbuf->port = pkt->iff; @@ -1334,20 +1411,6 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, return 0; } -/** - * is_new_response - check if a response is newly written - * @r: the response descriptor - * @q: the response queue - * - * Returns true if a response descriptor contains a yet unprocessed - * response. - */ -static inline bool is_new_response(const struct rsp_ctrl *r, - const struct sge_rspq *q) -{ - return (r->u.type_gen >> S_RSPD_GEN) == q->gen; -} - #define CXGB4_MSG_AN ((void *)1) /** @@ -1389,12 +1452,12 @@ static int process_responses(struct sge_rspq *q, int budget, struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); while (likely(budget_left)) { + if (q->cidx == ntohs(q->stat->pidx)) + break; + rc = (const struct rsp_ctrl *) ((const char *)q->cur_desc + (q->iqe_len - sizeof(*rc))); - if (!is_new_response(rc, q)) - break; - /* * Ensure response has been read */ @@ -1402,54 +1465,101 @@ static int process_responses(struct sge_rspq *q, int budget, rsp_type = G_RSPD_TYPE(rc->u.type_gen); if (likely(rsp_type == X_RSPD_TYPE_FLBUF)) { - const struct rx_sw_desc *rsd = - &rxq->fl.sdesc[rxq->fl.cidx]; - const struct rss_header *rss_hdr = - (const void *)q->cur_desc; - const struct cpl_rx_pkt *cpl = - (const void *)&q->cur_desc[1]; - bool csum_ok = cpl->csum_calc && !cpl->err_vec; - struct rte_mbuf *pkt; - u32 len = ntohl(rc->pldbuflen_qid); - - BUG_ON(!(len & F_RSPD_NEWBUF)); - pkt = rsd->buf; - pkt->data_len = G_RSPD_LEN(len); - pkt->pkt_len = pkt->data_len; - unmap_rx_buf(&rxq->fl); - - if (cpl->l2info & htonl(F_RXF_IP)) { -#ifdef RTE_NEXT_ABI - pkt->packet_type = RTE_PTYPE_L3_IPV4; -#else - pkt->ol_flags |= PKT_RX_IPV4_HDR; -#endif - if (unlikely(!csum_ok)) - pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; - - if ((cpl->l2info & - htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok) - pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; - } else if (cpl->l2info & htonl(F_RXF_IP6)) { -#ifdef RTE_NEXT_ABI - pkt->packet_type = RTE_PTYPE_L3_IPV6; -#else - pkt->ol_flags |= PKT_RX_IPV6_HDR; -#endif - } - - if (!rss_hdr->filter_tid && rss_hdr->hash_type) { - pkt->ol_flags |= PKT_RX_RSS_HASH; - pkt->hash.rss = ntohl(rss_hdr->hash_val); - } - - if (cpl->vlan_ex) { - pkt->ol_flags |= PKT_RX_VLAN_PKT; - pkt->vlan_tci = ntohs(cpl->vlan); + unsigned int stat_pidx; + int stat_pidx_diff; + + stat_pidx = ntohs(q->stat->pidx); + stat_pidx_diff = P_IDXDIFF(q, stat_pidx); + while (stat_pidx_diff && budget_left) { + const struct rx_sw_desc *rsd = + &rxq->fl.sdesc[rxq->fl.cidx]; + const struct rss_header *rss_hdr = + (const void *)q->cur_desc; + const struct cpl_rx_pkt *cpl = + (const void *)&q->cur_desc[1]; + struct rte_mbuf *pkt, *npkt; + u32 len, bufsz; + bool csum_ok; + u16 err_vec; + + rc = (const struct rsp_ctrl *) + ((const char *)q->cur_desc + + (q->iqe_len - sizeof(*rc))); + + rsp_type = G_RSPD_TYPE(rc->u.type_gen); + if (unlikely(rsp_type != X_RSPD_TYPE_FLBUF)) + break; + + len = ntohl(rc->pldbuflen_qid); + BUG_ON(!(len & F_RSPD_NEWBUF)); + pkt = rsd->buf; + npkt = pkt; + len = G_RSPD_LEN(len); + pkt->pkt_len = len; + + /* Compressed error vector is enabled for + * T6 only + */ + if (q->adapter->params.tp.rx_pkt_encap) + err_vec = G_T6_COMPR_RXERR_VEC( + ntohs(cpl->err_vec)); + else + err_vec = ntohs(cpl->err_vec); + csum_ok = cpl->csum_calc && !err_vec; + + /* Chain mbufs into len if necessary */ + while (len) { + struct rte_mbuf *new_pkt = rsd->buf; + + bufsz = min(get_buf_size(q->adapter, + rsd), len); + new_pkt->data_len = bufsz; + unmap_rx_buf(&rxq->fl); + len -= bufsz; + npkt->next = new_pkt; + npkt = new_pkt; + pkt->nb_segs++; + rsd = &rxq->fl.sdesc[rxq->fl.cidx]; + } + npkt->next = NULL; + pkt->nb_segs--; + + if (cpl->l2info & htonl(F_RXF_IP)) { + pkt->packet_type = RTE_PTYPE_L3_IPV4; + if (unlikely(!csum_ok)) + pkt->ol_flags |= + PKT_RX_IP_CKSUM_BAD; + + if ((cpl->l2info & + htonl(F_RXF_UDP | F_RXF_TCP)) && + !csum_ok) + pkt->ol_flags |= + PKT_RX_L4_CKSUM_BAD; + } else if (cpl->l2info & htonl(F_RXF_IP6)) { + pkt->packet_type = RTE_PTYPE_L3_IPV6; + } + + if (!rss_hdr->filter_tid && + rss_hdr->hash_type) { + pkt->ol_flags |= PKT_RX_RSS_HASH; + pkt->hash.rss = + ntohl(rss_hdr->hash_val); + } + + if (cpl->vlan_ex) { + pkt->ol_flags |= PKT_RX_VLAN_PKT; + pkt->vlan_tci = ntohs(cpl->vlan); + } + + rxq->stats.pkts++; + rxq->stats.rx_bytes += pkt->pkt_len; + rx_pkts[budget - budget_left] = pkt; + + rspq_next(q); + budget_left--; + stat_pidx_diff--; } - rxq->stats.pkts++; - rxq->stats.rx_bytes += pkt->pkt_len; - rx_pkts[budget - budget_left] = pkt; + continue; } else if (likely(rsp_type == X_RSPD_TYPE_CPL)) { ret = q->handler(q, q->cur_desc, NULL); } else { @@ -1464,34 +1574,6 @@ static int process_responses(struct sge_rspq *q, int budget, rspq_next(q); budget_left--; - - if (R_IDXDIFF(q, gts_idx) >= 64) { - unsigned int cidx_inc = R_IDXDIFF(q, gts_idx); - unsigned int params; - u32 val; - - __refill_fl(q->adapter, &rxq->fl); - params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX); - q->next_intr_params = params; - val = V_CIDXINC(cidx_inc) | V_SEINTARM(params); - - if (unlikely(!q->bar2_addr)) - t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), - val | - V_INGRESSQID((u32)q->cntxt_id)); - else { - writel(val | V_INGRESSQID(q->bar2_qid), - (void *)((uintptr_t)q->bar2_addr + - SGE_UDB_GTS)); - /* - * This Write memory Barrier will force the - * write to the User Doorbell area to be - * flushed. - */ - wmb(); - } - q->gts_idx = q->cidx; - } } /* @@ -1509,10 +1591,38 @@ static int process_responses(struct sge_rspq *q, int budget, int cxgbe_poll(struct sge_rspq *q, struct rte_mbuf **rx_pkts, unsigned int budget, unsigned int *work_done) { - int err = 0; + struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); + unsigned int cidx_inc; + unsigned int params; + u32 val; *work_done = process_responses(q, budget, rx_pkts); - return err; + + if (*work_done) { + cidx_inc = R_IDXDIFF(q, gts_idx); + + if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 64) + __refill_fl(q->adapter, &rxq->fl); + + params = q->intr_params; + q->next_intr_params = params; + val = V_CIDXINC(cidx_inc) | V_SEINTARM(params); + + if (unlikely(!q->bar2_addr)) { + t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), + val | V_INGRESSQID((u32)q->cntxt_id)); + } else { + writel(val | V_INGRESSQID(q->bar2_qid), + (void *)((uintptr_t)q->bar2_addr + SGE_UDB_GTS)); + /* This Write memory Barrier will force the + * write to the User Doorbell area to be + * flushed. + */ + wmb(); + } + q->gts_idx = q->cidx; + } + return 0; } /** @@ -1578,10 +1688,11 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, unsigned int nb_refill; /* Size needs to be multiple of 16, including status entry. */ - iq->size = roundup(iq->size, 16); + iq->size = cxgbe_roundup(iq->size, 16); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, fwevtq ? "fwq_ring" : "rx_ring", + eth_dev->device->driver->name, + fwevtq ? "fwq_ring" : "rx_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1601,24 +1712,25 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, V_FW_IQ_CMD_IQASYNCH(fwevtq) | V_FW_IQ_CMD_VIID(pi->viid) | V_FW_IQ_CMD_IQANDST(intr_idx < 0) | - V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT) | + V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_STATUS_PAGE) | V_FW_IQ_CMD_IQANDSTINDEX(intr_idx >= 0 ? intr_idx : -intr_idx - 1)); c.iqdroprss_to_iqesize = - htons(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) | + htons(V_FW_IQ_CMD_IQPCIECH(cong > 0 ? cxgbe_ffs(cong) - 1 : + pi->tx_chan) | F_FW_IQ_CMD_IQGTSMODE | V_FW_IQ_CMD_IQINTCNTTHRESH(iq->pktcnt_idx) | V_FW_IQ_CMD_IQESIZE(ilog2(iq->iqe_len) - 4)); c.iqsize = htons(iq->size); c.iqaddr = cpu_to_be64(iq->phys_addr); if (cong >= 0) - c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN); + c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN | + F_FW_IQ_CMD_IQRO); if (fl) { struct sge_eth_rxq *rxq = container_of(fl, struct sge_eth_rxq, fl); - enum chip_type chip = (enum chip_type)CHELSIO_CHIP_VERSION( - adap->params.chip); + unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip); /* * Allocate the ring for the hardware free list (with space @@ -1630,10 +1742,10 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, */ if (fl->size < s->fl_starve_thres - 1 + 2 * 8) fl->size = s->fl_starve_thres - 1 + 2 * 8; - fl->size = roundup(fl->size, 8); + fl->size = cxgbe_roundup(fl->size, 8); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, + eth_dev->device->driver->name, fwevtq ? "fwq_ring" : "fl_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1664,9 +1776,12 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, * Hence maximum allowed burst size will be 448 bytes. */ c.fl0dcaen_to_fl0cidxfthresh = - htons(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_128B) | - V_FW_IQ_CMD_FL0FBMAX((chip <= CHELSIO_T5) ? - X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); + htons(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 ? + X_FETCHBURSTMIN_128B : + X_FETCHBURSTMIN_64B) | + V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 ? + X_FETCHBURSTMAX_512B : + X_FETCHBURSTMAX_256B)); c.fl0size = htons(flsz); c.fl0addr = cpu_to_be64(fl->addr); } @@ -1685,6 +1800,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, iq->bar2_addr = bar2_address(adap, iq->cntxt_id, T4_BAR2_QTYPE_INGRESS, &iq->bar2_qid); iq->size--; /* subtract status entry */ + iq->stat = (void *)&iq->desc[iq->size * 8]; iq->eth_dev = eth_dev; iq->handler = hnd; iq->port_id = pi->port_id; @@ -1756,7 +1872,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, refill_fl_err: t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP, - iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff); + iq->cntxt_id, fl->cntxt_id, 0xffff); fl_nomem: ret = -ENOMEM; err: @@ -1829,7 +1945,7 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq, nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, "tx_ring", + eth_dev->device->driver->name, "tx_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -2064,7 +2180,7 @@ static int t4_sge_init_soft(struct adapter *adap) * The Page Size Buffer must be exactly equal to our Page Size and the * Large Page Size Buffer should be 0 (per above) or a power of 2. */ - if (fl_small_pg != PAGE_SIZE || + if (fl_small_pg != CXGBE_PAGE_SIZE || (fl_large_pg & (fl_large_pg - 1)) != 0) { dev_err(adap, "bad SGE FL page buffer sizes [%d, %d]\n", fl_small_pg, fl_large_pg); @@ -2122,8 +2238,7 @@ static int t4_sge_init_soft(struct adapter *adap) int t4_sge_init(struct adapter *adap) { struct sge *s = &adap->sge; - u32 sge_control, sge_control2, sge_conm_ctrl; - unsigned int ingpadboundary, ingpackboundary; + u32 sge_control, sge_conm_ctrl; int ret, egress_threshold; /* @@ -2133,34 +2248,7 @@ int t4_sge_init(struct adapter *adap) sge_control = t4_read_reg(adap, A_SGE_CONTROL); s->pktshift = G_PKTSHIFT(sge_control); s->stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; - - /* - * T4 uses a single control field to specify both the PCIe Padding and - * Packing Boundary. T5 introduced the ability to specify these - * separately. The actual Ingress Packet Data alignment boundary - * within Packed Buffer Mode is the maximum of these two - * specifications. - */ - ingpadboundary = 1 << (G_INGPADBOUNDARY(sge_control) + - X_INGPADBOUNDARY_SHIFT); - s->fl_align = ingpadboundary; - - if (!is_t4(adap->params.chip) && !adap->use_unpacked_mode) { - /* - * T5 has a weird interpretation of one of the PCIe Packing - * Boundary values. No idea why ... - */ - sge_control2 = t4_read_reg(adap, A_SGE_CONTROL2); - ingpackboundary = G_INGPACKBOUNDARY(sge_control2); - if (ingpackboundary == X_INGPACKBOUNDARY_16B) - ingpackboundary = 16; - else - ingpackboundary = 1 << (ingpackboundary + - X_INGPACKBOUNDARY_SHIFT); - - s->fl_align = max(ingpadboundary, ingpackboundary); - } - + s->fl_align = t4_fl_pkt_align(adap); ret = t4_sge_init_soft(adap); if (ret < 0) { dev_err(adap, "%s: t4_sge_init_soft failed, error %d\n",