X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=drivers%2Fnet%2Fcxgbe%2Fsge.c;h=736f08cee0adb258a6d7bad084f06efbfe074bbf;hb=a85e378cc60693793f39081307afe93e8fef4fd3;hp=4f4fdd687722bfe3e509dc3d43a906f9d7a40730;hpb=4a01078b4fd1d9dcfe3a154b4ad864452f34ab5e;p=dpdk.git diff --git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c index 4f4fdd6877..736f08cee0 100644 --- a/drivers/net/cxgbe/sge.c +++ b/drivers/net/cxgbe/sge.c @@ -31,7 +31,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include #include #include #include @@ -74,7 +73,7 @@ static inline void ship_tx_pkt_coalesce_wr(struct adapter *adap, /* * Max number of Rx buffers we replenish at a time. */ -#define MAX_RX_REFILL 16U +#define MAX_RX_REFILL 64U #define NOMEM_TMR_IDX (SGE_NTIMERS - 1) @@ -98,7 +97,8 @@ static inline unsigned int fl_mtu_bufsize(struct adapter *adapter, { struct sge *s = &adapter->sge; - return ALIGN(s->pktshift + ETH_HLEN + VLAN_HLEN + mtu, s->fl_align); + return CXGBE_ALIGN(s->pktshift + ETHER_HDR_LEN + VLAN_HLEN + mtu, + s->fl_align); } #define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL) @@ -199,11 +199,20 @@ static void free_tx_desc(struct sge_txq *q, unsigned int n) static void reclaim_tx_desc(struct sge_txq *q, unsigned int n) { + struct tx_sw_desc *d; unsigned int cidx = q->cidx; + d = &q->sdesc[cidx]; while (n--) { - if (++cidx == q->size) + if (d->mbuf) { /* an SGL is present */ + rte_pktmbuf_free(d->mbuf); + d->mbuf = NULL; + } + ++d; + if (++cidx == q->size) { cidx = 0; + d = q->sdesc; + } } q->cidx = cidx; } @@ -241,19 +250,10 @@ static inline bool fl_starving(const struct adapter *adapter, static inline unsigned int get_buf_size(struct adapter *adapter, const struct rx_sw_desc *d) { - struct sge *s = &adapter->sge; unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE; - unsigned int buf_size; + unsigned int buf_size = 0; switch (rx_buf_size_idx) { - case RX_SMALL_PG_BUF: - buf_size = PAGE_SIZE; - break; - - case RX_LARGE_PG_BUF: - buf_size = PAGE_SIZE << s->fl_pg_order; - break; - case RX_SMALL_MTU_BUF: buf_size = FL_MTU_SMALL_BUFSIZE(adapter); break; @@ -264,8 +264,7 @@ static inline unsigned int get_buf_size(struct adapter *adapter, default: BUG_ON(1); - buf_size = 0; /* deal with bogus compiler warnings */ - /* NOTREACHED */ + /* NOT REACHED */ } return buf_size; @@ -319,7 +318,7 @@ static void unmap_rx_buf(struct sge_fl *q) static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) { - if (q->pend_cred >= 8) { + if (q->pend_cred >= 64) { u32 val = adap->params.arch.sge_fl_db; if (is_t4(adap->params.chip)) @@ -356,15 +355,6 @@ static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q) } } -static inline struct rte_mbuf *cxgbe_rxmbuf_alloc(struct rte_mempool *mp) -{ - struct rte_mbuf *m; - - m = __rte_mbuf_raw_alloc(mp); - __rte_mbuf_sanity_check_raw(m, 0); - return m; -} - static inline void set_rx_sw_desc(struct rx_sw_desc *sd, void *buf, dma_addr_t mapping) { @@ -393,9 +383,28 @@ static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, __be64 *d = &q->desc[q->pidx]; struct rx_sw_desc *sd = &q->sdesc[q->pidx]; unsigned int buf_size_idx = RX_SMALL_MTU_BUF; + struct rte_mbuf *buf_bulk[n]; + int ret, i; + struct rte_pktmbuf_pool_private *mbp_priv; + u8 jumbo_en = rxq->rspq.eth_dev->data->dev_conf.rxmode.jumbo_frame; + + /* Use jumbo mtu buffers iff mbuf data room size can fit jumbo data. */ + mbp_priv = rte_mempool_get_priv(rxq->rspq.mb_pool); + if (jumbo_en && + ((mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM) >= 9000)) + buf_size_idx = RX_LARGE_MTU_BUF; + + ret = rte_mempool_get_bulk(rxq->rspq.mb_pool, (void *)buf_bulk, n); + if (unlikely(ret != 0)) { + dev_debug(adap, "%s: failed to allocated fl entries in bulk ..\n", + __func__); + q->alloc_failed++; + rxq->rspq.eth_dev->data->rx_mbuf_alloc_failed++; + goto out; + } - while (n--) { - struct rte_mbuf *mbuf = cxgbe_rxmbuf_alloc(rxq->rspq.mb_pool); + for (i = 0; i < n; i++) { + struct rte_mbuf *mbuf = buf_bulk[i]; dma_addr_t mapping; if (!mbuf) { @@ -405,11 +414,13 @@ static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q, goto out; } + rte_mbuf_refcnt_set(mbuf, 1); mbuf->data_off = RTE_PKTMBUF_HEADROOM; mbuf->next = NULL; + mbuf->nb_segs = 1; + mbuf->port = rxq->rspq.port_id; mapping = (dma_addr_t)(mbuf->buf_physaddr + mbuf->data_off); - mapping |= buf_size_idx; *d++ = cpu_to_be64(mapping); set_rx_sw_desc(sd, mbuf, mapping); @@ -668,6 +679,7 @@ static void write_sgl(struct rte_mbuf *mbuf, struct sge_txq *q, ((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head)) #define Q_IDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->size) +#define R_IDXDIFF(q, idx) IDXDIFF((q)->cidx, (q)->idx, (q)->size) /** * ring_tx_db - ring a Tx queue's doorbell @@ -796,20 +808,23 @@ static void tx_timer_cb(void *data) struct adapter *adap = (struct adapter *)data; struct sge_eth_txq *txq = &adap->sge.ethtxq[0]; int i; + unsigned int coal_idx; /* monitor any pending tx */ for (i = 0; i < adap->sge.max_ethqsets; i++, txq++) { - t4_os_lock(&txq->txq_lock); - if (txq->q.coalesce.idx) { - if (txq->q.coalesce.idx == txq->q.last_coal_idx && - txq->q.pidx == txq->q.last_pidx) { - ship_tx_pkt_coalesce_wr(adap, txq); - } else { - txq->q.last_coal_idx = txq->q.coalesce.idx; - txq->q.last_pidx = txq->q.pidx; + if (t4_os_trylock(&txq->txq_lock)) { + coal_idx = txq->q.coalesce.idx; + if (coal_idx) { + if (coal_idx == txq->q.last_coal_idx && + txq->q.pidx == txq->q.last_pidx) { + ship_tx_pkt_coalesce_wr(adap, txq); + } else { + txq->q.last_coal_idx = coal_idx; + txq->q.last_pidx = txq->q.pidx; + } } + t4_os_unlock(&txq->txq_lock); } - t4_os_unlock(&txq->txq_lock); } rte_eal_alarm_set(50, tx_timer_cb, (void *)adap); } @@ -1066,6 +1081,7 @@ int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf) u32 wr_mid; u64 cntrl, *end; bool v6; + u32 max_pkt_len = txq->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; /* Reject xmit if queue is stopped */ if (unlikely(txq->flags & EQ_STOPPED)) @@ -1081,7 +1097,10 @@ out_free: return 0; } - rte_prefetch0(&((&txq->q)->sdesc->mbuf->pool)); + if ((!(m->ol_flags & PKT_TX_TCP_SEG)) && + (unlikely(m->pkt_len > max_pkt_len))) + goto out_free; + pi = (struct port_info *)txq->eth_dev->data->dev_private; adap = pi->adapter; @@ -1089,7 +1108,7 @@ out_free: /* align the end of coalesce WR to a 512 byte boundary */ txq->q.coalesce.max = (8 - (txq->q.pidx & 7)) * 8; - if (!(m->ol_flags & PKT_TX_TCP_SEG)) { + if (!((m->ol_flags & PKT_TX_TCP_SEG) || (m->pkt_len > ETHER_MAX_LEN))) { if (should_tx_packet_coalesce(txq, mbuf, &cflits, adap)) { if (unlikely(map_mbuf(mbuf, addr) < 0)) { dev_warn(adap, "%s: mapping err for coalesce\n", @@ -1097,6 +1116,7 @@ out_free: txq->stats.mapping_err++; goto out_free; } + rte_prefetch0((volatile void *)addr); return tx_do_packet_coalesce(txq, mbuf, cflits, adap, pi, addr); } else { @@ -1135,33 +1155,46 @@ out_free: len = 0; len += sizeof(*cpl); - lso = (void *)(wr + 1); - v6 = (m->ol_flags & PKT_TX_IPV6) != 0; - l3hdr_len = m->l3_len; - l4hdr_len = m->l4_len; - eth_xtra_len = m->l2_len - ETHER_HDR_LEN; - len += sizeof(*lso); - wr->op_immdlen = htonl(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | - V_FW_WR_IMMDLEN(len)); - lso->lso_ctrl = htonl(V_LSO_OPCODE(CPL_TX_PKT_LSO) | - F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | - V_LSO_IPV6(v6) | - V_LSO_ETHHDR_LEN(eth_xtra_len / 4) | - V_LSO_IPHDR_LEN(l3hdr_len / 4) | - V_LSO_TCPHDR_LEN(l4hdr_len / 4)); - lso->ipid_ofst = htons(0); - lso->mss = htons(m->tso_segsz); - lso->seqno_offset = htonl(0); - if (is_t4(adap->params.chip)) - lso->len = htonl(m->pkt_len); - else - lso->len = htonl(V_LSO_T5_XFER_SIZE(m->pkt_len)); - cpl = (void *)(lso + 1); - cntrl = V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | - V_TXPKT_IPHDR_LEN(l3hdr_len) | - V_TXPKT_ETHHDR_LEN(eth_xtra_len); - txq->stats.tso++; - txq->stats.tx_cso += m->tso_segsz; + + /* Coalescing skipped and we send through normal path */ + if (!(m->ol_flags & PKT_TX_TCP_SEG)) { + wr->op_immdlen = htonl(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | + V_FW_WR_IMMDLEN(len)); + cpl = (void *)(wr + 1); + if (m->ol_flags & PKT_TX_IP_CKSUM) { + cntrl = hwcsum(adap->params.chip, m) | + F_TXPKT_IPCSUM_DIS; + txq->stats.tx_cso++; + } + } else { + lso = (void *)(wr + 1); + v6 = (m->ol_flags & PKT_TX_IPV6) != 0; + l3hdr_len = m->l3_len; + l4hdr_len = m->l4_len; + eth_xtra_len = m->l2_len - ETHER_HDR_LEN; + len += sizeof(*lso); + wr->op_immdlen = htonl(V_FW_WR_OP(FW_ETH_TX_PKT_WR) | + V_FW_WR_IMMDLEN(len)); + lso->lso_ctrl = htonl(V_LSO_OPCODE(CPL_TX_PKT_LSO) | + F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE | + V_LSO_IPV6(v6) | + V_LSO_ETHHDR_LEN(eth_xtra_len / 4) | + V_LSO_IPHDR_LEN(l3hdr_len / 4) | + V_LSO_TCPHDR_LEN(l4hdr_len / 4)); + lso->ipid_ofst = htons(0); + lso->mss = htons(m->tso_segsz); + lso->seqno_offset = htonl(0); + if (is_t4(adap->params.chip)) + lso->len = htonl(m->pkt_len); + else + lso->len = htonl(V_LSO_T5_XFER_SIZE(m->pkt_len)); + cpl = (void *)(lso + 1); + cntrl = V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 : TX_CSUM_TCPIP) | + V_TXPKT_IPHDR_LEN(l3hdr_len) | + V_TXPKT_ETHHDR_LEN(eth_xtra_len); + txq->stats.tso++; + txq->stats.tx_cso += m->tso_segsz; + } if (m->ol_flags & PKT_TX_VLAN_PKT) { txq->stats.vlan_ins++; @@ -1182,9 +1215,14 @@ out_free: last_desc -= txq->q.size; d = &txq->q.sdesc[last_desc]; - if (d->mbuf) { - rte_pktmbuf_free(d->mbuf); - d->mbuf = NULL; + if (d->coalesce.idx) { + int i; + + for (i = 0; i < d->coalesce.idx; i++) { + rte_pktmbuf_free(d->coalesce.mbuf[i]); + d->coalesce.mbuf[i] = NULL; + } + d->coalesce.idx = 0; } write_sgl(m, &txq->q, (struct ulptx_sgl *)(cpl + 1), end, 0, addr); @@ -1294,9 +1332,6 @@ static struct rte_mbuf *t4_pktgl_to_mbuf(const struct pkt_gl *gl) return t4_pktgl_to_mbuf_usembufs(gl); } -#define RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb) \ - ((dma_addr_t) ((mb)->buf_physaddr + (mb)->data_off)) - /** * t4_ethrx_handler - process an ingress ethernet packet * @q: the response queue that received the packet @@ -1326,14 +1361,14 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, mbuf->port = pkt->iff; if (pkt->l2info & htonl(F_RXF_IP)) { - mbuf->ol_flags |= PKT_RX_IPV4_HDR; + mbuf->packet_type = RTE_PTYPE_L3_IPV4; if (unlikely(!csum_ok)) mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; if ((pkt->l2info & htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok) mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; } else if (pkt->l2info & htonl(F_RXF_IP6)) { - mbuf->ol_flags |= PKT_RX_IPV6_HDR; + mbuf->packet_type = RTE_PTYPE_L3_IPV6; } mbuf->port = pkt->iff; @@ -1353,31 +1388,6 @@ int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp, return 0; } -/** - * restore_rx_bufs - put back a packet's Rx buffers - * @q: the SGE free list - * @frags: number of FL buffers to restore - * - * Puts back on an FL the Rx buffers. The buffers have already been - * unmapped and are left unmapped, we mark them so to prevent further - * unmapping attempts. - * - * This function undoes a series of @unmap_rx_buf calls when we find out - * that the current packet can't be processed right away afterall and we - * need to come back to it later. This is a very rare event and there's - * no effort to make this particularly efficient. - */ -static void restore_rx_bufs(struct sge_fl *q, int frags) -{ - while (frags--) { - if (q->cidx == 0) - q->cidx = q->size - 1; - else - q->cidx--; - q->avail++; - } -} - /** * is_new_response - check if a response is newly written * @r: the response descriptor @@ -1431,7 +1441,6 @@ static int process_responses(struct sge_rspq *q, int budget, int budget_left = budget; const struct rsp_ctrl *rc; struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq); - struct adapter *adapter = q->adapter; while (likely(budget_left)) { rc = (const struct rsp_ctrl *) @@ -1447,63 +1456,63 @@ static int process_responses(struct sge_rspq *q, int budget, rsp_type = G_RSPD_TYPE(rc->u.type_gen); if (likely(rsp_type == X_RSPD_TYPE_FLBUF)) { - struct pkt_gl si; - const struct rx_sw_desc *rsd; - struct rte_mbuf *pkt = NULL; - u32 len = ntohl(rc->pldbuflen_qid), bufsz, frags; - - si.usembufs = rxq->usembufs; - /* - * In "use mbufs" mode, we don't pack multiple - * ingress packets per buffer (mbuf) so we - * should _always_ get a "New Buffer" flags - * from the SGE. Also, since we hand the - * mbuf's up to the host stack for it to - * eventually free, we don't release the mbuf's - * in the driver (in contrast to the "packed - * page" mode where the driver needs to - * release its reference on the page buffers). - */ + const struct rx_sw_desc *rsd = + &rxq->fl.sdesc[rxq->fl.cidx]; + const struct rss_header *rss_hdr = + (const void *)q->cur_desc; + const struct cpl_rx_pkt *cpl = + (const void *)&q->cur_desc[1]; + bool csum_ok = cpl->csum_calc && !cpl->err_vec; + struct rte_mbuf *pkt, *npkt; + u32 len, bufsz; + + len = ntohl(rc->pldbuflen_qid); BUG_ON(!(len & F_RSPD_NEWBUF)); + pkt = rsd->buf; + npkt = pkt; len = G_RSPD_LEN(len); - si.tot_len = len; + pkt->pkt_len = len; - /* gather packet fragments */ - for (frags = 0; len; frags++) { - rsd = &rxq->fl.sdesc[rxq->fl.cidx]; - bufsz = min(get_buf_size(adapter, rsd), len); - pkt = rsd->buf; - pkt->data_len = bufsz; - pkt->pkt_len = bufsz; - si.mbufs[frags] = pkt; - len -= bufsz; + /* Chain mbufs into len if necessary */ + while (len) { + struct rte_mbuf *new_pkt = rsd->buf; + + bufsz = min(get_buf_size(q->adapter, rsd), len); + new_pkt->data_len = bufsz; unmap_rx_buf(&rxq->fl); + len -= bufsz; + npkt->next = new_pkt; + npkt = new_pkt; + pkt->nb_segs++; + rsd = &rxq->fl.sdesc[rxq->fl.cidx]; + } + npkt->next = NULL; + pkt->nb_segs--; + + if (cpl->l2info & htonl(F_RXF_IP)) { + pkt->packet_type = RTE_PTYPE_L3_IPV4; + if (unlikely(!csum_ok)) + pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; + + if ((cpl->l2info & + htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok) + pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; + } else if (cpl->l2info & htonl(F_RXF_IP6)) { + pkt->packet_type = RTE_PTYPE_L3_IPV6; } - si.va = RTE_PTR_ADD(si.mbufs[0]->buf_addr, - si.mbufs[0]->data_off); - rte_prefetch1(si.va); - - /* - * For the "use mbuf" case here, we can end up - * chewing through our Free List very rapidly - * with one entry per Ingress packet getting - * consumed. So if the handler() successfully - * consumed the mbuf, check to see if we can - * refill the Free List incrementally in the - * loop ... - */ - si.nfrags = frags; - ret = q->handler(q, q->cur_desc, &si); - - if (unlikely(ret != 0)) { - restore_rx_bufs(&rxq->fl, frags); - } else { - rx_pkts[budget - budget_left] = pkt; - if (fl_cap(&rxq->fl) - rxq->fl.avail >= 8) - __refill_fl(q->adapter, &rxq->fl); + if (!rss_hdr->filter_tid && rss_hdr->hash_type) { + pkt->ol_flags |= PKT_RX_RSS_HASH; + pkt->hash.rss = ntohl(rss_hdr->hash_val); } + if (cpl->vlan_ex) { + pkt->ol_flags |= PKT_RX_VLAN_PKT; + pkt->vlan_tci = ntohs(cpl->vlan); + } + rxq->stats.pkts++; + rxq->stats.rx_bytes += pkt->pkt_len; + rx_pkts[budget - budget_left] = pkt; } else if (likely(rsp_type == X_RSPD_TYPE_CPL)) { ret = q->handler(q, q->cur_desc, NULL); } else { @@ -1518,6 +1527,35 @@ static int process_responses(struct sge_rspq *q, int budget, rspq_next(q); budget_left--; + + if (R_IDXDIFF(q, gts_idx) >= 64) { + unsigned int cidx_inc = R_IDXDIFF(q, gts_idx); + unsigned int params; + u32 val; + + if (fl_cap(&rxq->fl) - rxq->fl.avail >= 64) + __refill_fl(q->adapter, &rxq->fl); + params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX); + q->next_intr_params = params; + val = V_CIDXINC(cidx_inc) | V_SEINTARM(params); + + if (unlikely(!q->bar2_addr)) + t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), + val | + V_INGRESSQID((u32)q->cntxt_id)); + else { + writel(val | V_INGRESSQID(q->bar2_qid), + (void *)((uintptr_t)q->bar2_addr + + SGE_UDB_GTS)); + /* + * This Write memory Barrier will force the + * write to the User Doorbell area to be + * flushed. + */ + wmb(); + } + q->gts_idx = q->cidx; + } } /* @@ -1526,7 +1564,7 @@ static int process_responses(struct sge_rspq *q, int budget, * refill the Free List. */ - if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 8) + if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 64) __refill_fl(q->adapter, &rxq->fl); return budget - budget_left; @@ -1535,36 +1573,9 @@ static int process_responses(struct sge_rspq *q, int budget, int cxgbe_poll(struct sge_rspq *q, struct rte_mbuf **rx_pkts, unsigned int budget, unsigned int *work_done) { - unsigned int params; - u32 val; int err = 0; *work_done = process_responses(q, budget, rx_pkts); - params = V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX); - q->next_intr_params = params; - val = V_CIDXINC(*work_done) | V_SEINTARM(params); - - if (*work_done) { - /* - * If we don't have access to the new User GTS (T5+), - * use the old doorbell mechanism; otherwise use the new - * BAR2 mechanism. - */ - if (unlikely(!q->bar2_addr)) - t4_write_reg(q->adapter, MYPF_REG(A_SGE_PF_GTS), - val | V_INGRESSQID((u32)q->cntxt_id)); - else { - writel(val | V_INGRESSQID(q->bar2_qid), - (void *)((uintptr_t)q->bar2_addr + - SGE_UDB_GTS)); - /* - * This Write memory Barrier will force the write to - * the User Doorbell area to be flushed. - */ - wmb(); - } - } - return err; } @@ -1631,10 +1642,11 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, unsigned int nb_refill; /* Size needs to be multiple of 16, including status entry. */ - iq->size = roundup(iq->size, 16); + iq->size = cxgbe_roundup(iq->size, 16); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, fwevtq ? "fwq_ring" : "rx_ring", + eth_dev->driver->pci_drv.driver.name, + fwevtq ? "fwq_ring" : "rx_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1670,7 +1682,8 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, if (fl) { struct sge_eth_rxq *rxq = container_of(fl, struct sge_eth_rxq, fl); - enum chip_type chip = CHELSIO_CHIP_VERSION(adap->params.chip); + enum chip_type chip = (enum chip_type)CHELSIO_CHIP_VERSION( + adap->params.chip); /* * Allocate the ring for the hardware free list (with space @@ -1682,10 +1695,10 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, */ if (fl->size < s->fl_starve_thres - 1 + 2 * 8) fl->size = s->fl_starve_thres - 1 + 2 * 8; - fl->size = roundup(fl->size, 8); + fl->size = cxgbe_roundup(fl->size, 8); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, + eth_dev->driver->pci_drv.driver.name, fwevtq ? "fwq_ring" : "fl_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1716,7 +1729,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, * Hence maximum allowed burst size will be 448 bytes. */ c.fl0dcaen_to_fl0cidxfthresh = - htons(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_64B) | + htons(V_FW_IQ_CMD_FL0FBMIN(X_FETCHBURSTMIN_128B) | V_FW_IQ_CMD_FL0FBMAX((chip <= CHELSIO_T5) ? X_FETCHBURSTMAX_512B : X_FETCHBURSTMAX_256B)); c.fl0size = htons(flsz); @@ -1729,6 +1742,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, iq->cur_desc = iq->desc; iq->cidx = 0; + iq->gts_idx = 0; iq->gen = 1; iq->next_intr_params = iq->intr_params; iq->cntxt_id = ntohs(c.iqid); @@ -1738,6 +1752,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, iq->size--; /* subtract status entry */ iq->eth_dev = eth_dev; iq->handler = hnd; + iq->port_id = pi->port_id; iq->mb_pool = mp; /* set offset to -1 to distinguish ingress queues without FL */ @@ -1806,7 +1821,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, refill_fl_err: t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP, - iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff); + iq->cntxt_id, fl->cntxt_id, 0xffff); fl_nomem: ret = -ENOMEM; err: @@ -1879,7 +1894,7 @@ int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq, nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc); snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d", - eth_dev->driver->pci_drv.name, "tx_ring", + eth_dev->driver->pci_drv.driver.name, "tx_ring", eth_dev->data->port_id, queue_id); snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name); @@ -1959,6 +1974,35 @@ static void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq, } } +/* + * Clear all queues of the port + * + * Note: This function must only be called after rx and tx path + * of the port have been disabled. + */ +void t4_sge_eth_clear_queues(struct port_info *pi) +{ + int i; + struct adapter *adap = pi->adapter; + struct sge_eth_rxq *rxq = &adap->sge.ethrxq[pi->first_qset]; + struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; + + for (i = 0; i < pi->n_rx_qsets; i++, rxq++) { + if (rxq->rspq.desc) + t4_sge_eth_rxq_stop(adap, &rxq->rspq); + } + for (i = 0; i < pi->n_tx_qsets; i++, txq++) { + if (txq->q.desc) { + struct sge_txq *q = &txq->q; + + t4_sge_eth_txq_stop(txq); + reclaim_completed_tx(q); + free_tx_desc(q, q->size); + q->equeidx = q->pidx; + } + } +} + void t4_sge_eth_rxq_release(struct adapter *adap, struct sge_eth_rxq *rxq) { if (rxq->rspq.desc) { @@ -1989,6 +2033,35 @@ void t4_sge_tx_monitor_stop(struct adapter *adap) rte_eal_alarm_cancel(tx_timer_cb, (void *)adap); } +/** + * t4_free_sge_resources - free SGE resources + * @adap: the adapter + * + * Frees resources used by the SGE queue sets. + */ +void t4_free_sge_resources(struct adapter *adap) +{ + int i; + struct sge_eth_rxq *rxq = &adap->sge.ethrxq[0]; + struct sge_eth_txq *txq = &adap->sge.ethtxq[0]; + + /* clean up Ethernet Tx/Rx queues */ + for (i = 0; i < adap->sge.max_ethqsets; i++, rxq++, txq++) { + /* Free only the queues allocated */ + if (rxq->rspq.desc) { + t4_sge_eth_rxq_release(adap, rxq); + rxq->rspq.eth_dev = NULL; + } + if (txq->q.desc) { + t4_sge_eth_txq_release(adap, txq); + txq->eth_dev = NULL; + } + } + + if (adap->sge.fw_evtq.desc) + free_rspq_fl(adap, &adap->sge.fw_evtq, NULL); +} + /** * t4_sge_init - initialize SGE * @adap: the adapter @@ -2056,7 +2129,7 @@ static int t4_sge_init_soft(struct adapter *adap) * The Page Size Buffer must be exactly equal to our Page Size and the * Large Page Size Buffer should be 0 (per above) or a power of 2. */ - if (fl_small_pg != PAGE_SIZE || + if (fl_small_pg != CXGBE_PAGE_SIZE || (fl_large_pg & (fl_large_pg - 1)) != 0) { dev_err(adap, "bad SGE FL page buffer sizes [%d, %d]\n", fl_small_pg, fl_large_pg);