X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fe1000%2Fem_rxtx.c;h=9994703cc25ed0ba23953fc5cff4bcc37f43972a;hb=c8834a3663b7ff3269a1b4e6a2772999dd485760;hp=a6b3e92a6b71ac39e1e58ffcf28fc2ac187d0001;hpb=a4996bd89c42590f8886454a06a994f71acf89dd;p=dpdk.git diff --git a/drivers/net/e1000/em_rxtx.c b/drivers/net/e1000/em_rxtx.c index a6b3e92a6b..9994703cc2 100644 --- a/drivers/net/e1000/em_rxtx.c +++ b/drivers/net/e1000/em_rxtx.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ #include #include #include -#include +#include #include #include #include @@ -50,6 +51,8 @@ #define E1000_RXDCTL_GRAN 0x01000000 /* RXDCTL Granularity */ #define E1000_TX_OFFLOAD_MASK ( \ + PKT_TX_IPV6 | \ + PKT_TX_IPV4 | \ PKT_TX_IP_CKSUM | \ PKT_TX_L4_MASK | \ PKT_TX_VLAN_PKT) @@ -57,6 +60,11 @@ #define E1000_TX_OFFLOAD_NOTSUP_MASK \ (PKT_TX_OFFLOAD_MASK ^ E1000_TX_OFFLOAD_MASK) +/* PCI offset for querying configuration status register */ +#define PCI_CFG_STATUS_REG 0x06 +#define FLUSH_DESC_REQUIRED 0x100 + + /** * Structure associated with each descriptor of the RX ring of a RX queue. */ @@ -96,6 +104,7 @@ struct em_rx_queue { uint8_t hthresh; /**< Host threshold register. */ uint8_t wthresh; /**< Write-back threshold register. */ uint8_t crc_len; /**< 0 if CRC stripped, 4 otherwise. */ + const struct rte_memzone *mz; }; /** @@ -165,6 +174,7 @@ struct em_tx_queue { struct em_ctx_info ctx_cache; /**< Hardware context history.*/ uint64_t offloads; /**< offloads of DEV_TX_OFFLOAD_* */ + const struct rte_memzone *mz; }; #if 1 @@ -220,7 +230,7 @@ em_set_xmit_ctx(struct em_tx_queue* txq, /* setup IPCS* fields */ ctx.lower_setup.ip_fields.ipcss = (uint8_t)l2len; ctx.lower_setup.ip_fields.ipcso = (uint8_t)(l2len + - offsetof(struct ipv4_hdr, hdr_checksum)); + offsetof(struct rte_ipv4_hdr, hdr_checksum)); /* * When doing checksum or TCP segmentation with IPv6 headers, @@ -242,12 +252,12 @@ em_set_xmit_ctx(struct em_tx_queue* txq, switch (flags & PKT_TX_L4_MASK) { case PKT_TX_UDP_CKSUM: ctx.upper_setup.tcp_fields.tucso = (uint8_t)(ipcse + - offsetof(struct udp_hdr, dgram_cksum)); + offsetof(struct rte_udp_hdr, dgram_cksum)); cmp_mask |= TX_MACIP_LEN_CMP_MASK; break; case PKT_TX_TCP_CKSUM: ctx.upper_setup.tcp_fields.tucso = (uint8_t)(ipcse + - offsetof(struct tcp_hdr, cksum)); + offsetof(struct rte_tcp_hdr, cksum)); cmd_len |= E1000_TXD_CMD_TCP; cmp_mask |= TX_MACIP_LEN_CMP_MASK; break; @@ -303,10 +313,10 @@ em_xmit_cleanup(struct em_tx_queue *txq) desc_to_clean_to = sw_ring[desc_to_clean_to].last_id; if (! (txr[desc_to_clean_to].upper.fields.status & E1000_TXD_STAT_DD)) { - PMD_TX_FREE_LOG(DEBUG, - "TX descriptor %4u is not done" - "(port=%d queue=%d)", desc_to_clean_to, - txq->port_id, txq->queue_id); + PMD_TX_LOG(DEBUG, + "TX descriptor %4u is not done" + "(port=%d queue=%d)", desc_to_clean_to, + txq->port_id, txq->queue_id); /* Failed to clean any descriptors, better luck next time */ return -(1); } @@ -319,11 +329,11 @@ em_xmit_cleanup(struct em_tx_queue *txq) nb_tx_to_clean = (uint16_t)(desc_to_clean_to - last_desc_cleaned); - PMD_TX_FREE_LOG(DEBUG, - "Cleaning %4u TX descriptors: %4u to %4u " - "(port=%d queue=%d)", nb_tx_to_clean, - last_desc_cleaned, desc_to_clean_to, txq->port_id, - txq->queue_id); + PMD_TX_LOG(DEBUG, + "Cleaning %4u TX descriptors: %4u to %4u " + "(port=%d queue=%d)", nb_tx_to_clean, + last_desc_cleaned, desc_to_clean_to, txq->port_id, + txq->queue_id); /* * The last descriptor to clean is done, so that means all the @@ -449,11 +459,11 @@ eth_em_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, * nb_used better be less than or equal to txq->tx_rs_thresh */ while (unlikely (nb_used > txq->nb_tx_free)) { - PMD_TX_FREE_LOG(DEBUG, "Not enough free TX descriptors " - "nb_used=%4u nb_free=%4u " - "(port=%d queue=%d)", - nb_used, txq->nb_tx_free, - txq->port_id, txq->queue_id); + PMD_TX_LOG(DEBUG, "Not enough free TX descriptors " + "nb_used=%4u nb_free=%4u " + "(port=%d queue=%d)", + nb_used, txq->nb_tx_free, + txq->port_id, txq->queue_id); if (em_xmit_cleanup(txq) != 0) { /* Could not clean any descriptors */ @@ -571,10 +581,10 @@ eth_em_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, /* Set RS bit only on threshold packets' last descriptor */ if (txq->nb_tx_used >= txq->tx_rs_thresh) { - PMD_TX_FREE_LOG(DEBUG, - "Setting RS bit on TXD id=%4u " - "(port=%d queue=%d)", - tx_last, txq->port_id, txq->queue_id); + PMD_TX_LOG(DEBUG, + "Setting RS bit on TXD id=%4u " + "(port=%d queue=%d)", + tx_last, txq->port_id, txq->queue_id); cmd_type_len |= E1000_TXD_CMD_RS; @@ -614,20 +624,20 @@ eth_em_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts, m = tx_pkts[i]; if (m->ol_flags & E1000_TX_OFFLOAD_NOTSUP_MASK) { - rte_errno = -ENOTSUP; + rte_errno = ENOTSUP; return i; } -#ifdef RTE_LIBRTE_ETHDEV_DEBUG +#ifdef RTE_ETHDEV_DEBUG_TX ret = rte_validate_tx_offload(m); if (ret != 0) { - rte_errno = ret; + rte_errno = -ret; return i; } #endif ret = rte_net_intel_cksum_prepare(m); if (ret != 0) { - rte_errno = ret; + rte_errno = -ret; return i; } } @@ -1003,17 +1013,17 @@ eth_em_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, */ rxm->next = NULL; if (unlikely(rxq->crc_len > 0)) { - first_seg->pkt_len -= ETHER_CRC_LEN; - if (data_len <= ETHER_CRC_LEN) { + first_seg->pkt_len -= RTE_ETHER_CRC_LEN; + if (data_len <= RTE_ETHER_CRC_LEN) { rte_pktmbuf_free_seg(rxm); first_seg->nb_segs--; last_seg->data_len = (uint16_t) (last_seg->data_len - - (ETHER_CRC_LEN - data_len)); + (RTE_ETHER_CRC_LEN - data_len)); last_seg->next = NULL; } else - rxm->data_len = - (uint16_t) (data_len - ETHER_CRC_LEN); + rxm->data_len = (uint16_t) + (data_len - RTE_ETHER_CRC_LEN); } /* @@ -1108,14 +1118,15 @@ em_tx_queue_release(struct em_tx_queue *txq) if (txq != NULL) { em_tx_queue_release_mbufs(txq); rte_free(txq->sw_ring); + rte_memzone_free(txq->mz); rte_free(txq); } } void -eth_em_tx_queue_release(void *txq) +eth_em_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) { - em_tx_queue_release(txq); + em_tx_queue_release(dev->data->tx_queues[qid]); } /* (Re)set dynamic em_tx_queue fields to defaults */ @@ -1160,6 +1171,7 @@ em_get_tx_port_offloads_capa(struct rte_eth_dev *dev) RTE_SET_USED(dev); tx_offload_capa = + DEV_TX_OFFLOAD_MULTI_SEGS | DEV_TX_OFFLOAD_VLAN_INSERT | DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM | @@ -1277,6 +1289,7 @@ eth_em_tx_queue_setup(struct rte_eth_dev *dev, RTE_CACHE_LINE_SIZE)) == NULL) return -ENOMEM; + txq->mz = tz; /* Allocate software ring */ if ((txq->sw_ring = rte_zmalloc("txq->sw_ring", sizeof(txq->sw_ring[0]) * nb_desc, @@ -1329,14 +1342,15 @@ em_rx_queue_release(struct em_rx_queue *rxq) if (rxq != NULL) { em_rx_queue_release_mbufs(rxq); rte_free(rxq->sw_ring); + rte_memzone_free(rxq->mz); rte_free(rxq); } } void -eth_em_rx_queue_release(void *rxq) +eth_em_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) { - em_rx_queue_release(rxq); + em_rx_queue_release(dev->data->rx_queues[qid]); } /* Reset dynamic em_rx_queue fields back to defaults */ @@ -1363,9 +1377,9 @@ em_get_rx_port_offloads_capa(struct rte_eth_dev *dev) DEV_RX_OFFLOAD_IPV4_CKSUM | DEV_RX_OFFLOAD_UDP_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM | - DEV_RX_OFFLOAD_CRC_STRIP | + DEV_RX_OFFLOAD_KEEP_CRC | DEV_RX_OFFLOAD_SCATTER; - if (max_rx_pktlen > ETHER_MAX_LEN) + if (max_rx_pktlen > RTE_ETHER_MAX_LEN) rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME; return rx_offload_capa; @@ -1416,12 +1430,13 @@ eth_em_rx_queue_setup(struct rte_eth_dev *dev, } /* - * EM devices don't support drop_en functionality + * EM devices don't support drop_en functionality. + * It's an optimization that does nothing on single-queue devices, + * so just log the issue and carry on. */ if (rx_conf->rx_drop_en) { - PMD_INIT_LOG(ERR, "drop_en functionality not supported by " + PMD_INIT_LOG(NOTICE, "drop_en functionality not supported by " "device"); - return -EINVAL; } /* Free memory prior to re-allocation if needed. */ @@ -1442,6 +1457,7 @@ eth_em_rx_queue_setup(struct rte_eth_dev *dev, RTE_CACHE_LINE_SIZE)) == NULL) return -ENOMEM; + rxq->mz = rz; /* Allocate software ring. */ if ((rxq->sw_ring = rte_zmalloc("rxq->sw_ring", sizeof (rxq->sw_ring[0]) * nb_desc, @@ -1458,8 +1474,10 @@ eth_em_rx_queue_setup(struct rte_eth_dev *dev, rxq->rx_free_thresh = rx_conf->rx_free_thresh; rxq->queue_id = queue_idx; rxq->port_id = dev->data->port_id; - rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.offloads & - DEV_RX_OFFLOAD_CRC_STRIP) ? 0 : ETHER_CRC_LEN); + if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) + rxq->crc_len = RTE_ETHER_CRC_LEN; + else + rxq->crc_len = 0; rxq->rdt_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_RDT(queue_idx)); rxq->rdh_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_RDH(queue_idx)); @@ -1499,23 +1517,6 @@ eth_em_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) return desc; } -int -eth_em_rx_descriptor_done(void *rx_queue, uint16_t offset) -{ - volatile struct e1000_rx_desc *rxdp; - struct em_rx_queue *rxq = rx_queue; - uint32_t desc; - - if (unlikely(offset >= rxq->nb_rx_desc)) - return 0; - desc = rxq->rx_tail + offset; - if (desc >= rxq->nb_rx_desc) - desc -= rxq->nb_rx_desc; - - rxdp = &rxq->rx_ring[desc]; - return !!(rxdp->status & E1000_RXD_STAT_DD); -} - int eth_em_rx_descriptor_status(void *rx_queue, uint16_t offset) { @@ -1597,13 +1598,13 @@ em_dev_free_queues(struct rte_eth_dev *dev) uint16_t i; for (i = 0; i < dev->data->nb_rx_queues; i++) { - eth_em_rx_queue_release(dev->data->rx_queues[i]); + eth_em_rx_queue_release(dev, i); dev->data->rx_queues[i] = NULL; } dev->data->nb_rx_queues = 0; for (i = 0; i < dev->data->nb_tx_queues; i++) { - eth_em_tx_queue_release(dev->data->tx_queues[i]); + eth_em_tx_queue_release(dev, i); dev->data->tx_queues[i] = NULL; } dev->data->nb_tx_queues = 0; @@ -1792,9 +1793,10 @@ eth_em_rx_init(struct rte_eth_dev *dev) * Reset crc_len in case it was changed after queue setup by a * call to configure */ - rxq->crc_len = - (uint8_t)(dev->data->dev_conf.rxmode.offloads & - DEV_RX_OFFLOAD_CRC_STRIP ? 0 : ETHER_CRC_LEN); + if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) + rxq->crc_len = RTE_ETHER_CRC_LEN; + else + rxq->crc_len = 0; bus_addr = rxq->rx_ring_phys_addr; E1000_WRITE_REG(hw, E1000_RDLEN(i), @@ -1825,7 +1827,7 @@ eth_em_rx_init(struct rte_eth_dev *dev) * one buffer. */ if (rxmode->offloads & DEV_RX_OFFLOAD_JUMBO_FRAME || - rctl_bsize < ETHER_MAX_LEN) { + rctl_bsize < RTE_ETHER_MAX_LEN) { if (!dev->data->scattered_rx) PMD_INIT_LOG(DEBUG, "forcing scatter mode"); dev->rx_pkt_burst = @@ -1873,10 +1875,10 @@ eth_em_rx_init(struct rte_eth_dev *dev) } /* Setup the Receive Control Register. */ - if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_CRC_STRIP) - rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC. */ - else + if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_KEEP_CRC) rctl &= ~E1000_RCTL_SECRC; /* Do not Strip Ethernet CRC. */ + else + rctl |= E1000_RCTL_SECRC; /* Strip Ethernet CRC. */ rctl &= ~(3 << E1000_RCTL_MO_SHIFT); rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | @@ -1957,6 +1959,22 @@ eth_em_tx_init(struct rte_eth_dev *dev) tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); + /* SPT and CNP Si errata workaround to avoid data corruption */ + if (hw->mac.type == e1000_pch_spt) { + uint32_t reg_val; + reg_val = E1000_READ_REG(hw, E1000_IOSFPC); + reg_val |= E1000_RCTL_RDMTS_HEX; + E1000_WRITE_REG(hw, E1000_IOSFPC, reg_val); + + /* Dropping the number of outstanding requests from + * 3 to 2 in order to avoid a buffer overrun. + */ + reg_val = E1000_READ_REG(hw, E1000_TARC(0)); + reg_val &= ~E1000_TARC0_CB_MULTIQ_3_REQ; + reg_val |= E1000_TARC0_CB_MULTIQ_2_REQ; + E1000_WRITE_REG(hw, E1000_TARC(0), reg_val); + } + /* This write will effectively turn on the transmit unit. */ E1000_WRITE_REG(hw, E1000_TCTL, tctl); } @@ -1993,3 +2011,119 @@ em_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh; qinfo->conf.offloads = txq->offloads; } + +static void +e1000_flush_tx_ring(struct rte_eth_dev *dev) +{ + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + volatile struct e1000_data_desc *tx_desc; + volatile uint32_t *tdt_reg_addr; + uint32_t tdt, tctl, txd_lower = E1000_TXD_CMD_IFCS; + uint16_t size = 512; + struct em_tx_queue *txq; + int i; + + if (dev->data->tx_queues == NULL) + return; + tctl = E1000_READ_REG(hw, E1000_TCTL); + E1000_WRITE_REG(hw, E1000_TCTL, tctl | E1000_TCTL_EN); + for (i = 0; i < dev->data->nb_tx_queues && + i < E1000_I219_MAX_TX_QUEUE_NUM; i++) { + txq = dev->data->tx_queues[i]; + tdt = E1000_READ_REG(hw, E1000_TDT(i)); + if (tdt != txq->tx_tail) + return; + tx_desc = &txq->tx_ring[txq->tx_tail]; + tx_desc->buffer_addr = rte_cpu_to_le_64(txq->tx_ring_phys_addr); + tx_desc->lower.data = rte_cpu_to_le_32(txd_lower | size); + tx_desc->upper.data = 0; + + rte_io_wmb(); + txq->tx_tail++; + if (txq->tx_tail == txq->nb_tx_desc) + txq->tx_tail = 0; + tdt_reg_addr = E1000_PCI_REG_ADDR(hw, E1000_TDT(i)); + E1000_PCI_REG_WRITE(tdt_reg_addr, txq->tx_tail); + usec_delay(250); + } +} + +static void +e1000_flush_rx_ring(struct rte_eth_dev *dev) +{ + uint32_t rctl, rxdctl; + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + int i; + + rctl = E1000_READ_REG(hw, E1000_RCTL); + E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); + E1000_WRITE_FLUSH(hw); + usec_delay(150); + + for (i = 0; i < dev->data->nb_rx_queues && + i < E1000_I219_MAX_RX_QUEUE_NUM; i++) { + rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); + /* zero the lower 14 bits (prefetch and host thresholds) */ + rxdctl &= 0xffffc000; + + /* update thresholds: prefetch threshold to 31, + * host threshold to 1 and make sure the granularity + * is "descriptors" and not "cache lines" + */ + rxdctl |= (0x1F | (1UL << 8) | E1000_RXDCTL_THRESH_UNIT_DESC); + + E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); + } + /* momentarily enable the RX ring for the changes to take effect */ + E1000_WRITE_REG(hw, E1000_RCTL, rctl | E1000_RCTL_EN); + E1000_WRITE_FLUSH(hw); + usec_delay(150); + E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); +} + +/** + * em_flush_desc_rings - remove all descriptors from the descriptor rings + * + * In i219, the descriptor rings must be emptied before resetting/closing the + * HW. Failure to do this will cause the HW to enter a unit hang state which + * can only be released by PCI reset on the device + * + */ + +void +em_flush_desc_rings(struct rte_eth_dev *dev) +{ + uint32_t fextnvm11, tdlen; + struct e1000_hw *hw = E1000_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); + uint16_t pci_cfg_status = 0; + int ret; + + fextnvm11 = E1000_READ_REG(hw, E1000_FEXTNVM11); + E1000_WRITE_REG(hw, E1000_FEXTNVM11, + fextnvm11 | E1000_FEXTNVM11_DISABLE_MULR_FIX); + tdlen = E1000_READ_REG(hw, E1000_TDLEN(0)); + ret = rte_pci_read_config(pci_dev, &pci_cfg_status, + sizeof(pci_cfg_status), PCI_CFG_STATUS_REG); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", + PCI_CFG_STATUS_REG); + return; + } + + /* do nothing if we're not in faulty state, or if the queue is empty */ + if ((pci_cfg_status & FLUSH_DESC_REQUIRED) && tdlen) { + /* flush desc ring */ + e1000_flush_tx_ring(dev); + ret = rte_pci_read_config(pci_dev, &pci_cfg_status, + sizeof(pci_cfg_status), PCI_CFG_STATUS_REG); + if (ret < 0) { + PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", + PCI_CFG_STATUS_REG); + return; + } + + if (pci_cfg_status & FLUSH_DESC_REQUIRED) + e1000_flush_rx_ring(dev); + } +}