mpipe: fix link initialization ordering

[dpdk.git] / drivers / net / mpipe / mpipe_tilegx.c

diff --git a/drivers/net/mpipe/mpipe_tilegx.c b/drivers/net/mpipe/mpipe_tilegx.c
index 743feef..b887de6 100644 (file)
--- a/drivers/net/mpipe/mpipe_tilegx.c
+++ b/drivers/net/mpipe/mpipe_tilegx.c
@@ -78,8 +78,16 @@ struct mpipe_context {
        struct mpipe_channel_config channels[MPIPE_MAX_CHANNELS];
 };
 
+/* Per-core local data. */
+struct mpipe_local {
+       int mbuf_push_debt[RTE_MAX_ETHPORTS];   /* Buffer push debt. */
+} __rte_cache_aligned;
+
+#define MPIPE_BUF_DEBT_THRESHOLD       32
+static __thread struct mpipe_local mpipe_local;
 static struct mpipe_context mpipe_contexts[GXIO_MPIPE_INSTANCE_MAX];
 static int mpipe_instances;
+static const char *drivername = "MPIPE PMD";
 
 /* Per queue statistics. */
 struct mpipe_queue_stats {
@@ -122,7 +130,6 @@ struct mpipe_dev_priv {
        int channel;                    /* Device channel. */
        int port_id;                    /* DPDK port index. */
        struct rte_eth_dev *eth_dev;    /* DPDK device. */
-       struct rte_pci_device pci_dev;  /* PCI device data. */
        struct rte_mbuf **tx_comps;     /* TX completion array. */
        struct rte_mempool *rx_mpool;   /* mpool used by the rx queues. */
        unsigned rx_offset;             /* Receive head room. */
@@ -137,7 +144,7 @@ struct mpipe_dev_priv {
        int first_bucket;               /* mPIPE bucket start index. */
        int first_ring;                 /* mPIPE notif ring start index. */
        int notif_group;                /* mPIPE notif group. */
-       rte_atomic32_t dp_count;        /* Active datapath thread count. */
+       rte_atomic32_t dp_count __rte_cache_aligned;    /* DP Entry count. */
        int tx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
        int rx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
 };
@@ -361,8 +368,8 @@ static inline int
 mpipe_link_compare(struct rte_eth_link *link1,
                   struct rte_eth_link *link2)
 {
-       return ((*(uint64_t *)link1 == *(uint64_t *)link2)
-               ? -1 : 0);
+       return (*(uint64_t *)link1 == *(uint64_t *)link2)
+               ? -1 : 0;
 }
 
 static int
@@ -461,6 +468,14 @@ mpipe_dp_wait(struct mpipe_dev_priv *priv)
        }
 }
 
+static inline int
+mpipe_mbuf_stack_index(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
+{
+       return (mbuf->port < RTE_MAX_ETHPORTS) ?
+               mpipe_priv(&rte_eth_devices[mbuf->port])->stack :
+               priv->stack;
+}
+
 static inline struct rte_mbuf *
 mpipe_recv_mbuf(struct mpipe_dev_priv *priv, gxio_mpipe_idesc_t *idesc,
                int in_port)
@@ -737,13 +752,6 @@ mpipe_init(struct mpipe_dev_priv *priv)
        if (priv->initialized)
                return 0;
 
-       rc = mpipe_link_init(priv);
-       if (rc < 0) {
-               RTE_LOG(ERR, PMD, "%s: Failed to init link.\n",
-                       mpipe_name(priv));
-               return rc;
-       }
-
        rc = mpipe_recv_init(priv);
        if (rc < 0) {
                RTE_LOG(ERR, PMD, "%s: Failed to init rx.\n",
@@ -1267,6 +1275,7 @@ mpipe_do_xmit(struct mpipe_tx_queue *tx_queue, struct rte_mbuf **tx_pkts,
        unsigned nb_bytes = 0;
        unsigned nb_sent = 0;
        int nb_slots, i;
+       uint8_t port_id;
 
        PMD_DEBUG_TX("Trying to transmit %d packets on %s:%d.\n",
                     nb_pkts, mpipe_name(tx_queue->q.priv),
@@ -1315,14 +1324,22 @@ mpipe_do_xmit(struct mpipe_tx_queue *tx_queue, struct rte_mbuf **tx_pkts,
                        if (priv->tx_comps[idx])
                                rte_pktmbuf_free_seg(priv->tx_comps[idx]);
 
+                       port_id = (mbuf->port < RTE_MAX_ETHPORTS) ?
+                                               mbuf->port : priv->port_id;
                        desc = (gxio_mpipe_edesc_t) { {
                                .va        = rte_pktmbuf_mtod(mbuf, uintptr_t),
                                .xfer_size = rte_pktmbuf_data_len(mbuf),
                                .bound     = next ? 0 : 1,
+                               .stack_idx = mpipe_mbuf_stack_index(priv, mbuf),
                        } };
+                       if (mpipe_local.mbuf_push_debt[port_id] > 0) {
+                               mpipe_local.mbuf_push_debt[port_id]--;
+                               desc.hwb = 1;
+                               priv->tx_comps[idx] = NULL;
+                       } else
+                               priv->tx_comps[idx] = mbuf;
 
                        nb_bytes += mbuf->data_len;
-                       priv->tx_comps[idx] = mbuf;
                        gxio_mpipe_equeue_put_at(equeue, desc, slot + i);
 
                        PMD_DEBUG_TX("%s:%d: Sending packet %p, len %d\n",
@@ -1443,17 +1460,22 @@ mpipe_do_recv(struct mpipe_rx_queue *rx_queue, struct rte_mbuf **rx_pkts,
                                continue;
                        }
 
-                       mbuf = __rte_mbuf_raw_alloc(priv->rx_mpool);
-                       if (unlikely(!mbuf)) {
-                               nb_nomem++;
-                               gxio_mpipe_iqueue_drop(iqueue, idesc);
-                               PMD_DEBUG_RX("%s:%d: RX alloc failure\n",
+                       if (mpipe_local.mbuf_push_debt[in_port] <
+                                       MPIPE_BUF_DEBT_THRESHOLD)
+                               mpipe_local.mbuf_push_debt[in_port]++;
+                       else {
+                               mbuf = __rte_mbuf_raw_alloc(priv->rx_mpool);
+                               if (unlikely(!mbuf)) {
+                                       nb_nomem++;
+                                       gxio_mpipe_iqueue_drop(iqueue, idesc);
+                                       PMD_DEBUG_RX("%s:%d: alloc failure\n",
                                             mpipe_name(rx_queue->q.priv),
                                             rx_queue->q.queue_idx);
-                               continue;
-                       }
+                                       continue;
+                               }
 
-                       mpipe_recv_push(priv, mbuf);
+                               mpipe_recv_push(priv, mbuf);
+                       }
 
                        /* Get and setup the mbuf for the received packet. */
                        mbuf = mpipe_recv_mbuf(priv, idesc, in_port);
@@ -1567,7 +1589,6 @@ rte_pmd_mpipe_devinit(const char *ifname,
        priv->context = context;
        priv->instance = instance;
        priv->is_xaui = (strncmp(ifname, "xgbe", 4) == 0);
-       priv->pci_dev.numa_node = instance;
        priv->channel = -1;
 
        mac = priv->mac_addr.addr_bytes;
@@ -1582,6 +1603,7 @@ rte_pmd_mpipe_devinit(const char *ifname,
        if (!eth_dev) {
                RTE_LOG(ERR, PMD, "%s: Failed to allocate device.\n", ifname);
                rte_free(priv);
+               return -ENOMEM;
        }
 
        RTE_LOG(INFO, PMD, "%s: Initialized mpipe device"
@@ -1591,13 +1613,25 @@ rte_pmd_mpipe_devinit(const char *ifname,
        priv->eth_dev = eth_dev;
        priv->port_id = eth_dev->data->port_id;
        eth_dev->data->dev_private = priv;
-       eth_dev->pci_dev = &priv->pci_dev;
        eth_dev->data->mac_addrs = &priv->mac_addr;
 
+       eth_dev->data->dev_flags = 0;
+       eth_dev->data->kdrv = RTE_KDRV_NONE;
+       eth_dev->driver = NULL;
+       eth_dev->data->drv_name = drivername;
+       eth_dev->data->numa_node = instance;
+
        eth_dev->dev_ops      = &mpipe_dev_ops;
        eth_dev->rx_pkt_burst = &mpipe_recv_pkts;
        eth_dev->tx_pkt_burst = &mpipe_xmit_pkts;
 
+       rc = mpipe_link_init(priv);
+       if (rc < 0) {
+               RTE_LOG(ERR, PMD, "%s: Failed to init link.\n",
+                       mpipe_name(priv));
+               return rc;
+       }
+
        return 0;
 }