mbuf: fix bulk allocation when debug enabled

[dpdk.git] / lib / librte_mbuf / rte_mbuf.h

diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h
index 323a1ac..1cb0310 100644 (file)
--- a/lib/librte_mbuf/rte_mbuf.h
+++ b/lib/librte_mbuf/rte_mbuf.h
@@ -184,6 +184,11 @@ extern "C" {
  */
 #define PKT_RX_LRO           (1ULL << 16)
 
+/**
+ * Indicate that the timestamp field in the mbuf is valid.
+ */
+#define PKT_RX_TIMESTAMP     (1ULL << 17)
+
 /* add new RX flags here */
 
 /* add new TX flags here */
@@ -398,7 +403,13 @@ struct rte_mbuf {
        MARKER cacheline0;
 
        void *buf_addr;           /**< Virtual address of segment buffer. */
-       phys_addr_t buf_physaddr; /**< Physical address of segment buffer. */
+       /**
+        * Physical address of segment buffer.
+        * Force alignment to 8-bytes, so as to ensure we have the exact
+        * same mbuf cacheline0 layout for 32-bit and 64-bit. This makes
+        * working on vector drivers easier.
+        */
+       phys_addr_t buf_physaddr __rte_aligned(sizeof(phys_addr_t));
 
        /* next 8 bytes are initialised on RX descriptor rearm */
        MARKER64 rearm_data;
@@ -477,12 +488,16 @@ struct rte_mbuf {
                uint32_t usr;     /**< User defined tags. See rte_distributor_process() */
        } hash;                   /**< hash information */
 
-       uint32_t seqn; /**< Sequence number. See also rte_reorder_insert() */
-
        /** Outer VLAN TCI (CPU order), valid if PKT_RX_QINQ_STRIPPED is set. */
        uint16_t vlan_tci_outer;
 
        uint16_t buf_len;         /**< Length of segment buffer. */
+
+       /** Valid if PKT_RX_TIMESTAMP is set. The unit and time reference
+        * are not normalized but are always the same for a given port.
+        */
+       uint64_t timestamp;
+
        /* second cache line - fields only used in slow path or on TX */
        MARKER cacheline1 __rte_cache_min_aligned;
 
@@ -523,6 +538,10 @@ struct rte_mbuf {
 
        /** Timesync flags for use with IEEE1588. */
        uint16_t timesync;
+
+       /** Sequence number. See also rte_reorder_insert(). */
+       uint32_t seqn;
+
 } __rte_cache_aligned;
 
 /**
@@ -769,6 +788,13 @@ rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
 void
 rte_mbuf_sanity_check(const struct rte_mbuf *m, int is_header);
 
+#define MBUF_RAW_ALLOC_CHECK(m) do {                           \
+       RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);               \
+       RTE_ASSERT((m)->next == NULL);                          \
+       RTE_ASSERT((m)->nb_segs == 1);                          \
+       __rte_mbuf_sanity_check(m, 0);                          \
+} while (0)
+
 /**
  * Allocate an unitialized mbuf from mempool *mp*.
  *
@@ -796,11 +822,7 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
        if (rte_mempool_get(mp, &mb) < 0)
                return NULL;
        m = (struct rte_mbuf *)mb;
-       RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
-       RTE_ASSERT(m->next == NULL);
-       RTE_ASSERT(m->nb_segs == 1);
-       __rte_mbuf_sanity_check(m, 0);
-
+       MBUF_RAW_ALLOC_CHECK(m);
        return m;
 }
 
@@ -1133,25 +1155,25 @@ static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool,
        switch (count % 4) {
        case 0:
                while (idx != count) {
-                       RTE_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0);
-                       rte_mbuf_refcnt_set(mbufs[idx], 1);
+                       MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
                        rte_pktmbuf_reset(mbufs[idx]);
                        idx++;
+                       /* fall-through */
        case 3:
-                       RTE_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0);
-                       rte_mbuf_refcnt_set(mbufs[idx], 1);
+                       MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
                        rte_pktmbuf_reset(mbufs[idx]);
                        idx++;
+                       /* fall-through */
        case 2:
-                       RTE_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0);
-                       rte_mbuf_refcnt_set(mbufs[idx], 1);
+                       MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
                        rte_pktmbuf_reset(mbufs[idx]);
                        idx++;
+                       /* fall-through */
        case 1:
-                       RTE_ASSERT(rte_mbuf_refcnt_read(mbufs[idx]) == 0);
-                       rte_mbuf_refcnt_set(mbufs[idx], 1);
+                       MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
                        rte_pktmbuf_reset(mbufs[idx]);
                        idx++;
+                       /* fall-through */
                }
        }
        return 0;
@@ -1206,6 +1228,7 @@ static inline void rte_pktmbuf_attach(struct rte_mbuf *mi, struct rte_mbuf *m)
        mi->nb_segs = 1;
        mi->ol_flags = m->ol_flags | IND_ATTACHED_MBUF;
        mi->packet_type = m->packet_type;
+       mi->timestamp = m->timestamp;
 
        __rte_mbuf_sanity_check(mi, 1);
        __rte_mbuf_sanity_check(m, 0);