#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_memory.h>
-#include <rte_atomic.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_byteorder.h>
return mb->buf_iova + mb->data_off;
}
-__rte_deprecated
-static inline phys_addr_t
-rte_mbuf_data_dma_addr(const struct rte_mbuf *mb)
-{
- return rte_mbuf_data_iova(mb);
-}
-
/**
* Return the default IO address of the beginning of the mbuf data
*
return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
}
-__rte_deprecated
-static inline phys_addr_t
-rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb)
-{
- return rte_mbuf_data_iova_default(mb);
-}
-
/**
* Return the mbuf owning the data buffer address of an indirect mbuf.
*
static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
- return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic));
+ return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
}
/**
static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
- rte_atomic16_set(&m->refcnt_atomic, (int16_t)new_value);
+ __atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
}
/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
- return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
+ return __atomic_add_fetch(&m->refcnt, (uint16_t)value,
+ __ATOMIC_ACQ_REL);
}
/**
static inline uint16_t
rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
{
- return (uint16_t)(rte_atomic16_read(&shinfo->refcnt_atomic));
+ return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
}
/**
rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
uint16_t new_value)
{
- rte_atomic16_set(&shinfo->refcnt_atomic, (int16_t)new_value);
+ __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
}
/**
return (uint16_t)value;
}
- return (uint16_t)rte_atomic16_add_return(&shinfo->refcnt_atomic, value);
+ return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
+ __ATOMIC_ACQ_REL);
}
/** Mbuf prefetch */
int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
const char **reason);
-#define MBUF_RAW_ALLOC_CHECK(m) do { \
- RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); \
- RTE_ASSERT((m)->next == NULL); \
- RTE_ASSERT((m)->nb_segs == 1); \
- __rte_mbuf_sanity_check(m, 0); \
-} while (0)
+/**
+ * Sanity checks on a reinitialized mbuf in debug mode.
+ *
+ * Check the consistency of the given reinitialized mbuf.
+ * The function will cause a panic if corruption is detected.
+ *
+ * Check that the mbuf is properly reinitialized (refcnt=1, next=NULL,
+ * nb_segs=1), as done by rte_pktmbuf_prefree_seg().
+ *
+ * @param m
+ * The mbuf to be checked.
+ */
+static __rte_always_inline void
+__rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
+{
+ RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
+ RTE_ASSERT(m->next == NULL);
+ RTE_ASSERT(m->nb_segs == 1);
+ __rte_mbuf_sanity_check(m, 0);
+}
+
+/** For backwards compatibility. */
+#define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
/**
* Allocate an uninitialized mbuf from mempool *mp*.
if (rte_mempool_get(mp, (void **)&m) < 0)
return NULL;
- MBUF_RAW_ALLOC_CHECK(m);
+ __rte_mbuf_raw_sanity_check(m);
return m;
}
{
RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
(!RTE_MBUF_HAS_EXTBUF(m) || RTE_MBUF_HAS_PINNED_EXTBUF(m)));
- RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
- RTE_ASSERT(m->next == NULL);
- RTE_ASSERT(m->nb_segs == 1);
- __rte_mbuf_sanity_check(m, 0);
+ __rte_mbuf_raw_sanity_check(m);
rte_mempool_put(m->pool, m);
}
* @param m
* The packet mbuf to be reset.
*/
-#define MBUF_INVALID_PORT UINT16_MAX
-
static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
{
m->next = NULL;
m->vlan_tci = 0;
m->vlan_tci_outer = 0;
m->nb_segs = 1;
- m->port = MBUF_INVALID_PORT;
+ m->port = RTE_MBUF_PORT_INVALID;
m->ol_flags &= EXT_ATTACHED_MBUF;
m->packet_type = 0;
switch (count % 4) {
case 0:
while (idx != count) {
- MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
+ __rte_mbuf_raw_sanity_check(mbufs[idx]);
rte_pktmbuf_reset(mbufs[idx]);
idx++;
/* fall-through */
case 3:
- MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
+ __rte_mbuf_raw_sanity_check(mbufs[idx]);
rte_pktmbuf_reset(mbufs[idx]);
idx++;
/* fall-through */
case 2:
- MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
+ __rte_mbuf_raw_sanity_check(mbufs[idx]);
rte_pktmbuf_reset(mbufs[idx]);
idx++;
/* fall-through */
case 1:
- MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
+ __rte_mbuf_raw_sanity_check(mbufs[idx]);
rte_pktmbuf_reset(mbufs[idx]);
idx++;
/* fall-through */
mdst->tx_offload = msrc->tx_offload;
mdst->hash = msrc->hash;
mdst->packet_type = msrc->packet_type;
- mdst->timestamp = msrc->timestamp;
rte_mbuf_dynfield_copy(mdst, msrc);
}
* Direct usage of add primitive to avoid
* duplication of comparing with one.
*/
- if (likely(rte_atomic16_add_return
- (&shinfo->refcnt_atomic, -1)))
+ if (likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
+ __ATOMIC_ACQ_REL)))
return 1;
/* Reinitialize counter before mbuf freeing. */
if (likely(rte_mbuf_refcnt_read(m) == 1)) {
if (!RTE_MBUF_DIRECT(m)) {
- if (!RTE_MBUF_HAS_EXTBUF(m) ||
- !RTE_MBUF_HAS_PINNED_EXTBUF(m))
- rte_pktmbuf_detach(m);
- else if (__rte_pktmbuf_pinned_extbuf_decref(m))
+ rte_pktmbuf_detach(m);
+ if (RTE_MBUF_HAS_EXTBUF(m) &&
+ RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
+ __rte_pktmbuf_pinned_extbuf_decref(m))
return NULL;
}
- if (m->next != NULL) {
+ if (m->next != NULL)
m->next = NULL;
+ if (m->nb_segs != 1)
m->nb_segs = 1;
- }
return m;
} else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
if (!RTE_MBUF_DIRECT(m)) {
- if (!RTE_MBUF_HAS_EXTBUF(m) ||
- !RTE_MBUF_HAS_PINNED_EXTBUF(m))
- rte_pktmbuf_detach(m);
- else if (__rte_pktmbuf_pinned_extbuf_decref(m))
+ rte_pktmbuf_detach(m);
+ if (RTE_MBUF_HAS_EXTBUF(m) &&
+ RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
+ __rte_pktmbuf_pinned_extbuf_decref(m))
return NULL;
}
- if (m->next != NULL) {
+ if (m->next != NULL)
m->next = NULL;
+ if (m->nb_segs != 1)
m->nb_segs = 1;
- }
rte_mbuf_refcnt_set(m, 1);
return m;
return m;
}
-/* deprecated */
-#define rte_pktmbuf_mtophys_offset(m, o) \
- rte_pktmbuf_iova_offset(m, o)
-
-/* deprecated */
-#define rte_pktmbuf_mtophys(m) rte_pktmbuf_iova(m)
-
/**
* A macro that returns the length of the packet.
*
static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
{
__rte_mbuf_sanity_check(m, 1);
- return !!(m->nb_segs == 1);
+ return m->nb_segs == 1;
}
/**