#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_memory.h>
-#include <rte_atomic.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_byteorder.h>
return mb->buf_iova + mb->data_off;
}
-__rte_deprecated
-static inline phys_addr_t
-rte_mbuf_data_dma_addr(const struct rte_mbuf *mb)
-{
- return rte_mbuf_data_iova(mb);
-}
-
/**
* Return the default IO address of the beginning of the mbuf data
*
return mb->buf_iova + RTE_PKTMBUF_HEADROOM;
}
-__rte_deprecated
-static inline phys_addr_t
-rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb)
-{
- return rte_mbuf_data_iova_default(mb);
-}
-
/**
* Return the mbuf owning the data buffer address of an indirect mbuf.
*
struct rte_pktmbuf_pool_private {
uint16_t mbuf_data_room_size; /**< Size of data space in each mbuf. */
uint16_t mbuf_priv_size; /**< Size of private area in each mbuf. */
+ uint32_t flags; /**< reserved for future use. */
};
+/**
+ * Return the flags from private data in an mempool structure.
+ *
+ * @param mp
+ * A pointer to the mempool structure.
+ * @return
+ * The flags from the private data structure.
+ */
+static inline uint32_t
+rte_pktmbuf_priv_flags(struct rte_mempool *mp)
+{
+ struct rte_pktmbuf_pool_private *mbp_priv;
+
+ mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp);
+ return mbp_priv->flags;
+}
+
+/**
+ * When set, pktmbuf mempool will hold only mbufs with pinned external
+ * buffer. The external buffer will be attached to the mbuf at the
+ * memory pool creation and will never be detached by the mbuf free calls.
+ * mbuf should not contain any room for data after the mbuf structure.
+ */
+#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0)
+
+/**
+ * Returns non zero if given mbuf has a pinned external buffer, or zero
+ * otherwise. The pinned external buffer is allocated at pool creation
+ * time and should not be freed on mbuf freeing.
+ *
+ * External buffer is a user-provided anonymous buffer.
+ */
+#define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \
+ (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF)
+
#ifdef RTE_LIBRTE_MBUF_DEBUG
/** check mbuf type in debug mode */
static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
- return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic));
+ return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
}
/**
static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
- rte_atomic16_set(&m->refcnt_atomic, (int16_t)new_value);
+ __atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
}
/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
- return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
+ return __atomic_add_fetch(&m->refcnt, (uint16_t)value,
+ __ATOMIC_ACQ_REL);
}
/**
static inline uint16_t
rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
{
- return (uint16_t)(rte_atomic16_read(&shinfo->refcnt_atomic));
+ return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
}
/**
rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
uint16_t new_value)
{
- rte_atomic16_set(&shinfo->refcnt_atomic, (int16_t)new_value);
+ __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
}
/**
return (uint16_t)value;
}
- return (uint16_t)rte_atomic16_add_return(&shinfo->refcnt_atomic, value);
+ return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
+ __ATOMIC_ACQ_REL);
}
/** Mbuf prefetch */
int rte_mbuf_check(const struct rte_mbuf *m, int is_header,
const char **reason);
-#define MBUF_RAW_ALLOC_CHECK(m) do { \
- RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); \
- RTE_ASSERT((m)->next == NULL); \
- RTE_ASSERT((m)->nb_segs == 1); \
- __rte_mbuf_sanity_check(m, 0); \
-} while (0)
+/**
+ * Sanity checks on a reinitialized mbuf in debug mode.
+ *
+ * Check the consistency of the given reinitialized mbuf.
+ * The function will cause a panic if corruption is detected.
+ *
+ * Check that the mbuf is properly reinitialized (refcnt=1, next=NULL,
+ * nb_segs=1), as done by rte_pktmbuf_prefree_seg().
+ *
+ * @param m
+ * The mbuf to be checked.
+ */
+static __rte_always_inline void
+__rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)
+{
+ RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
+ RTE_ASSERT(m->next == NULL);
+ RTE_ASSERT(m->nb_segs == 1);
+ __rte_mbuf_sanity_check(m, 0);
+}
+
+/** For backwards compatibility. */
+#define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m)
/**
* Allocate an uninitialized mbuf from mempool *mp*.
if (rte_mempool_get(mp, (void **)&m) < 0)
return NULL;
- MBUF_RAW_ALLOC_CHECK(m);
+ __rte_mbuf_raw_sanity_check(m);
return m;
}
static __rte_always_inline void
rte_mbuf_raw_free(struct rte_mbuf *m)
{
- RTE_ASSERT(RTE_MBUF_DIRECT(m));
- RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);
- RTE_ASSERT(m->next == NULL);
- RTE_ASSERT(m->nb_segs == 1);
- __rte_mbuf_sanity_check(m, 0);
+ RTE_ASSERT(!RTE_MBUF_CLONED(m) &&
+ (!RTE_MBUF_HAS_EXTBUF(m) || RTE_MBUF_HAS_PINNED_EXTBUF(m)));
+ __rte_mbuf_raw_sanity_check(m);
rte_mempool_put(m->pool, m);
}
void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg,
void *m, unsigned i);
-
/**
* A packet mbuf pool constructor.
*
unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size,
int socket_id, const char *ops_name);
+/** A structure that describes the pinned external buffer segment. */
+struct rte_pktmbuf_extmem {
+ void *buf_ptr; /**< The virtual address of data buffer. */
+ rte_iova_t buf_iova; /**< The IO address of the data buffer. */
+ size_t buf_len; /**< External buffer length in bytes. */
+ uint16_t elt_size; /**< mbuf element size in bytes. */
+};
+
+/**
+ * Create a mbuf pool with external pinned data buffers.
+ *
+ * This function creates and initializes a packet mbuf pool that contains
+ * only mbufs with external buffer. It is a wrapper to rte_mempool functions.
+ *
+ * @param name
+ * The name of the mbuf pool.
+ * @param n
+ * The number of elements in the mbuf pool. The optimum size (in terms
+ * of memory usage) for a mempool is when n is a power of two minus one:
+ * n = (2^q - 1).
+ * @param cache_size
+ * Size of the per-core object cache. See rte_mempool_create() for
+ * details.
+ * @param priv_size
+ * Size of application private are between the rte_mbuf structure
+ * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN.
+ * @param data_room_size
+ * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM.
+ * @param socket_id
+ * The socket identifier where the memory should be allocated. The
+ * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the
+ * reserved zone.
+ * @param ext_mem
+ * Pointer to the array of structures describing the external memory
+ * for data buffers. It is caller responsibility to register this memory
+ * with rte_extmem_register() (if needed), map this memory to appropriate
+ * physical device, etc.
+ * @param ext_num
+ * Number of elements in the ext_mem array.
+ * @return
+ * The pointer to the new allocated mempool, on success. NULL on error
+ * with rte_errno set appropriately. Possible rte_errno values include:
+ * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure
+ * - E_RTE_SECONDARY - function was called from a secondary process instance
+ * - EINVAL - cache size provided is too large, or priv_size is not aligned.
+ * - ENOSPC - the maximum number of memzones has already been allocated
+ * - EEXIST - a memzone with the same name already exists
+ * - ENOMEM - no appropriate memory area found in which to create memzone
+ */
+__rte_experimental
+struct rte_mempool *
+rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n,
+ unsigned int cache_size, uint16_t priv_size,
+ uint16_t data_room_size, int socket_id,
+ const struct rte_pktmbuf_extmem *ext_mem,
+ unsigned int ext_num);
+
/**
* Get the data room size of mbufs stored in a pktmbuf_pool
*
* @param m
* The packet mbuf to be reset.
*/
-#define MBUF_INVALID_PORT UINT16_MAX
-
static inline void rte_pktmbuf_reset(struct rte_mbuf *m)
{
m->next = NULL;
m->vlan_tci = 0;
m->vlan_tci_outer = 0;
m->nb_segs = 1;
- m->port = MBUF_INVALID_PORT;
+ m->port = RTE_MBUF_PORT_INVALID;
- m->ol_flags = 0;
+ m->ol_flags &= EXT_ATTACHED_MBUF;
m->packet_type = 0;
rte_pktmbuf_reset_headroom(m);
switch (count % 4) {
case 0:
while (idx != count) {
- MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
+ __rte_mbuf_raw_sanity_check(mbufs[idx]);
rte_pktmbuf_reset(mbufs[idx]);
idx++;
/* fall-through */
case 3:
- MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
+ __rte_mbuf_raw_sanity_check(mbufs[idx]);
rte_pktmbuf_reset(mbufs[idx]);
idx++;
/* fall-through */
case 2:
- MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
+ __rte_mbuf_raw_sanity_check(mbufs[idx]);
rte_pktmbuf_reset(mbufs[idx]);
idx++;
/* fall-through */
case 1:
- MBUF_RAW_ALLOC_CHECK(mbufs[idx]);
+ __rte_mbuf_raw_sanity_check(mbufs[idx]);
rte_pktmbuf_reset(mbufs[idx]);
idx++;
/* fall-through */
* provided via shinfo. This callback function will be called once all the
* mbufs are detached from the buffer (refcnt becomes zero).
*
- * The headroom for the attaching mbuf will be set to zero and this can be
- * properly adjusted after attachment. For example, ``rte_pktmbuf_adj()``
+ * The headroom length of the attaching mbuf will be set to zero and this
+ * can be properly adjusted after attachment. For example, ``rte_pktmbuf_adj()``
* or ``rte_pktmbuf_reset_headroom()`` might be used.
*
+ * Similarly, the packet length is initialized to 0. If the buffer contains
+ * data, the user has to adjust ``data_len`` and the ``pkt_len`` field of
+ * the mbuf accordingly.
+ *
* More mbufs can be attached to the same external buffer by
* ``rte_pktmbuf_attach()`` once the external buffer has been attached by
* this API.
mdst->tx_offload = msrc->tx_offload;
mdst->hash = msrc->hash;
mdst->packet_type = msrc->packet_type;
- mdst->timestamp = msrc->timestamp;
rte_mbuf_dynfield_copy(mdst, msrc);
}
*
* All other fields of the given packet mbuf will be left intact.
*
+ * If the packet mbuf was allocated from the pool with pinned
+ * external buffers the rte_pktmbuf_detach does nothing with the
+ * mbuf of this kind, because the pinned buffers are not supposed
+ * to be detached.
+ *
* @param m
* The indirect attached packet mbuf.
*/
uint32_t mbuf_size, buf_len;
uint16_t priv_size;
- if (RTE_MBUF_HAS_EXTBUF(m))
+ if (RTE_MBUF_HAS_EXTBUF(m)) {
+ /*
+ * The mbuf has the external attached buffer,
+ * we should check the type of the memory pool where
+ * the mbuf was allocated from to detect the pinned
+ * external buffer.
+ */
+ uint32_t flags = rte_pktmbuf_priv_flags(mp);
+
+ if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) {
+ /*
+ * The pinned external buffer should not be
+ * detached from its backing mbuf, just exit.
+ */
+ return;
+ }
__rte_pktmbuf_free_extbuf(m);
- else
+ } else {
__rte_pktmbuf_free_direct(m);
-
+ }
priv_size = rte_pktmbuf_priv_size(mp);
mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size);
buf_len = rte_pktmbuf_data_room_size(mp);
m->ol_flags = 0;
}
+/**
+ * @internal Handle the packet mbufs with attached pinned external buffer
+ * on the mbuf freeing:
+ *
+ * - return zero if reference counter in shinfo is one. It means there is
+ * no more reference to this pinned buffer and mbuf can be returned to
+ * the pool
+ *
+ * - otherwise (if reference counter is not one), decrement reference
+ * counter and return non-zero value to prevent freeing the backing mbuf.
+ *
+ * Returns non zero if mbuf should not be freed.
+ */
+static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m)
+{
+ struct rte_mbuf_ext_shared_info *shinfo;
+
+ /* Clear flags, mbuf is being freed. */
+ m->ol_flags = EXT_ATTACHED_MBUF;
+ shinfo = m->shinfo;
+
+ /* Optimize for performance - do not dec/reinit */
+ if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1))
+ return 0;
+
+ /*
+ * Direct usage of add primitive to avoid
+ * duplication of comparing with one.
+ */
+ if (likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
+ __ATOMIC_ACQ_REL)))
+ return 1;
+
+ /* Reinitialize counter before mbuf freeing. */
+ rte_mbuf_ext_refcnt_set(shinfo, 1);
+ return 0;
+}
+
/**
* Decrease reference counter and unlink a mbuf segment
*
if (likely(rte_mbuf_refcnt_read(m) == 1)) {
- if (!RTE_MBUF_DIRECT(m))
+ if (!RTE_MBUF_DIRECT(m)) {
rte_pktmbuf_detach(m);
+ if (RTE_MBUF_HAS_EXTBUF(m) &&
+ RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
+ __rte_pktmbuf_pinned_extbuf_decref(m))
+ return NULL;
+ }
if (m->next != NULL) {
m->next = NULL;
} else if (__rte_mbuf_refcnt_update(m, -1) == 0) {
- if (!RTE_MBUF_DIRECT(m))
+ if (!RTE_MBUF_DIRECT(m)) {
rte_pktmbuf_detach(m);
+ if (RTE_MBUF_HAS_EXTBUF(m) &&
+ RTE_MBUF_HAS_PINNED_EXTBUF(m) &&
+ __rte_pktmbuf_pinned_extbuf_decref(m))
+ return NULL;
+ }
if (m->next != NULL) {
m->next = NULL;
return m;
}
-/* deprecated */
-#define rte_pktmbuf_mtophys_offset(m, o) \
- rte_pktmbuf_iova_offset(m, o)
-
-/* deprecated */
-#define rte_pktmbuf_mtophys(m) rte_pktmbuf_iova(m)
-
/**
* A macro that returns the length of the packet.
*
static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m)
{
__rte_mbuf_sanity_check(m, 1);
- return !!(m->nb_segs == 1);
+ return m->nb_segs == 1;
}
/**
git.droids-corp.org / dpdk.git / blobdiff