X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_mbuf%2Frte_mbuf.h;h=c4c9ebfaa03b4fc21db8a641983226d909ed4172;hb=039b769f7c018351fbadf240f7dd692d3b02518f;hp=92d81972abbe419215adbb8075ca8ba17fe522ec;hpb=4958ca3a443a33ed49b2601232349037dbb7124f;p=dpdk.git diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h index 92d81972ab..c4c9ebfaa0 100644 --- a/lib/librte_mbuf/rte_mbuf.h +++ b/lib/librte_mbuf/rte_mbuf.h @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -152,13 +151,6 @@ rte_mbuf_data_iova(const struct rte_mbuf *mb) return mb->buf_iova + mb->data_off; } -__rte_deprecated -static inline phys_addr_t -rte_mbuf_data_dma_addr(const struct rte_mbuf *mb) -{ - return rte_mbuf_data_iova(mb); -} - /** * Return the default IO address of the beginning of the mbuf data * @@ -177,13 +169,6 @@ rte_mbuf_data_iova_default(const struct rte_mbuf *mb) return mb->buf_iova + RTE_PKTMBUF_HEADROOM; } -__rte_deprecated -static inline phys_addr_t -rte_mbuf_data_dma_addr_default(const struct rte_mbuf *mb) -{ - return rte_mbuf_data_iova_default(mb); -} - /** * Return the mbuf owning the data buffer address of an indirect mbuf. * @@ -303,8 +288,44 @@ rte_mbuf_to_priv(struct rte_mbuf *m) struct rte_pktmbuf_pool_private { uint16_t mbuf_data_room_size; /**< Size of data space in each mbuf. */ uint16_t mbuf_priv_size; /**< Size of private area in each mbuf. */ + uint32_t flags; /**< reserved for future use. */ }; +/** + * Return the flags from private data in an mempool structure. + * + * @param mp + * A pointer to the mempool structure. + * @return + * The flags from the private data structure. + */ +static inline uint32_t +rte_pktmbuf_priv_flags(struct rte_mempool *mp) +{ + struct rte_pktmbuf_pool_private *mbp_priv; + + mbp_priv = (struct rte_pktmbuf_pool_private *)rte_mempool_get_priv(mp); + return mbp_priv->flags; +} + +/** + * When set, pktmbuf mempool will hold only mbufs with pinned external + * buffer. The external buffer will be attached to the mbuf at the + * memory pool creation and will never be detached by the mbuf free calls. + * mbuf should not contain any room for data after the mbuf structure. + */ +#define RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF (1 << 0) + +/** + * Returns non zero if given mbuf has a pinned external buffer, or zero + * otherwise. The pinned external buffer is allocated at pool creation + * time and should not be freed on mbuf freeing. + * + * External buffer is a user-provided anonymous buffer. + */ +#define RTE_MBUF_HAS_PINNED_EXTBUF(mb) \ + (rte_pktmbuf_priv_flags(mb->pool) & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) + #ifdef RTE_LIBRTE_MBUF_DEBUG /** check mbuf type in debug mode */ @@ -329,7 +350,7 @@ struct rte_pktmbuf_pool_private { static inline uint16_t rte_mbuf_refcnt_read(const struct rte_mbuf *m) { - return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic)); + return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED); } /** @@ -342,14 +363,15 @@ rte_mbuf_refcnt_read(const struct rte_mbuf *m) static inline void rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value) { - rte_atomic16_set(&m->refcnt_atomic, (int16_t)new_value); + __atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED); } /* internal */ static inline uint16_t __rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value) { - return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value)); + return __atomic_add_fetch(&m->refcnt, (uint16_t)value, + __ATOMIC_ACQ_REL); } /** @@ -430,7 +452,7 @@ rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value) static inline uint16_t rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo) { - return (uint16_t)(rte_atomic16_read(&shinfo->refcnt_atomic)); + return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED); } /** @@ -445,7 +467,7 @@ static inline void rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo, uint16_t new_value) { - rte_atomic16_set(&shinfo->refcnt_atomic, (int16_t)new_value); + __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED); } /** @@ -469,7 +491,8 @@ rte_mbuf_ext_refcnt_update(struct rte_mbuf_ext_shared_info *shinfo, return (uint16_t)value; } - return (uint16_t)rte_atomic16_add_return(&shinfo->refcnt_atomic, value); + return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value, + __ATOMIC_ACQ_REL); } /** Mbuf prefetch */ @@ -517,12 +540,29 @@ __rte_experimental int rte_mbuf_check(const struct rte_mbuf *m, int is_header, const char **reason); -#define MBUF_RAW_ALLOC_CHECK(m) do { \ - RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); \ - RTE_ASSERT((m)->next == NULL); \ - RTE_ASSERT((m)->nb_segs == 1); \ - __rte_mbuf_sanity_check(m, 0); \ -} while (0) +/** + * Sanity checks on a reinitialized mbuf in debug mode. + * + * Check the consistency of the given reinitialized mbuf. + * The function will cause a panic if corruption is detected. + * + * Check that the mbuf is properly reinitialized (refcnt=1, next=NULL, + * nb_segs=1), as done by rte_pktmbuf_prefree_seg(). + * + * @param m + * The mbuf to be checked. + */ +static __rte_always_inline void +__rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m) +{ + RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); + RTE_ASSERT(m->next == NULL); + RTE_ASSERT(m->nb_segs == 1); + __rte_mbuf_sanity_check(m, 0); +} + +/** For backwards compatibility. */ +#define MBUF_RAW_ALLOC_CHECK(m) __rte_mbuf_raw_sanity_check(m) /** * Allocate an uninitialized mbuf from mempool *mp*. @@ -549,7 +589,7 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) if (rte_mempool_get(mp, (void **)&m) < 0) return NULL; - MBUF_RAW_ALLOC_CHECK(m); + __rte_mbuf_raw_sanity_check(m); return m; } @@ -570,11 +610,9 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m) { - RTE_ASSERT(RTE_MBUF_DIRECT(m)); - RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1); - RTE_ASSERT(m->next == NULL); - RTE_ASSERT(m->nb_segs == 1); - __rte_mbuf_sanity_check(m, 0); + RTE_ASSERT(!RTE_MBUF_CLONED(m) && + (!RTE_MBUF_HAS_EXTBUF(m) || RTE_MBUF_HAS_PINNED_EXTBUF(m))); + __rte_mbuf_raw_sanity_check(m); rte_mempool_put(m->pool, m); } @@ -600,7 +638,6 @@ rte_mbuf_raw_free(struct rte_mbuf *m) void rte_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *m, unsigned i); - /** * A packet mbuf pool constructor. * @@ -701,6 +738,63 @@ rte_pktmbuf_pool_create_by_ops(const char *name, unsigned int n, unsigned int cache_size, uint16_t priv_size, uint16_t data_room_size, int socket_id, const char *ops_name); +/** A structure that describes the pinned external buffer segment. */ +struct rte_pktmbuf_extmem { + void *buf_ptr; /**< The virtual address of data buffer. */ + rte_iova_t buf_iova; /**< The IO address of the data buffer. */ + size_t buf_len; /**< External buffer length in bytes. */ + uint16_t elt_size; /**< mbuf element size in bytes. */ +}; + +/** + * Create a mbuf pool with external pinned data buffers. + * + * This function creates and initializes a packet mbuf pool that contains + * only mbufs with external buffer. It is a wrapper to rte_mempool functions. + * + * @param name + * The name of the mbuf pool. + * @param n + * The number of elements in the mbuf pool. The optimum size (in terms + * of memory usage) for a mempool is when n is a power of two minus one: + * n = (2^q - 1). + * @param cache_size + * Size of the per-core object cache. See rte_mempool_create() for + * details. + * @param priv_size + * Size of application private are between the rte_mbuf structure + * and the data buffer. This value must be aligned to RTE_MBUF_PRIV_ALIGN. + * @param data_room_size + * Size of data buffer in each mbuf, including RTE_PKTMBUF_HEADROOM. + * @param socket_id + * The socket identifier where the memory should be allocated. The + * value can be *SOCKET_ID_ANY* if there is no NUMA constraint for the + * reserved zone. + * @param ext_mem + * Pointer to the array of structures describing the external memory + * for data buffers. It is caller responsibility to register this memory + * with rte_extmem_register() (if needed), map this memory to appropriate + * physical device, etc. + * @param ext_num + * Number of elements in the ext_mem array. + * @return + * The pointer to the new allocated mempool, on success. NULL on error + * with rte_errno set appropriately. Possible rte_errno values include: + * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure + * - E_RTE_SECONDARY - function was called from a secondary process instance + * - EINVAL - cache size provided is too large, or priv_size is not aligned. + * - ENOSPC - the maximum number of memzones has already been allocated + * - EEXIST - a memzone with the same name already exists + * - ENOMEM - no appropriate memory area found in which to create memzone + */ +__rte_experimental +struct rte_mempool * +rte_pktmbuf_pool_create_extbuf(const char *name, unsigned int n, + unsigned int cache_size, uint16_t priv_size, + uint16_t data_room_size, int socket_id, + const struct rte_pktmbuf_extmem *ext_mem, + unsigned int ext_num); + /** * Get the data room size of mbufs stored in a pktmbuf_pool * @@ -764,8 +858,6 @@ static inline void rte_pktmbuf_reset_headroom(struct rte_mbuf *m) * @param m * The packet mbuf to be reset. */ -#define MBUF_INVALID_PORT UINT16_MAX - static inline void rte_pktmbuf_reset(struct rte_mbuf *m) { m->next = NULL; @@ -774,9 +866,9 @@ static inline void rte_pktmbuf_reset(struct rte_mbuf *m) m->vlan_tci = 0; m->vlan_tci_outer = 0; m->nb_segs = 1; - m->port = MBUF_INVALID_PORT; + m->port = RTE_MBUF_PORT_INVALID; - m->ol_flags = 0; + m->ol_flags &= EXT_ATTACHED_MBUF; m->packet_type = 0; rte_pktmbuf_reset_headroom(m); @@ -837,22 +929,22 @@ static inline int rte_pktmbuf_alloc_bulk(struct rte_mempool *pool, switch (count % 4) { case 0: while (idx != count) { - MBUF_RAW_ALLOC_CHECK(mbufs[idx]); + __rte_mbuf_raw_sanity_check(mbufs[idx]); rte_pktmbuf_reset(mbufs[idx]); idx++; /* fall-through */ case 3: - MBUF_RAW_ALLOC_CHECK(mbufs[idx]); + __rte_mbuf_raw_sanity_check(mbufs[idx]); rte_pktmbuf_reset(mbufs[idx]); idx++; /* fall-through */ case 2: - MBUF_RAW_ALLOC_CHECK(mbufs[idx]); + __rte_mbuf_raw_sanity_check(mbufs[idx]); rte_pktmbuf_reset(mbufs[idx]); idx++; /* fall-through */ case 1: - MBUF_RAW_ALLOC_CHECK(mbufs[idx]); + __rte_mbuf_raw_sanity_check(mbufs[idx]); rte_pktmbuf_reset(mbufs[idx]); idx++; /* fall-through */ @@ -923,10 +1015,14 @@ rte_pktmbuf_ext_shinfo_init_helper(void *buf_addr, uint16_t *buf_len, * provided via shinfo. This callback function will be called once all the * mbufs are detached from the buffer (refcnt becomes zero). * - * The headroom for the attaching mbuf will be set to zero and this can be - * properly adjusted after attachment. For example, ``rte_pktmbuf_adj()`` + * The headroom length of the attaching mbuf will be set to zero and this + * can be properly adjusted after attachment. For example, ``rte_pktmbuf_adj()`` * or ``rte_pktmbuf_reset_headroom()`` might be used. * + * Similarly, the packet length is initialized to 0. If the buffer contains + * data, the user has to adjust ``data_len`` and the ``pkt_len`` field of + * the mbuf accordingly. + * * More mbufs can be attached to the same external buffer by * ``rte_pktmbuf_attach()`` once the external buffer has been attached by * this API. @@ -1024,7 +1120,6 @@ __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc) mdst->tx_offload = msrc->tx_offload; mdst->hash = msrc->hash; mdst->packet_type = msrc->packet_type; - mdst->timestamp = msrc->timestamp; rte_mbuf_dynfield_copy(mdst, msrc); } @@ -1131,6 +1226,11 @@ __rte_pktmbuf_free_direct(struct rte_mbuf *m) * * All other fields of the given packet mbuf will be left intact. * + * If the packet mbuf was allocated from the pool with pinned + * external buffers the rte_pktmbuf_detach does nothing with the + * mbuf of this kind, because the pinned buffers are not supposed + * to be detached. + * * @param m * The indirect attached packet mbuf. */ @@ -1140,11 +1240,26 @@ static inline void rte_pktmbuf_detach(struct rte_mbuf *m) uint32_t mbuf_size, buf_len; uint16_t priv_size; - if (RTE_MBUF_HAS_EXTBUF(m)) + if (RTE_MBUF_HAS_EXTBUF(m)) { + /* + * The mbuf has the external attached buffer, + * we should check the type of the memory pool where + * the mbuf was allocated from to detect the pinned + * external buffer. + */ + uint32_t flags = rte_pktmbuf_priv_flags(mp); + + if (flags & RTE_PKTMBUF_POOL_F_PINNED_EXT_BUF) { + /* + * The pinned external buffer should not be + * detached from its backing mbuf, just exit. + */ + return; + } __rte_pktmbuf_free_extbuf(m); - else + } else { __rte_pktmbuf_free_direct(m); - + } priv_size = rte_pktmbuf_priv_size(mp); mbuf_size = (uint32_t)(sizeof(struct rte_mbuf) + priv_size); buf_len = rte_pktmbuf_data_room_size(mp); @@ -1158,6 +1273,44 @@ static inline void rte_pktmbuf_detach(struct rte_mbuf *m) m->ol_flags = 0; } +/** + * @internal Handle the packet mbufs with attached pinned external buffer + * on the mbuf freeing: + * + * - return zero if reference counter in shinfo is one. It means there is + * no more reference to this pinned buffer and mbuf can be returned to + * the pool + * + * - otherwise (if reference counter is not one), decrement reference + * counter and return non-zero value to prevent freeing the backing mbuf. + * + * Returns non zero if mbuf should not be freed. + */ +static inline int __rte_pktmbuf_pinned_extbuf_decref(struct rte_mbuf *m) +{ + struct rte_mbuf_ext_shared_info *shinfo; + + /* Clear flags, mbuf is being freed. */ + m->ol_flags = EXT_ATTACHED_MBUF; + shinfo = m->shinfo; + + /* Optimize for performance - do not dec/reinit */ + if (likely(rte_mbuf_ext_refcnt_read(shinfo) == 1)) + return 0; + + /* + * Direct usage of add primitive to avoid + * duplication of comparing with one. + */ + if (likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1, + __ATOMIC_ACQ_REL))) + return 1; + + /* Reinitialize counter before mbuf freeing. */ + rte_mbuf_ext_refcnt_set(shinfo, 1); + return 0; +} + /** * Decrease reference counter and unlink a mbuf segment * @@ -1179,8 +1332,13 @@ rte_pktmbuf_prefree_seg(struct rte_mbuf *m) if (likely(rte_mbuf_refcnt_read(m) == 1)) { - if (!RTE_MBUF_DIRECT(m)) + if (!RTE_MBUF_DIRECT(m)) { rte_pktmbuf_detach(m); + if (RTE_MBUF_HAS_EXTBUF(m) && + RTE_MBUF_HAS_PINNED_EXTBUF(m) && + __rte_pktmbuf_pinned_extbuf_decref(m)) + return NULL; + } if (m->next != NULL) { m->next = NULL; @@ -1191,8 +1349,13 @@ rte_pktmbuf_prefree_seg(struct rte_mbuf *m) } else if (__rte_mbuf_refcnt_update(m, -1) == 0) { - if (!RTE_MBUF_DIRECT(m)) + if (!RTE_MBUF_DIRECT(m)) { rte_pktmbuf_detach(m); + if (RTE_MBUF_HAS_EXTBUF(m) && + RTE_MBUF_HAS_PINNED_EXTBUF(m) && + __rte_pktmbuf_pinned_extbuf_decref(m)) + return NULL; + } if (m->next != NULL) { m->next = NULL; @@ -1371,13 +1534,6 @@ static inline struct rte_mbuf *rte_pktmbuf_lastseg(struct rte_mbuf *m) return m; } -/* deprecated */ -#define rte_pktmbuf_mtophys_offset(m, o) \ - rte_pktmbuf_iova_offset(m, o) - -/* deprecated */ -#define rte_pktmbuf_mtophys(m) rte_pktmbuf_iova(m) - /** * A macro that returns the length of the packet. * @@ -1534,7 +1690,7 @@ static inline int rte_pktmbuf_trim(struct rte_mbuf *m, uint16_t len) static inline int rte_pktmbuf_is_contiguous(const struct rte_mbuf *m) { __rte_mbuf_sanity_check(m, 1); - return !!(m->nb_segs == 1); + return m->nb_segs == 1; } /**