From: Thomas Monjalon Date: Sun, 9 Feb 2020 17:39:19 +0000 (+0100) Subject: replace hot attributes X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=e3866e73555091d56b33b4d8916e0e484ccae114;p=dpdk.git replace hot attributes The new macro __rte_hot, for compiler hinting, is now used where appropriate for consistency. Signed-off-by: Thomas Monjalon --- diff --git a/drivers/common/cpt/cpt_common.h b/drivers/common/cpt/cpt_common.h index dff45f0fe9..56bfea4953 100644 --- a/drivers/common/cpt/cpt_common.h +++ b/drivers/common/cpt/cpt_common.h @@ -31,10 +31,6 @@ #define ROUNDUP16(val) (((val) + 15) & 0xfffffff0) #endif -#ifndef __hot -#define __hot __attribute__((hot)) -#endif - #define MOD_INC(i, l) ((i) == (l - 1) ? (i) = 0 : (i)++) struct cpt_qp_meta_info { diff --git a/drivers/common/cpt/cpt_ucode.h b/drivers/common/cpt/cpt_ucode.h index c310ea7cb4..0d3a01b37a 100644 --- a/drivers/common/cpt/cpt_ucode.h +++ b/drivers/common/cpt/cpt_ucode.h @@ -2455,7 +2455,7 @@ cpt_fc_dec_hmac_prep(uint32_t flags, return prep_req; } -static __rte_always_inline void *__hot +static __rte_always_inline void *__rte_hot cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, fc_params_t *fc_params, void *op) { diff --git a/drivers/common/cpt/cpt_ucode_asym.h b/drivers/common/cpt/cpt_ucode_asym.h index a67efb361c..5d1c7b5f02 100644 --- a/drivers/common/cpt/cpt_ucode_asym.h +++ b/drivers/common/cpt/cpt_ucode_asym.h @@ -464,7 +464,7 @@ cpt_rsa_crt_prep(struct asym_op_params *rsa_params, cpt_fill_req_comp_addr(req, caddr); } -static __rte_always_inline int __hot +static __rte_always_inline int __rte_hot cpt_enqueue_rsa_op(struct rte_crypto_op *op, struct asym_op_params *params, struct cpt_asym_sess_misc *sess) @@ -811,7 +811,7 @@ cpt_ecdsa_verify_prep(struct rte_crypto_ecdsa_op_param *ecdsa, cpt_fill_req_comp_addr(req, caddr); } -static __rte_always_inline int __hot +static __rte_always_inline int __rte_hot cpt_enqueue_ecdsa_op(struct rte_crypto_op *op, struct asym_op_params *params, struct cpt_asym_sess_misc *sess, diff --git a/drivers/common/dpaax/dpaax_iova_table.h b/drivers/common/dpaax/dpaax_iova_table.h index fef97f6dde..fc3b9e7a8f 100644 --- a/drivers/common/dpaax/dpaax_iova_table.h +++ b/drivers/common/dpaax/dpaax_iova_table.h @@ -66,7 +66,7 @@ void dpaax_iova_table_depopulate(void); int dpaax_iova_table_update(phys_addr_t paddr, void *vaddr, size_t length); void dpaax_iova_table_dump(void); -static inline void *dpaax_iova_table_get_va(phys_addr_t paddr) __attribute__((hot)); +static inline void *dpaax_iova_table_get_va(phys_addr_t paddr) __rte_hot; static inline void * dpaax_iova_table_get_va(phys_addr_t paddr) { diff --git a/drivers/common/octeontx2/otx2_common.h b/drivers/common/octeontx2/otx2_common.h index bf5ea86b3a..e62cdea076 100644 --- a/drivers/common/octeontx2/otx2_common.h +++ b/drivers/common/octeontx2/otx2_common.h @@ -49,11 +49,6 @@ (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h)))) #endif -/* Compiler attributes */ -#ifndef __hot -#define __hot __attribute__((hot)) -#endif - /* Intra device related functions */ struct otx2_npa_lf; struct otx2_idev_cfg { diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index 6caa760cc9..561b553dc8 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -3530,7 +3530,7 @@ void dpaa2_sec_stats_reset(struct rte_cryptodev *dev) } } -static void __attribute__((hot)) +static void __rte_hot dpaa2_sec_process_parallel_event(struct qbman_swp *swp, const struct qbman_fd *fd, const struct qbman_result *dq, diff --git a/drivers/crypto/octeontx/otx_cryptodev_ops.c b/drivers/crypto/octeontx/otx_cryptodev_ops.c index 51520e7535..120e7fcc06 100644 --- a/drivers/crypto/octeontx/otx_cryptodev_ops.c +++ b/drivers/crypto/octeontx/otx_cryptodev_ops.c @@ -358,7 +358,7 @@ otx_cpt_asym_session_clear(struct rte_cryptodev *dev, rte_mempool_put(sess_mp, priv); } -static __rte_always_inline int32_t __hot +static __rte_always_inline int32_t __rte_hot otx_cpt_request_enqueue(struct cpt_instance *instance, struct pending_queue *pqueue, void *req) @@ -391,7 +391,7 @@ otx_cpt_request_enqueue(struct cpt_instance *instance, return 0; } -static __rte_always_inline int __hot +static __rte_always_inline int __rte_hot otx_cpt_enq_single_asym(struct cpt_instance *instance, struct rte_crypto_op *op, struct pending_queue *pqueue) @@ -471,7 +471,7 @@ req_fail: return ret; } -static __rte_always_inline int __hot +static __rte_always_inline int __rte_hot otx_cpt_enq_single_sym(struct cpt_instance *instance, struct rte_crypto_op *op, struct pending_queue *pqueue) @@ -513,7 +513,7 @@ otx_cpt_enq_single_sym(struct cpt_instance *instance, return 0; } -static __rte_always_inline int __hot +static __rte_always_inline int __rte_hot otx_cpt_enq_single_sym_sessless(struct cpt_instance *instance, struct rte_crypto_op *op, struct pending_queue *pqueue) @@ -573,7 +573,7 @@ exit: #define OP_TYPE_SYM 0 #define OP_TYPE_ASYM 1 -static __rte_always_inline int __hot +static __rte_always_inline int __rte_hot otx_cpt_enq_single(struct cpt_instance *inst, struct rte_crypto_op *op, struct pending_queue *pqueue, @@ -598,7 +598,7 @@ otx_cpt_enq_single(struct cpt_instance *inst, return -ENOTSUP; } -static __rte_always_inline uint16_t __hot +static __rte_always_inline uint16_t __rte_hot otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops, const uint8_t op_type) { @@ -724,7 +724,7 @@ otx_cpt_asym_dequeue_ecpm_op(struct rte_crypto_ecpm_op_param *ecpm, ecpm->r.y.length = prime_len; } -static __rte_always_inline void __hot +static __rte_always_inline void __rte_hot otx_cpt_asym_post_process(struct rte_crypto_op *cop, struct cpt_request_info *req) { @@ -756,7 +756,7 @@ otx_cpt_asym_post_process(struct rte_crypto_op *cop, } } -static __rte_always_inline void __hot +static __rte_always_inline void __rte_hot otx_cpt_dequeue_post_process(struct rte_crypto_op *cop, uintptr_t *rsp, const uint8_t op_type) { @@ -782,7 +782,7 @@ otx_cpt_dequeue_post_process(struct rte_crypto_op *cop, uintptr_t *rsp, return; } -static __rte_always_inline uint16_t __hot +static __rte_always_inline uint16_t __rte_hot otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops, const uint8_t op_type) { diff --git a/drivers/crypto/octeontx2/otx2_cryptodev_ops.c b/drivers/crypto/octeontx2/otx2_cryptodev_ops.c index a8305f41f4..0ae04c518d 100644 --- a/drivers/crypto/octeontx2/otx2_cryptodev_ops.c +++ b/drivers/crypto/octeontx2/otx2_cryptodev_ops.c @@ -395,7 +395,7 @@ sym_session_clear(int driver_id, struct rte_cryptodev_sym_session *sess) rte_mempool_put(pool, priv); } -static __rte_always_inline int32_t __hot +static __rte_always_inline int32_t __rte_hot otx2_cpt_enqueue_req(const struct otx2_cpt_qp *qp, struct pending_queue *pend_q, struct cpt_request_info *req) @@ -443,7 +443,7 @@ otx2_cpt_enqueue_req(const struct otx2_cpt_qp *qp, return 0; } -static __rte_always_inline int32_t __hot +static __rte_always_inline int32_t __rte_hot otx2_cpt_enqueue_asym(struct otx2_cpt_qp *qp, struct rte_crypto_op *op, struct pending_queue *pend_q) @@ -526,7 +526,7 @@ req_fail: return ret; } -static __rte_always_inline int __hot +static __rte_always_inline int __rte_hot otx2_cpt_enqueue_sym(struct otx2_cpt_qp *qp, struct rte_crypto_op *op, struct pending_queue *pend_q) { @@ -569,7 +569,7 @@ otx2_cpt_enqueue_sym(struct otx2_cpt_qp *qp, struct rte_crypto_op *op, return ret; } -static __rte_always_inline int __hot +static __rte_always_inline int __rte_hot otx2_cpt_enqueue_sym_sessless(struct otx2_cpt_qp *qp, struct rte_crypto_op *op, struct pending_queue *pend_q) { diff --git a/drivers/event/octeontx/ssovf_worker.c b/drivers/event/octeontx/ssovf_worker.c index d940b5dd65..a60c4fc50f 100644 --- a/drivers/event/octeontx/ssovf_worker.c +++ b/drivers/event/octeontx/ssovf_worker.c @@ -91,7 +91,7 @@ ssows_release_event(struct ssows *ws) ssows_swtag_untag(ws); } -__rte_always_inline uint16_t __hot +__rte_always_inline uint16_t __rte_hot ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks) { struct ssows *ws = port; @@ -107,7 +107,7 @@ ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks) } } -__rte_always_inline uint16_t __hot +__rte_always_inline uint16_t __rte_hot ssows_deq_timeout(void *port, struct rte_event *ev, uint64_t timeout_ticks) { struct ssows *ws = port; @@ -125,7 +125,7 @@ ssows_deq_timeout(void *port, struct rte_event *ev, uint64_t timeout_ticks) return ret; } -uint16_t __hot +uint16_t __rte_hot ssows_deq_burst(void *port, struct rte_event ev[], uint16_t nb_events, uint64_t timeout_ticks) { @@ -134,7 +134,7 @@ ssows_deq_burst(void *port, struct rte_event ev[], uint16_t nb_events, return ssows_deq(port, ev, timeout_ticks); } -uint16_t __hot +uint16_t __rte_hot ssows_deq_timeout_burst(void *port, struct rte_event ev[], uint16_t nb_events, uint64_t timeout_ticks) { @@ -143,7 +143,7 @@ ssows_deq_timeout_burst(void *port, struct rte_event ev[], uint16_t nb_events, return ssows_deq_timeout(port, ev, timeout_ticks); } -__rte_always_inline uint16_t __hot +__rte_always_inline uint16_t __rte_hot ssows_enq(void *port, const struct rte_event *ev) { struct ssows *ws = port; @@ -166,14 +166,14 @@ ssows_enq(void *port, const struct rte_event *ev) return ret; } -uint16_t __hot +uint16_t __rte_hot ssows_enq_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { RTE_SET_USED(nb_events); return ssows_enq(port, ev); } -uint16_t __hot +uint16_t __rte_hot ssows_enq_new_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { uint16_t i; @@ -186,7 +186,7 @@ ssows_enq_new_burst(void *port, const struct rte_event ev[], uint16_t nb_events) return nb_events; } -uint16_t __hot +uint16_t __rte_hot ssows_enq_fwd_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { struct ssows *ws = port; diff --git a/drivers/event/octeontx/ssovf_worker.h b/drivers/event/octeontx/ssovf_worker.h index c4f886d63a..009b9c18a5 100644 --- a/drivers/event/octeontx/ssovf_worker.h +++ b/drivers/event/octeontx/ssovf_worker.h @@ -17,10 +17,6 @@ enum { SSO_SYNC_EMPTY }; -#ifndef __hot -#define __hot __attribute__((hot)) -#endif - /* SSO Operations */ static __rte_always_inline struct rte_mbuf * diff --git a/drivers/event/octeontx2/otx2_worker.c b/drivers/event/octeontx2/otx2_worker.c index 8bec59e066..88bac391c7 100644 --- a/drivers/event/octeontx2/otx2_worker.c +++ b/drivers/event/octeontx2/otx2_worker.c @@ -82,7 +82,7 @@ otx2_ssogws_release_event(struct otx2_ssogws *ws) } #define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_deq_ ##name(void *port, struct rte_event *ev, \ uint64_t timeout_ticks) \ { \ @@ -99,7 +99,7 @@ otx2_ssogws_deq_ ##name(void *port, struct rte_event *ev, \ return otx2_ssogws_get_work(ws, ev, flags, ws->lookup_mem); \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_deq_burst_ ##name(void *port, struct rte_event ev[], \ uint16_t nb_events, \ uint64_t timeout_ticks) \ @@ -109,7 +109,7 @@ otx2_ssogws_deq_burst_ ##name(void *port, struct rte_event ev[], \ return otx2_ssogws_deq_ ##name(port, ev, timeout_ticks); \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_deq_timeout_ ##name(void *port, struct rte_event *ev, \ uint64_t timeout_ticks) \ { \ @@ -131,7 +131,7 @@ otx2_ssogws_deq_timeout_ ##name(void *port, struct rte_event *ev, \ return ret; \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_deq_timeout_burst_ ##name(void *port, struct rte_event ev[],\ uint16_t nb_events, \ uint64_t timeout_ticks) \ @@ -141,7 +141,7 @@ otx2_ssogws_deq_timeout_burst_ ##name(void *port, struct rte_event ev[],\ return otx2_ssogws_deq_timeout_ ##name(port, ev, timeout_ticks);\ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_deq_seg_ ##name(void *port, struct rte_event *ev, \ uint64_t timeout_ticks) \ { \ @@ -159,7 +159,7 @@ otx2_ssogws_deq_seg_ ##name(void *port, struct rte_event *ev, \ ws->lookup_mem); \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_deq_seg_burst_ ##name(void *port, struct rte_event ev[], \ uint16_t nb_events, \ uint64_t timeout_ticks) \ @@ -169,7 +169,7 @@ otx2_ssogws_deq_seg_burst_ ##name(void *port, struct rte_event ev[], \ return otx2_ssogws_deq_seg_ ##name(port, ev, timeout_ticks); \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_deq_seg_timeout_ ##name(void *port, struct rte_event *ev, \ uint64_t timeout_ticks) \ { \ @@ -193,7 +193,7 @@ otx2_ssogws_deq_seg_timeout_ ##name(void *port, struct rte_event *ev, \ return ret; \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_deq_seg_timeout_burst_ ##name(void *port, \ struct rte_event ev[], \ uint16_t nb_events, \ @@ -208,7 +208,7 @@ otx2_ssogws_deq_seg_timeout_burst_ ##name(void *port, \ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R -uint16_t __hot +uint16_t __rte_hot otx2_ssogws_enq(void *port, const struct rte_event *ev) { struct otx2_ssogws *ws = port; @@ -230,7 +230,7 @@ otx2_ssogws_enq(void *port, const struct rte_event *ev) return 1; } -uint16_t __hot +uint16_t __rte_hot otx2_ssogws_enq_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { @@ -238,7 +238,7 @@ otx2_ssogws_enq_burst(void *port, const struct rte_event ev[], return otx2_ssogws_enq(port, ev); } -uint16_t __hot +uint16_t __rte_hot otx2_ssogws_enq_new_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { @@ -255,7 +255,7 @@ otx2_ssogws_enq_new_burst(void *port, const struct rte_event ev[], return nb_events; } -uint16_t __hot +uint16_t __rte_hot otx2_ssogws_enq_fwd_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { @@ -268,7 +268,7 @@ otx2_ssogws_enq_fwd_burst(void *port, const struct rte_event ev[], } #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_tx_adptr_enq_ ## name(void *port, struct rte_event ev[], \ uint16_t nb_events) \ { \ @@ -282,7 +282,7 @@ SSO_TX_ADPTR_ENQ_FASTPATH_FUNC #undef T #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_tx_adptr_enq_seg_ ## name(void *port, struct rte_event ev[],\ uint16_t nb_events) \ { \ diff --git a/drivers/event/octeontx2/otx2_worker_dual.c b/drivers/event/octeontx2/otx2_worker_dual.c index 3cba09caff..3d55d921be 100644 --- a/drivers/event/octeontx2/otx2_worker_dual.c +++ b/drivers/event/octeontx2/otx2_worker_dual.c @@ -79,7 +79,7 @@ otx2_ssogws_dual_forward_event(struct otx2_ssogws_dual *ws, } } -uint16_t __hot +uint16_t __rte_hot otx2_ssogws_dual_enq(void *port, const struct rte_event *ev) { struct otx2_ssogws_dual *ws = port; @@ -102,7 +102,7 @@ otx2_ssogws_dual_enq(void *port, const struct rte_event *ev) return 1; } -uint16_t __hot +uint16_t __rte_hot otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { @@ -110,7 +110,7 @@ otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[], return otx2_ssogws_dual_enq(port, ev); } -uint16_t __hot +uint16_t __rte_hot otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { @@ -127,7 +127,7 @@ otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[], return nb_events; } -uint16_t __hot +uint16_t __rte_hot otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[], uint16_t nb_events) { @@ -141,7 +141,7 @@ otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[], } #define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev, \ uint64_t timeout_ticks) \ { \ @@ -166,7 +166,7 @@ otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev, \ return gw; \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_deq_burst_ ##name(void *port, struct rte_event ev[], \ uint16_t nb_events, \ uint64_t timeout_ticks) \ @@ -176,7 +176,7 @@ otx2_ssogws_dual_deq_burst_ ##name(void *port, struct rte_event ev[], \ return otx2_ssogws_dual_deq_ ##name(port, ev, timeout_ticks); \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_deq_timeout_ ##name(void *port, struct rte_event *ev, \ uint64_t timeout_ticks) \ { \ @@ -208,7 +208,7 @@ otx2_ssogws_dual_deq_timeout_ ##name(void *port, struct rte_event *ev, \ return gw; \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port, \ struct rte_event ev[], \ uint16_t nb_events, \ @@ -220,7 +220,7 @@ otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port, \ timeout_ticks); \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev, \ uint64_t timeout_ticks) \ { \ @@ -245,7 +245,7 @@ otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev, \ return gw; \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_deq_seg_burst_ ##name(void *port, \ struct rte_event ev[], \ uint16_t nb_events, \ @@ -257,7 +257,7 @@ otx2_ssogws_dual_deq_seg_burst_ ##name(void *port, \ timeout_ticks); \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port, \ struct rte_event *ev, \ uint64_t timeout_ticks) \ @@ -292,7 +292,7 @@ otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port, \ return gw; \ } \ \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port, \ struct rte_event ev[], \ uint16_t nb_events, \ @@ -308,7 +308,7 @@ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port, \ struct rte_event ev[], \ uint16_t nb_events) \ @@ -325,7 +325,7 @@ SSO_TX_ADPTR_ENQ_FASTPATH_FUNC #undef T #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ -uint16_t __hot \ +uint16_t __rte_hot \ otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port, \ struct rte_event ev[], \ uint16_t nb_events) \ diff --git a/drivers/mempool/octeontx2/otx2_mempool_ops.c b/drivers/mempool/octeontx2/otx2_mempool_ops.c index ac2d618616..162b7f01da 100644 --- a/drivers/mempool/octeontx2/otx2_mempool_ops.c +++ b/drivers/mempool/octeontx2/otx2_mempool_ops.c @@ -7,7 +7,7 @@ #include "otx2_mempool.h" -static int __hot +static int __rte_hot otx2_npa_enq(struct rte_mempool *mp, void * const *obj_table, unsigned int n) { unsigned int index; const uint64_t aura_handle = mp->pool_id; @@ -281,7 +281,7 @@ otx2_npa_clear_alloc(struct rte_mempool *mp, void **obj_table, unsigned int n) } } -static __rte_noinline int __hot +static __rte_noinline int __rte_hot otx2_npa_deq_arm64(struct rte_mempool *mp, void **obj_table, unsigned int n) { const int64_t wdata = npa_lf_aura_handle_to_aura(mp->pool_id); @@ -308,7 +308,7 @@ otx2_npa_deq_arm64(struct rte_mempool *mp, void **obj_table, unsigned int n) #else -static inline int __hot +static inline int __rte_hot otx2_npa_deq(struct rte_mempool *mp, void **obj_table, unsigned int n) { const int64_t wdata = npa_lf_aura_handle_to_aura(mp->pool_id); diff --git a/drivers/net/dpaa2/dpaa2_rxtx.c b/drivers/net/dpaa2/dpaa2_rxtx.c index 4159096155..703f0549ad 100644 --- a/drivers/net/dpaa2/dpaa2_rxtx.c +++ b/drivers/net/dpaa2/dpaa2_rxtx.c @@ -25,7 +25,7 @@ #include "dpaa2_ethdev.h" #include "base/dpaa2_hw_dpni_annot.h" -static inline uint32_t __attribute__((hot)) +static inline uint32_t __rte_hot dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, struct dpaa2_annot_hdr *annotation); @@ -41,7 +41,7 @@ static void enable_tx_tstamp(struct qbman_fd *fd) __rte_unused; DPAA2_RESET_FD_FLC(_fd); \ } while (0) -static inline void __attribute__((hot)) +static inline void __rte_hot dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd) { struct dpaa2_annot_hdr *annotation; @@ -122,7 +122,7 @@ dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd) frc, m->packet_type, m->ol_flags); } -static inline uint32_t __attribute__((hot)) +static inline uint32_t __rte_hot dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf, struct dpaa2_annot_hdr *annotation) { @@ -212,7 +212,7 @@ parse_done: return pkt_type; } -static inline uint32_t __attribute__((hot)) +static inline uint32_t __rte_hot dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr) { struct dpaa2_annot_hdr *annotation = @@ -259,7 +259,7 @@ dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr) return dpaa2_dev_rx_parse_slow(mbuf, annotation); } -static inline struct rte_mbuf *__attribute__((hot)) +static inline struct rte_mbuf *__rte_hot eth_sg_fd_to_mbuf(const struct qbman_fd *fd, int port_id) { @@ -320,7 +320,7 @@ eth_sg_fd_to_mbuf(const struct qbman_fd *fd, return (void *)first_seg; } -static inline struct rte_mbuf *__attribute__((hot)) +static inline struct rte_mbuf *__rte_hot eth_fd_to_mbuf(const struct qbman_fd *fd, int port_id) { @@ -363,7 +363,7 @@ eth_fd_to_mbuf(const struct qbman_fd *fd, return mbuf; } -static int __rte_noinline __attribute__((hot)) +static int __rte_noinline __rte_hot eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf, struct qbman_fd *fd, uint16_t bpid) { @@ -434,7 +434,7 @@ static void eth_mbuf_to_fd(struct rte_mbuf *mbuf, struct qbman_fd *fd, uint16_t bpid) __rte_unused; -static void __rte_noinline __attribute__((hot)) +static void __rte_noinline __rte_hot eth_mbuf_to_fd(struct rte_mbuf *mbuf, struct qbman_fd *fd, uint16_t bpid) { @@ -463,7 +463,7 @@ eth_mbuf_to_fd(struct rte_mbuf *mbuf, } } -static inline int __attribute__((hot)) +static inline int __rte_hot eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf, struct qbman_fd *fd, uint16_t bpid) { @@ -664,7 +664,7 @@ dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) return num_rx; } -void __attribute__((hot)) +void __rte_hot dpaa2_dev_process_parallel_event(struct qbman_swp *swp, const struct qbman_fd *fd, const struct qbman_result *dq, @@ -687,7 +687,7 @@ dpaa2_dev_process_parallel_event(struct qbman_swp *swp, qbman_swp_dqrr_consume(swp, dq); } -void __attribute__((hot)) +void __rte_hot dpaa2_dev_process_atomic_event(struct qbman_swp *swp __rte_unused, const struct qbman_fd *fd, const struct qbman_result *dq, @@ -716,7 +716,7 @@ dpaa2_dev_process_atomic_event(struct qbman_swp *swp __rte_unused, DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf; } -void __attribute__((hot)) +void __rte_hot dpaa2_dev_process_ordered_event(struct qbman_swp *swp, const struct qbman_fd *fd, const struct qbman_result *dq, diff --git a/drivers/net/enetc/enetc_rxtx.c b/drivers/net/enetc/enetc_rxtx.c index e57ecf2d4f..412322523d 100644 --- a/drivers/net/enetc/enetc_rxtx.c +++ b/drivers/net/enetc/enetc_rxtx.c @@ -252,7 +252,7 @@ static inline void enetc_slow_parsing(struct rte_mbuf *m, } -static inline void __attribute__((hot)) +static inline void __rte_hot enetc_dev_rx_parse(struct rte_mbuf *m, uint16_t parse_results) { ENETC_PMD_DP_DEBUG("parse summary = 0x%x ", parse_results); diff --git a/drivers/net/octeontx/octeontx_rxtx.c b/drivers/net/octeontx/octeontx_rxtx.c index 1e201f3228..4a65b340b0 100644 --- a/drivers/net/octeontx/octeontx_rxtx.c +++ b/drivers/net/octeontx/octeontx_rxtx.c @@ -19,7 +19,7 @@ #include "octeontx_rxtx.h" #include "octeontx_logs.h" -uint16_t __hot +uint16_t __rte_hot octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { int count; @@ -43,7 +43,7 @@ octeontx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) return count; /* return number of pkts transmitted */ } -uint16_t __hot +uint16_t __rte_hot octeontx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct octeontx_rxq *rxq; diff --git a/drivers/net/octeontx/octeontx_rxtx.h b/drivers/net/octeontx/octeontx_rxtx.h index d0d73b3046..062c991506 100644 --- a/drivers/net/octeontx/octeontx_rxtx.h +++ b/drivers/net/octeontx/octeontx_rxtx.h @@ -7,10 +7,6 @@ #include -#ifndef __hot -#define __hot __attribute__((hot)) -#endif - /* Packet type table */ #define PTYPE_SIZE OCCTX_PKI_LTYPE_LAST diff --git a/drivers/net/octeontx2/otx2_rx.c b/drivers/net/octeontx2/otx2_rx.c index 47641c7572..ac40704b6e 100644 --- a/drivers/net/octeontx2/otx2_rx.c +++ b/drivers/net/octeontx2/otx2_rx.c @@ -331,14 +331,14 @@ nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts, #endif #define R(name, f6, f5, f4, f3, f2, f1, f0, flags) \ -static uint16_t __rte_noinline __hot \ +static uint16_t __rte_noinline __rte_hot \ otx2_nix_recv_pkts_ ## name(void *rx_queue, \ struct rte_mbuf **rx_pkts, uint16_t pkts) \ { \ return nix_recv_pkts(rx_queue, rx_pkts, pkts, (flags)); \ } \ \ -static uint16_t __rte_noinline __hot \ +static uint16_t __rte_noinline __rte_hot \ otx2_nix_recv_pkts_mseg_ ## name(void *rx_queue, \ struct rte_mbuf **rx_pkts, uint16_t pkts) \ { \ @@ -346,7 +346,7 @@ otx2_nix_recv_pkts_mseg_ ## name(void *rx_queue, \ (flags) | NIX_RX_MULTI_SEG_F); \ } \ \ -static uint16_t __rte_noinline __hot \ +static uint16_t __rte_noinline __rte_hot \ otx2_nix_recv_pkts_vec_ ## name(void *rx_queue, \ struct rte_mbuf **rx_pkts, uint16_t pkts) \ { \ diff --git a/drivers/net/octeontx2/otx2_tx.c b/drivers/net/octeontx2/otx2_tx.c index 4f2036cb45..1af6fa649a 100644 --- a/drivers/net/octeontx2/otx2_tx.c +++ b/drivers/net/octeontx2/otx2_tx.c @@ -947,7 +947,7 @@ nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts, #endif #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ -static uint16_t __rte_noinline __hot \ +static uint16_t __rte_noinline __rte_hot \ otx2_nix_xmit_pkts_ ## name(void *tx_queue, \ struct rte_mbuf **tx_pkts, uint16_t pkts) \ { \ @@ -964,7 +964,7 @@ NIX_TX_FASTPATH_MODES #undef T #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ -static uint16_t __rte_noinline __hot \ +static uint16_t __rte_noinline __rte_hot \ otx2_nix_xmit_pkts_mseg_ ## name(void *tx_queue, \ struct rte_mbuf **tx_pkts, uint16_t pkts) \ { \ @@ -982,7 +982,7 @@ NIX_TX_FASTPATH_MODES #undef T #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags) \ -static uint16_t __rte_noinline __hot \ +static uint16_t __rte_noinline __rte_hot \ otx2_nix_xmit_pkts_vec_ ## name(void *tx_queue, \ struct rte_mbuf **tx_pkts, uint16_t pkts) \ { \ diff --git a/drivers/net/thunderx/nicvf_rxtx.c b/drivers/net/thunderx/nicvf_rxtx.c index 90a6098675..d0ea95f1d1 100644 --- a/drivers/net/thunderx/nicvf_rxtx.c +++ b/drivers/net/thunderx/nicvf_rxtx.c @@ -25,7 +25,7 @@ #include "nicvf_rxtx.h" #include "nicvf_logs.h" -static inline void __hot +static inline void __rte_hot fill_sq_desc_header(union sq_entry_t *entry, struct rte_mbuf *pkt) { /* Local variable sqe to avoid read from sq desc memory*/ @@ -61,7 +61,7 @@ fill_sq_desc_header(union sq_entry_t *entry, struct rte_mbuf *pkt) entry->buff[0] = sqe.buff[0]; } -static inline void __hot +static inline void __rte_hot fill_sq_desc_header_zero_w1(union sq_entry_t *entry, struct rte_mbuf *pkt) { @@ -69,7 +69,7 @@ fill_sq_desc_header_zero_w1(union sq_entry_t *entry, entry->buff[1] = 0ULL; } -void __hot +void __rte_hot nicvf_single_pool_free_xmited_buffers(struct nicvf_txq *sq) { int j = 0; @@ -92,7 +92,7 @@ nicvf_single_pool_free_xmited_buffers(struct nicvf_txq *sq) NICVF_TX_ASSERT(sq->xmit_bufs >= 0); } -void __hot +void __rte_hot nicvf_multi_pool_free_xmited_buffers(struct nicvf_txq *sq) { uint32_t n = 0; @@ -115,7 +115,7 @@ nicvf_multi_pool_free_xmited_buffers(struct nicvf_txq *sq) NICVF_TX_ASSERT(sq->xmit_bufs >= 0); } -static inline uint32_t __hot +static inline uint32_t __rte_hot nicvf_free_tx_desc(struct nicvf_txq *sq) { return ((sq->head - sq->tail - 1) & sq->qlen_mask); @@ -124,7 +124,7 @@ nicvf_free_tx_desc(struct nicvf_txq *sq) /* Send Header + Packet */ #define TX_DESC_PER_PKT 2 -static inline uint32_t __hot +static inline uint32_t __rte_hot nicvf_free_xmitted_buffers(struct nicvf_txq *sq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { @@ -142,7 +142,7 @@ nicvf_free_xmitted_buffers(struct nicvf_txq *sq, struct rte_mbuf **tx_pkts, return free_desc; } -uint16_t __hot +uint16_t __rte_hot nicvf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { int i; @@ -181,7 +181,7 @@ nicvf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) return i; } -uint16_t __hot +uint16_t __rte_hot nicvf_xmit_pkts_multiseg(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { @@ -333,13 +333,13 @@ static const uint32_t ptype_table[16][16] __rte_cache_aligned = { [L3_OTHER][L4_NVGRE] = RTE_PTYPE_TUNNEL_NVGRE, }; -static inline uint32_t __hot +static inline uint32_t __rte_hot nicvf_rx_classify_pkt(cqe_rx_word0_t cqe_rx_w0) { return ptype_table[cqe_rx_w0.l3_type][cqe_rx_w0.l4_type]; } -static inline uint64_t __hot +static inline uint64_t __rte_hot nicvf_set_olflags(const cqe_rx_word0_t cqe_rx_w0) { static const uint64_t flag_table[3] __rte_cache_aligned = { @@ -353,7 +353,7 @@ nicvf_set_olflags(const cqe_rx_word0_t cqe_rx_w0) return flag_table[idx]; } -static inline int __hot +static inline int __rte_hot nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill) { int i; @@ -392,7 +392,7 @@ nicvf_fill_rbdr(struct nicvf_rxq *rxq, int to_fill) return to_fill; } -static inline int32_t __hot +static inline int32_t __rte_hot nicvf_rx_pkts_to_process(struct nicvf_rxq *rxq, uint16_t nb_pkts, int32_t available_space) { @@ -403,7 +403,7 @@ nicvf_rx_pkts_to_process(struct nicvf_rxq *rxq, uint16_t nb_pkts, return RTE_MIN(nb_pkts, available_space); } -static inline void __hot +static inline void __rte_hot nicvf_rx_offload(cqe_rx_word0_t cqe_rx_w0, cqe_rx_word2_t cqe_rx_w2, struct rte_mbuf *pkt) { @@ -484,7 +484,7 @@ nicvf_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, return to_process; } -uint16_t __hot +uint16_t __rte_hot nicvf_recv_pkts_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { @@ -492,7 +492,7 @@ nicvf_recv_pkts_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts, NICVF_RX_OFFLOAD_NONE); } -uint16_t __hot +uint16_t __rte_hot nicvf_recv_pkts_cksum(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { @@ -500,7 +500,7 @@ nicvf_recv_pkts_cksum(void *rx_queue, struct rte_mbuf **rx_pkts, NICVF_RX_OFFLOAD_CKSUM); } -uint16_t __hot +uint16_t __rte_hot nicvf_recv_pkts_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { @@ -508,7 +508,7 @@ nicvf_recv_pkts_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts, NICVF_RX_OFFLOAD_NONE | NICVF_RX_OFFLOAD_VLAN_STRIP); } -uint16_t __hot +uint16_t __rte_hot nicvf_recv_pkts_cksum_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { @@ -516,7 +516,7 @@ nicvf_recv_pkts_cksum_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts, NICVF_RX_OFFLOAD_CKSUM | NICVF_RX_OFFLOAD_VLAN_STRIP); } -static __rte_always_inline uint16_t __hot +static __rte_always_inline uint16_t __rte_hot nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, uint64_t mbuf_phys_off, struct rte_mbuf **rx_pkt, uint8_t rbptr_offset, @@ -572,7 +572,7 @@ nicvf_process_cq_mseg_entry(struct cqe_rx_t *cqe_rx, return nb_segs; } -static __rte_always_inline uint16_t __hot +static __rte_always_inline uint16_t __rte_hot nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts, const uint32_t flag) { @@ -616,7 +616,7 @@ nicvf_recv_pkts_multiseg(void *rx_queue, struct rte_mbuf **rx_pkts, return to_process; } -uint16_t __hot +uint16_t __rte_hot nicvf_recv_pkts_multiseg_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { @@ -624,7 +624,7 @@ nicvf_recv_pkts_multiseg_no_offload(void *rx_queue, struct rte_mbuf **rx_pkts, NICVF_RX_OFFLOAD_NONE); } -uint16_t __hot +uint16_t __rte_hot nicvf_recv_pkts_multiseg_cksum(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { @@ -632,7 +632,7 @@ nicvf_recv_pkts_multiseg_cksum(void *rx_queue, struct rte_mbuf **rx_pkts, NICVF_RX_OFFLOAD_CKSUM); } -uint16_t __hot +uint16_t __rte_hot nicvf_recv_pkts_multiseg_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { @@ -640,7 +640,7 @@ nicvf_recv_pkts_multiseg_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts, NICVF_RX_OFFLOAD_NONE | NICVF_RX_OFFLOAD_VLAN_STRIP); } -uint16_t __hot +uint16_t __rte_hot nicvf_recv_pkts_multiseg_cksum_vlan_strip(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { diff --git a/drivers/net/thunderx/nicvf_rxtx.h b/drivers/net/thunderx/nicvf_rxtx.h index a39808cb6b..1f274b2954 100644 --- a/drivers/net/thunderx/nicvf_rxtx.h +++ b/drivers/net/thunderx/nicvf_rxtx.h @@ -14,10 +14,6 @@ #define NICVF_TX_OFFLOAD_MASK (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK) -#ifndef __hot -#define __hot __attribute__((hot)) -#endif - #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN static inline uint16_t __attribute__((const)) nicvf_frag_num(uint16_t i) @@ -25,7 +21,7 @@ nicvf_frag_num(uint16_t i) return (i & ~3) + 3 - (i & 3); } -static inline void __hot +static inline void __rte_hot fill_sq_desc_gather(union sq_entry_t *entry, struct rte_mbuf *pkt) { /* Local variable sqe to avoid read from sq desc memory*/ @@ -50,7 +46,7 @@ nicvf_frag_num(uint16_t i) return i; } -static inline void __hot +static inline void __rte_hot fill_sq_desc_gather(union sq_entry_t *entry, struct rte_mbuf *pkt) { entry->buff[0] = (uint64_t)SQ_DESC_TYPE_GATHER << 60 | diff --git a/lib/librte_eal/include/rte_common.h b/lib/librte_eal/include/rte_common.h index 9d92f1f36b..0c002b4be6 100644 --- a/lib/librte_eal/include/rte_common.h +++ b/lib/librte_eal/include/rte_common.h @@ -192,6 +192,11 @@ static void __attribute__((destructor(RTE_PRIO(prio)), used)) func(void) */ #define __rte_noinline __attribute__((noinline)) +/** + * Hint function in the hot path + */ +#define __rte_hot __attribute__((hot)) + /*********** Macros for pointer arithmetic ********/ /**