#include "otx2_mempool.h"
-static int __hot
+static int __rte_hot
otx2_npa_enq(struct rte_mempool *mp, void * const *obj_table, unsigned int n)
{
unsigned int index; const uint64_t aura_handle = mp->pool_id;
const uint64_t addr = npa_lf_aura_handle_to_base(aura_handle) +
NPA_LF_AURA_OP_FREE0;
+ /* Ensure mbuf init changes are written before the free pointers
+ * are enqueued to the stack.
+ */
+ rte_io_wmb();
for (index = 0; index < n; index++)
otx2_store_pair((uint64_t)obj_table[index], reg, addr);
}
}
-static __rte_noinline int __hot
+static __rte_noinline int __rte_hot
otx2_npa_deq_arm64(struct rte_mempool *mp, void **obj_table, unsigned int n)
{
const int64_t wdata = npa_lf_aura_handle_to_aura(mp->pool_id);
#else
-static inline int __hot
+static inline int __rte_hot
otx2_npa_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
{
const int64_t wdata = npa_lf_aura_handle_to_aura(mp->pool_id);
struct npa_aq_enq_req *aura_init_req, *pool_init_req;
struct npa_aq_enq_rsp *aura_init_rsp, *pool_init_rsp;
struct otx2_mbox_dev *mdev = &mbox->dev[0];
+ struct otx2_idev_cfg *idev;
int rc, off;
+ idev = otx2_intra_dev_get_cfg();
+ if (idev == NULL)
+ return -ENOMEM;
+
aura_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
aura_init_req->aura_id = aura_id;
return 0;
else
return NPA_LF_ERR_AURA_POOL_INIT;
+
+ if (!(idev->npa_lock_mask & BIT_ULL(aura_id)))
+ return 0;
+
+ aura_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
+ aura_init_req->aura_id = aura_id;
+ aura_init_req->ctype = NPA_AQ_CTYPE_AURA;
+ aura_init_req->op = NPA_AQ_INSTOP_LOCK;
+
+ pool_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
+ if (!pool_init_req) {
+ /* The shared memory buffer can be full.
+ * Flush it and retry
+ */
+ otx2_mbox_msg_send(mbox, 0);
+ rc = otx2_mbox_wait_for_rsp(mbox, 0);
+ if (rc < 0) {
+ otx2_err("Failed to LOCK AURA context");
+ return -ENOMEM;
+ }
+
+ pool_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
+ if (!pool_init_req) {
+ otx2_err("Failed to LOCK POOL context");
+ return -ENOMEM;
+ }
+ }
+ pool_init_req->aura_id = aura_id;
+ pool_init_req->ctype = NPA_AQ_CTYPE_POOL;
+ pool_init_req->op = NPA_AQ_INSTOP_LOCK;
+
+ rc = otx2_mbox_process(mbox);
+ if (rc < 0) {
+ otx2_err("Failed to lock POOL ctx to NDC");
+ return -ENOMEM;
+ }
+
+ return 0;
}
static int
struct npa_aq_enq_rsp *aura_rsp, *pool_rsp;
struct otx2_mbox_dev *mdev = &mbox->dev[0];
struct ndc_sync_op *ndc_req;
+ struct otx2_idev_cfg *idev;
int rc, off;
+ idev = otx2_intra_dev_get_cfg();
+ if (idev == NULL)
+ return -EINVAL;
+
/* Procedure for disabling an aura/pool */
rte_delay_us(10);
npa_lf_aura_op_alloc(aura_handle, 0);
otx2_err("Error on NDC-NPA LF sync, rc %d", rc);
return NPA_LF_ERR_AURA_POOL_FINI;
}
+
+ if (!(idev->npa_lock_mask & BIT_ULL(aura_id)))
+ return 0;
+
+ aura_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
+ aura_req->aura_id = aura_id;
+ aura_req->ctype = NPA_AQ_CTYPE_AURA;
+ aura_req->op = NPA_AQ_INSTOP_UNLOCK;
+
+ rc = otx2_mbox_process(mbox);
+ if (rc < 0) {
+ otx2_err("Failed to unlock AURA ctx to NDC");
+ return -EINVAL;
+ }
+
+ pool_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
+ pool_req->aura_id = aura_id;
+ pool_req->ctype = NPA_AQ_CTYPE_POOL;
+ pool_req->op = NPA_AQ_INSTOP_UNLOCK;
+
+ rc = otx2_mbox_process(mbox);
+ if (rc < 0) {
+ otx2_err("Failed to unlock POOL ctx to NDC");
+ return -EINVAL;
+ }
+
return 0;
}
/* Update aura fields */
aura->pool_addr = pool_id;/* AF will translate to associated poolctx */
aura->ena = 1;
- aura->shift = __builtin_clz(block_count) - 8;
+ aura->shift = rte_log2_u32(block_count);
+ aura->shift = aura->shift < 8 ? 0 : aura->shift - 8;
aura->limit = block_count;
aura->pool_caching = 1;
aura->err_int_ena = BIT(NPA_AURA_ERR_INT_AURA_ADD_OVER);
pool->ena = 1;
pool->buf_size = block_size / OTX2_ALIGN;
pool->stack_max_pages = stack_size;
- pool->shift = __builtin_clz(block_count) - 8;
+ pool->shift = rte_log2_u32(block_count);
+ pool->shift = pool->shift < 8 ? 0 : pool->shift - 8;
pool->ptr_start = 0;
pool->ptr_end = ~0;
pool->stack_caching = 1;
otx2_npa_alloc(struct rte_mempool *mp)
{
uint32_t block_size, block_count;
+ uint64_t aura_handle = 0;
struct otx2_npa_lf *lf;
struct npa_aura_s aura;
struct npa_pool_s pool;
- uint64_t aura_handle;
size_t padding;
int rc;
size_t total_elt_sz;
uint8_t set;
size_t off;
+ int i;
if (iova == RTE_BAD_IOVA)
return -EINVAL;
otx2_npa_dbg("requested objects %"PRIu64", possible objects %"PRIu64"",
(uint64_t)max_objs, (uint64_t)(len / total_elt_sz));
otx2_npa_dbg("L1D set distribution :");
- for (int i = 0; i < OTX2_L1D_NB_SETS; i++)
+ for (i = 0; i < OTX2_L1D_NB_SETS; i++)
otx2_npa_dbg("set[%d] : objects : %"PRIu64"", i,
distribution[i]);