X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fmempool%2Focteontx2%2Fotx2_mempool_ops.c;h=ea4b1c45d2c29347183aa3d50ff0dfcfe2738104;hb=84626a0d61a624dad11614946accc8eebd52353b;hp=97146d1fe4e30bdbf4453ecb31417c09aec2152a;hpb=da5fa4d4d6d1b1fd82dee4830fa5364d6ac9270f;p=dpdk.git diff --git a/drivers/mempool/octeontx2/otx2_mempool_ops.c b/drivers/mempool/octeontx2/otx2_mempool_ops.c index 97146d1fe4..ea4b1c45d2 100644 --- a/drivers/mempool/octeontx2/otx2_mempool_ops.c +++ b/drivers/mempool/octeontx2/otx2_mempool_ops.c @@ -54,233 +54,206 @@ npa_lf_aura_op_search_alloc(const int64_t wdata, int64_t * const addr, return 0; } -/* - * Some versions of the compiler don't have support for __int128_t for - * CASP inline-asm. i.e. if the optimization level is reduced to -O0 the - * CASP restrictions aren't followed and the compiler might end up violation the - * CASP rules. Fix it by explicitly providing ((optimize("-O3"))). - * - * Example: - * ccSPMGzq.s:1648: Error: reg pair must start from even reg at - * operand 1 - `casp x21,x22,x0,x1,[x19]' - */ -static __attribute__((optimize("-O3"))) __rte_noinline int __hot +static __rte_noinline int npa_lf_aura_op_alloc_bulk(const int64_t wdata, int64_t * const addr, unsigned int n, void **obj_table) { - const __uint128_t wdata128 = ((__uint128_t)wdata << 64) | wdata; + register const uint64_t wdata64 __asm("x26") = wdata; + register const uint64_t wdata128 __asm("x27") = wdata; uint64x2_t failed = vdupq_n_u64(~0); switch (n) { case 32: { - __uint128_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9; - __uint128_t t10, t11; - asm volatile ( ".cpu generic+lse\n" - "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t2], %H[t2], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t3], %H[t3], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t4], %H[t4], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t5], %H[t5], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t6], %H[t6], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t7], %H[t7], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t8], %H[t8], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t9], %H[t9], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t10], %H[t10], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t11], %H[t11], %[wdata], %H[wdata], [%[loc]]\n" - "fmov d16, %[t0]\n" - "fmov v16.D[1], %H[t0]\n" - "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n" - "fmov d17, %[t1]\n" - "fmov v17.D[1], %H[t1]\n" - "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n" - "fmov d18, %[t2]\n" - "fmov v18.D[1], %H[t2]\n" - "casp %[t2], %H[t2], %[wdata], %H[wdata], [%[loc]]\n" - "fmov d19, %[t3]\n" - "fmov v19.D[1], %H[t3]\n" - "casp %[t3], %H[t3], %[wdata], %H[wdata], [%[loc]]\n" + "casp x0, x1, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x2, x3, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x4, x5, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x6, x7, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x8, x9, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x10, x11, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x12, x13, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x14, x15, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x16, x17, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x18, x19, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x20, x21, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x22, x23, %[wdata64], %[wdata128], [%[loc]]\n" + "fmov d16, x0\n" + "fmov v16.D[1], x1\n" + "casp x0, x1, %[wdata64], %[wdata128], [%[loc]]\n" + "fmov d17, x2\n" + "fmov v17.D[1], x3\n" + "casp x2, x3, %[wdata64], %[wdata128], [%[loc]]\n" + "fmov d18, x4\n" + "fmov v18.D[1], x5\n" + "casp x4, x5, %[wdata64], %[wdata128], [%[loc]]\n" + "fmov d19, x6\n" + "fmov v19.D[1], x7\n" + "casp x6, x7, %[wdata64], %[wdata128], [%[loc]]\n" "and %[failed].16B, %[failed].16B, v16.16B\n" "and %[failed].16B, %[failed].16B, v17.16B\n" "and %[failed].16B, %[failed].16B, v18.16B\n" "and %[failed].16B, %[failed].16B, v19.16B\n" - "fmov d20, %[t4]\n" - "fmov v20.D[1], %H[t4]\n" - "fmov d21, %[t5]\n" - "fmov v21.D[1], %H[t5]\n" - "fmov d22, %[t6]\n" - "fmov v22.D[1], %H[t6]\n" - "fmov d23, %[t7]\n" - "fmov v23.D[1], %H[t7]\n" + "fmov d20, x8\n" + "fmov v20.D[1], x9\n" + "fmov d21, x10\n" + "fmov v21.D[1], x11\n" + "fmov d22, x12\n" + "fmov v22.D[1], x13\n" + "fmov d23, x14\n" + "fmov v23.D[1], x15\n" "and %[failed].16B, %[failed].16B, v20.16B\n" "and %[failed].16B, %[failed].16B, v21.16B\n" "and %[failed].16B, %[failed].16B, v22.16B\n" "and %[failed].16B, %[failed].16B, v23.16B\n" "st1 { v16.2d, v17.2d, v18.2d, v19.2d}, [%[dst]], 64\n" "st1 { v20.2d, v21.2d, v22.2d, v23.2d}, [%[dst]], 64\n" - "fmov d16, %[t8]\n" - "fmov v16.D[1], %H[t8]\n" - "fmov d17, %[t9]\n" - "fmov v17.D[1], %H[t9]\n" - "fmov d18, %[t10]\n" - "fmov v18.D[1], %H[t10]\n" - "fmov d19, %[t11]\n" - "fmov v19.D[1], %H[t11]\n" + "fmov d16, x16\n" + "fmov v16.D[1], x17\n" + "fmov d17, x18\n" + "fmov v17.D[1], x19\n" + "fmov d18, x20\n" + "fmov v18.D[1], x21\n" + "fmov d19, x22\n" + "fmov v19.D[1], x23\n" "and %[failed].16B, %[failed].16B, v16.16B\n" "and %[failed].16B, %[failed].16B, v17.16B\n" "and %[failed].16B, %[failed].16B, v18.16B\n" "and %[failed].16B, %[failed].16B, v19.16B\n" - "fmov d20, %[t0]\n" - "fmov v20.D[1], %H[t0]\n" - "fmov d21, %[t1]\n" - "fmov v21.D[1], %H[t1]\n" - "fmov d22, %[t2]\n" - "fmov v22.D[1], %H[t2]\n" - "fmov d23, %[t3]\n" - "fmov v23.D[1], %H[t3]\n" + "fmov d20, x0\n" + "fmov v20.D[1], x1\n" + "fmov d21, x2\n" + "fmov v21.D[1], x3\n" + "fmov d22, x4\n" + "fmov v22.D[1], x5\n" + "fmov d23, x6\n" + "fmov v23.D[1], x7\n" "and %[failed].16B, %[failed].16B, v20.16B\n" "and %[failed].16B, %[failed].16B, v21.16B\n" "and %[failed].16B, %[failed].16B, v22.16B\n" "and %[failed].16B, %[failed].16B, v23.16B\n" "st1 { v16.2d, v17.2d, v18.2d, v19.2d}, [%[dst]], 64\n" "st1 { v20.2d, v21.2d, v22.2d, v23.2d}, [%[dst]], 64\n" - : "+Q" (*addr), [failed] "=&w" (failed), - [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2), - [t3] "=&r" (t3), [t4] "=&r" (t4), [t5] "=&r" (t5), - [t6] "=&r" (t6), [t7] "=&r" (t7), [t8] "=&r" (t8), - [t9] "=&r" (t9), [t10] "=&r" (t10), [t11] "=&r" (t11) - : [wdata] "r" (wdata128), [dst] "r" (obj_table), - [loc] "r" (addr) - : "memory", "v16", "v17", "v18", - "v19", "v20", "v21", "v22", "v23" + : "+Q" (*addr), [failed] "=&w" (failed) + : [wdata64] "r" (wdata64), [wdata128] "r" (wdata128), + [dst] "r" (obj_table), [loc] "r" (addr) + : "memory", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", + "x17", "x18", "x19", "x20", "x21", "x22", "x23", "v16", "v17", + "v18", "v19", "v20", "v21", "v22", "v23" ); break; } case 16: { - __uint128_t t0, t1, t2, t3, t4, t5, t6, t7; - asm volatile ( ".cpu generic+lse\n" - "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t2], %H[t2], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t3], %H[t3], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t4], %H[t4], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t5], %H[t5], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t6], %H[t6], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t7], %H[t7], %[wdata], %H[wdata], [%[loc]]\n" - "fmov d16, %[t0]\n" - "fmov v16.D[1], %H[t0]\n" - "fmov d17, %[t1]\n" - "fmov v17.D[1], %H[t1]\n" - "fmov d18, %[t2]\n" - "fmov v18.D[1], %H[t2]\n" - "fmov d19, %[t3]\n" - "fmov v19.D[1], %H[t3]\n" + "casp x0, x1, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x2, x3, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x4, x5, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x6, x7, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x8, x9, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x10, x11, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x12, x13, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x14, x15, %[wdata64], %[wdata128], [%[loc]]\n" + "fmov d16, x0\n" + "fmov v16.D[1], x1\n" + "fmov d17, x2\n" + "fmov v17.D[1], x3\n" + "fmov d18, x4\n" + "fmov v18.D[1], x5\n" + "fmov d19, x6\n" + "fmov v19.D[1], x7\n" "and %[failed].16B, %[failed].16B, v16.16B\n" "and %[failed].16B, %[failed].16B, v17.16B\n" "and %[failed].16B, %[failed].16B, v18.16B\n" "and %[failed].16B, %[failed].16B, v19.16B\n" - "fmov d20, %[t4]\n" - "fmov v20.D[1], %H[t4]\n" - "fmov d21, %[t5]\n" - "fmov v21.D[1], %H[t5]\n" - "fmov d22, %[t6]\n" - "fmov v22.D[1], %H[t6]\n" - "fmov d23, %[t7]\n" - "fmov v23.D[1], %H[t7]\n" + "fmov d20, x8\n" + "fmov v20.D[1], x9\n" + "fmov d21, x10\n" + "fmov v21.D[1], x11\n" + "fmov d22, x12\n" + "fmov v22.D[1], x13\n" + "fmov d23, x14\n" + "fmov v23.D[1], x15\n" "and %[failed].16B, %[failed].16B, v20.16B\n" "and %[failed].16B, %[failed].16B, v21.16B\n" "and %[failed].16B, %[failed].16B, v22.16B\n" "and %[failed].16B, %[failed].16B, v23.16B\n" "st1 { v16.2d, v17.2d, v18.2d, v19.2d}, [%[dst]], 64\n" "st1 { v20.2d, v21.2d, v22.2d, v23.2d}, [%[dst]], 64\n" - : "+Q" (*addr), [failed] "=&w" (failed), - [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2), - [t3] "=&r" (t3), [t4] "=&r" (t4), [t5] "=&r" (t5), - [t6] "=&r" (t6), [t7] "=&r" (t7) - : [wdata] "r" (wdata128), [dst] "r" (obj_table), - [loc] "r" (addr) - : "memory", "v16", "v17", "v18", "v19", - "v20", "v21", "v22", "v23" + : "+Q" (*addr), [failed] "=&w" (failed) + : [wdata64] "r" (wdata64), [wdata128] "r" (wdata128), + [dst] "r" (obj_table), [loc] "r" (addr) + : "memory", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "v16", + "v17", "v18", "v19", "v20", "v21", "v22", "v23" ); break; } case 8: { - __uint128_t t0, t1, t2, t3; - asm volatile ( ".cpu generic+lse\n" - "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t2], %H[t2], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t3], %H[t3], %[wdata], %H[wdata], [%[loc]]\n" - "fmov d16, %[t0]\n" - "fmov v16.D[1], %H[t0]\n" - "fmov d17, %[t1]\n" - "fmov v17.D[1], %H[t1]\n" - "fmov d18, %[t2]\n" - "fmov v18.D[1], %H[t2]\n" - "fmov d19, %[t3]\n" - "fmov v19.D[1], %H[t3]\n" + "casp x0, x1, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x2, x3, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x4, x5, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x6, x7, %[wdata64], %[wdata128], [%[loc]]\n" + "fmov d16, x0\n" + "fmov v16.D[1], x1\n" + "fmov d17, x2\n" + "fmov v17.D[1], x3\n" + "fmov d18, x4\n" + "fmov v18.D[1], x5\n" + "fmov d19, x6\n" + "fmov v19.D[1], x7\n" "and %[failed].16B, %[failed].16B, v16.16B\n" "and %[failed].16B, %[failed].16B, v17.16B\n" "and %[failed].16B, %[failed].16B, v18.16B\n" "and %[failed].16B, %[failed].16B, v19.16B\n" "st1 { v16.2d, v17.2d, v18.2d, v19.2d}, [%[dst]], 64\n" - : "+Q" (*addr), [failed] "=&w" (failed), - [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2), - [t3] "=&r" (t3) - : [wdata] "r" (wdata128), [dst] "r" (obj_table), - [loc] "r" (addr) - : "memory", "v16", "v17", "v18", "v19" + : "+Q" (*addr), [failed] "=&w" (failed) + : [wdata64] "r" (wdata64), [wdata128] "r" (wdata128), + [dst] "r" (obj_table), [loc] "r" (addr) + : "memory", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", + "v16", "v17", "v18", "v19" ); break; } case 4: { - __uint128_t t0, t1; - asm volatile ( ".cpu generic+lse\n" - "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n" - "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n" - "fmov d16, %[t0]\n" - "fmov v16.D[1], %H[t0]\n" - "fmov d17, %[t1]\n" - "fmov v17.D[1], %H[t1]\n" + "casp x0, x1, %[wdata64], %[wdata128], [%[loc]]\n" + "casp x2, x3, %[wdata64], %[wdata128], [%[loc]]\n" + "fmov d16, x0\n" + "fmov v16.D[1], x1\n" + "fmov d17, x2\n" + "fmov v17.D[1], x3\n" "and %[failed].16B, %[failed].16B, v16.16B\n" "and %[failed].16B, %[failed].16B, v17.16B\n" "st1 { v16.2d, v17.2d}, [%[dst]], 32\n" - : "+Q" (*addr), [failed] "=&w" (failed), - [t0] "=&r" (t0), [t1] "=&r" (t1) - : [wdata] "r" (wdata128), [dst] "r" (obj_table), - [loc] "r" (addr) - : "memory", "v16", "v17" + : "+Q" (*addr), [failed] "=&w" (failed) + : [wdata64] "r" (wdata64), [wdata128] "r" (wdata128), + [dst] "r" (obj_table), [loc] "r" (addr) + : "memory", "x0", "x1", "x2", "x3", "v16", "v17" ); break; } case 2: { - __uint128_t t0; - asm volatile ( ".cpu generic+lse\n" - "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n" - "fmov d16, %[t0]\n" - "fmov v16.D[1], %H[t0]\n" + "casp x0, x1, %[wdata64], %[wdata128], [%[loc]]\n" + "fmov d16, x0\n" + "fmov v16.D[1], x1\n" "and %[failed].16B, %[failed].16B, v16.16B\n" "st1 { v16.2d}, [%[dst]], 16\n" - : "+Q" (*addr), [failed] "=&w" (failed), - [t0] "=&r" (t0) - : [wdata] "r" (wdata128), [dst] "r" (obj_table), - [loc] "r" (addr) - : "memory", "v16" + : "+Q" (*addr), [failed] "=&w" (failed) + : [wdata64] "r" (wdata64), [wdata128] "r" (wdata128), + [dst] "r" (obj_table), [loc] "r" (addr) + : "memory", "x0", "x1", "v16" ); break; } @@ -308,7 +281,7 @@ otx2_npa_clear_alloc(struct rte_mempool *mp, void **obj_table, unsigned int n) } } -static inline int __hot +static __rte_noinline int __hot otx2_npa_deq_arm64(struct rte_mempool *mp, void **obj_table, unsigned int n) { const int64_t wdata = npa_lf_aura_handle_to_aura(mp->pool_id); @@ -332,7 +305,8 @@ otx2_npa_deq_arm64(struct rte_mempool *mp, void **obj_table, unsigned int n) return 0; } -#endif + +#else static inline int __hot otx2_npa_deq(struct rte_mempool *mp, void **obj_table, unsigned int n) @@ -359,6 +333,8 @@ otx2_npa_deq(struct rte_mempool *mp, void **obj_table, unsigned int n) return 0; } +#endif + static unsigned int otx2_npa_get_count(const struct rte_mempool *mp) { @@ -379,14 +355,14 @@ npa_lf_aura_pool_init(struct otx2_mbox *mbox, uint32_t aura_id, aura_init_req->aura_id = aura_id; aura_init_req->ctype = NPA_AQ_CTYPE_AURA; aura_init_req->op = NPA_AQ_INSTOP_INIT; - memcpy(&aura_init_req->aura, aura, sizeof(*aura)); + otx2_mbox_memcpy(&aura_init_req->aura, aura, sizeof(*aura)); pool_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox); pool_init_req->aura_id = aura_id; pool_init_req->ctype = NPA_AQ_CTYPE_POOL; pool_init_req->op = NPA_AQ_INSTOP_INIT; - memcpy(&pool_init_req->pool, pool, sizeof(*pool)); + otx2_mbox_memcpy(&pool_init_req->pool, pool, sizeof(*pool)); otx2_mbox_msg_send(mbox, 0); rc = otx2_mbox_wait_for_rsp(mbox, 0); @@ -623,6 +599,40 @@ npa_lf_aura_pool_pair_free(struct otx2_npa_lf *lf, uint64_t aura_handle) return rc; } +static int +npa_lf_aura_range_update_check(uint64_t aura_handle) +{ + uint64_t aura_id = npa_lf_aura_handle_to_aura(aura_handle); + struct otx2_npa_lf *lf = otx2_npa_lf_obj_get(); + struct npa_aura_lim *lim = lf->aura_lim; + __otx2_io struct npa_pool_s *pool; + struct npa_aq_enq_req *req; + struct npa_aq_enq_rsp *rsp; + int rc; + + req = otx2_mbox_alloc_msg_npa_aq_enq(lf->mbox); + + req->aura_id = aura_id; + req->ctype = NPA_AQ_CTYPE_POOL; + req->op = NPA_AQ_INSTOP_READ; + + rc = otx2_mbox_process_msg(lf->mbox, (void *)&rsp); + if (rc) { + otx2_err("Failed to get pool(0x%"PRIx64") context", aura_id); + return rc; + } + + pool = &rsp->pool; + + if (lim[aura_id].ptr_start != pool->ptr_start || + lim[aura_id].ptr_end != pool->ptr_end) { + otx2_err("Range update failed on pool(0x%"PRIx64")", aura_id); + return -ERANGE; + } + + return 0; +} + static int otx2_npa_alloc(struct rte_mempool *mp) { @@ -703,24 +713,15 @@ static ssize_t otx2_npa_calc_mem_size(const struct rte_mempool *mp, uint32_t obj_num, uint32_t pg_shift, size_t *min_chunk_size, size_t *align) { - ssize_t mem_size; + size_t total_elt_sz; - /* - * Simply need space for one more object to be able to - * fulfill alignment requirements. + /* Need space for one more obj on each chunk to fulfill + * alignment requirements. */ - mem_size = rte_mempool_op_calc_mem_size_default(mp, obj_num + 1, - pg_shift, - min_chunk_size, align); - if (mem_size >= 0) { - /* - * Memory area which contains objects must be physically - * contiguous. - */ - *min_chunk_size = mem_size; - } - - return mem_size; + total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; + return rte_mempool_op_calc_mem_size_helper(mp, obj_num, pg_shift, + total_elt_sz, min_chunk_size, + align); } static int @@ -737,7 +738,7 @@ otx2_npa_populate(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size; /* Align object start address to a multiple of total_elt_sz */ - off = total_elt_sz - ((uintptr_t)vaddr % total_elt_sz); + off = total_elt_sz - ((((uintptr_t)vaddr - 1) % total_elt_sz) + 1); if (len < off) return -EINVAL; @@ -748,8 +749,13 @@ otx2_npa_populate(struct rte_mempool *mp, unsigned int max_objs, void *vaddr, npa_lf_aura_op_range_set(mp->pool_id, iova, iova + len); - return rte_mempool_op_populate_default(mp, max_objs, vaddr, iova, len, - obj_cb, obj_cb_arg); + if (npa_lf_aura_range_update_check(mp->pool_id) < 0) + return -EBUSY; + + return rte_mempool_op_populate_helper(mp, + RTE_MEMPOOL_POPULATE_F_ALIGN_OBJ, + max_objs, vaddr, iova, len, + obj_cb, obj_cb_arg); } static struct rte_mempool_ops otx2_npa_ops = {