-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2017 Intel Corporation. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017 Intel Corporation
*/
#include <rte_malloc.h>
+#include <rte_mbuf_pool_ops.h>
#include "cperf_test_common.h"
-static struct rte_mbuf *
-cperf_mbuf_create(struct rte_mempool *mempool,
- uint32_t segment_sz,
- uint32_t segments_nb,
- const struct cperf_options *options,
- const struct cperf_test_vector *test_vector)
+struct obj_params {
+ uint32_t src_buf_offset;
+ uint32_t dst_buf_offset;
+ uint16_t segment_sz;
+ uint16_t headroom_sz;
+ uint16_t data_len;
+ uint16_t segments_nb;
+};
+
+static void
+fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp,
+ void *obj, uint32_t mbuf_offset, uint16_t segment_sz,
+ uint16_t headroom, uint16_t data_len)
{
- struct rte_mbuf *mbuf;
- uint8_t *mbuf_data;
- uint8_t *test_data =
- (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
- test_vector->plaintext.data :
- test_vector->ciphertext.data;
- uint32_t remaining_bytes = options->max_buffer_size;
-
- mbuf = rte_pktmbuf_alloc(mempool);
- if (mbuf == NULL)
- goto error;
-
- mbuf_data = (uint8_t *)rte_pktmbuf_append(mbuf, segment_sz);
- if (mbuf_data == NULL)
- goto error;
-
- if (options->max_buffer_size <= segment_sz) {
- memcpy(mbuf_data, test_data, options->max_buffer_size);
- test_data += options->max_buffer_size;
- remaining_bytes = 0;
- } else {
- memcpy(mbuf_data, test_data, segment_sz);
- test_data += segment_sz;
- remaining_bytes -= segment_sz;
- }
- segments_nb--;
-
- while (remaining_bytes) {
- struct rte_mbuf *m;
-
- m = rte_pktmbuf_alloc(mempool);
- if (m == NULL)
- goto error;
-
- rte_pktmbuf_chain(mbuf, m);
-
- mbuf_data = (uint8_t *)rte_pktmbuf_append(mbuf, segment_sz);
- if (mbuf_data == NULL)
- goto error;
-
- if (remaining_bytes <= segment_sz) {
- memcpy(mbuf_data, test_data, remaining_bytes);
- remaining_bytes = 0;
- test_data += remaining_bytes;
- } else {
- memcpy(mbuf_data, test_data, segment_sz);
- remaining_bytes -= segment_sz;
- test_data += segment_sz;
- }
- segments_nb--;
- }
-
- /*
- * If there was not enough room for the digest at the end
- * of the last segment, allocate a new one
- */
- if (segments_nb != 0) {
- struct rte_mbuf *m;
- m = rte_pktmbuf_alloc(mempool);
-
- if (m == NULL)
- goto error;
-
- rte_pktmbuf_chain(mbuf, m);
- mbuf_data = (uint8_t *)rte_pktmbuf_append(mbuf, segment_sz);
- if (mbuf_data == NULL)
- goto error;
- }
+ uint32_t mbuf_hdr_size = sizeof(struct rte_mbuf);
+
+ /* start of buffer is after mbuf structure and priv data */
+ m->priv_size = 0;
+ m->buf_addr = (char *)m + mbuf_hdr_size;
+ m->buf_iova = rte_mempool_virt2iova(obj) +
+ mbuf_offset + mbuf_hdr_size;
+ m->buf_len = segment_sz;
+ m->data_len = data_len;
+
+ /* Use headroom specified for the buffer */
+ m->data_off = headroom;
+
+ /* init some constant fields */
+ m->pool = mp;
+ m->nb_segs = 1;
+ m->port = 0xff;
+ rte_mbuf_refcnt_set(m, 1);
+ m->next = NULL;
+}
- return mbuf;
-error:
- if (mbuf != NULL)
- rte_pktmbuf_free(mbuf);
+static void
+fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp,
+ void *obj, uint32_t mbuf_offset, uint16_t segment_sz,
+ uint16_t headroom, uint16_t data_len, uint16_t segments_nb)
+{
+ uint16_t mbuf_hdr_size = sizeof(struct rte_mbuf);
+ uint16_t remaining_segments = segments_nb;
+ struct rte_mbuf *next_mbuf;
+ rte_iova_t next_seg_phys_addr = rte_mempool_virt2iova(obj) +
+ mbuf_offset + mbuf_hdr_size;
+
+ do {
+ /* start of buffer is after mbuf structure and priv data */
+ m->priv_size = 0;
+ m->buf_addr = (char *)m + mbuf_hdr_size;
+ m->buf_iova = next_seg_phys_addr;
+ next_seg_phys_addr += mbuf_hdr_size + segment_sz;
+ m->buf_len = segment_sz;
+ m->data_len = data_len;
+
+ /* Use headroom specified for the buffer */
+ m->data_off = headroom;
+
+ /* init some constant fields */
+ m->pool = mp;
+ m->nb_segs = segments_nb;
+ m->port = 0xff;
+ rte_mbuf_refcnt_set(m, 1);
+ next_mbuf = (struct rte_mbuf *) ((uint8_t *) m +
+ mbuf_hdr_size + segment_sz);
+ m->next = next_mbuf;
+ m = next_mbuf;
+ remaining_segments--;
+
+ } while (remaining_segments > 0);
+
+ m->next = NULL;
+}
- return NULL;
+static void
+mempool_obj_init(struct rte_mempool *mp,
+ void *opaque_arg,
+ void *obj,
+ __attribute__((unused)) unsigned int i)
+{
+ struct obj_params *params = opaque_arg;
+ struct rte_crypto_op *op = obj;
+ struct rte_mbuf *m = (struct rte_mbuf *) ((uint8_t *) obj +
+ params->src_buf_offset);
+ /* Set crypto operation */
+ op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
+ op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
+ op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
+ op->phys_addr = rte_mem_virt2iova(obj);
+ op->mempool = mp;
+
+ /* Set source buffer */
+ op->sym->m_src = m;
+ if (params->segments_nb == 1)
+ fill_single_seg_mbuf(m, mp, obj, params->src_buf_offset,
+ params->segment_sz, params->headroom_sz,
+ params->data_len);
+ else
+ fill_multi_seg_mbuf(m, mp, obj, params->src_buf_offset,
+ params->segment_sz, params->headroom_sz,
+ params->data_len, params->segments_nb);
+
+
+ /* Set destination buffer */
+ if (params->dst_buf_offset) {
+ m = (struct rte_mbuf *) ((uint8_t *) obj +
+ params->dst_buf_offset);
+ fill_single_seg_mbuf(m, mp, obj, params->dst_buf_offset,
+ params->segment_sz, params->headroom_sz,
+ params->data_len);
+ op->sym->m_dst = m;
+ } else
+ op->sym->m_dst = NULL;
}
int
cperf_alloc_common_memory(const struct cperf_options *options,
const struct cperf_test_vector *test_vector,
- uint8_t dev_id, size_t extra_op_priv_size,
- struct rte_mempool **pkt_mbuf_pool_in,
- struct rte_mempool **pkt_mbuf_pool_out,
- struct rte_mbuf ***mbufs_in,
- struct rte_mbuf ***mbufs_out,
- struct rte_mempool **crypto_op_pool)
+ uint8_t dev_id, uint16_t qp_id,
+ size_t extra_op_priv_size,
+ uint32_t *src_buf_offset,
+ uint32_t *dst_buf_offset,
+ struct rte_mempool **pool)
{
- unsigned int mbuf_idx = 0;
+ const char *mp_ops_name;
char pool_name[32] = "";
+ int ret;
- snprintf(pool_name, sizeof(pool_name), "cperf_pool_in_cdev_%d",
- dev_id);
+ /* Calculate the object size */
+ uint16_t crypto_op_size = sizeof(struct rte_crypto_op) +
+ sizeof(struct rte_crypto_sym_op);
+ uint16_t crypto_op_private_size;
+ /*
+ * If doing AES-CCM, IV field needs to be 16 bytes long,
+ * and AAD field needs to be long enough to have 18 bytes,
+ * plus the length of the AAD, and all rounded to a
+ * multiple of 16 bytes.
+ */
+ if (options->aead_algo == RTE_CRYPTO_AEAD_AES_CCM) {
+ crypto_op_private_size = extra_op_priv_size +
+ test_vector->cipher_iv.length +
+ test_vector->auth_iv.length +
+ RTE_ALIGN_CEIL(test_vector->aead_iv.length, 16) +
+ RTE_ALIGN_CEIL(options->aead_aad_sz + 18, 16);
+ } else {
+ crypto_op_private_size = extra_op_priv_size +
+ test_vector->cipher_iv.length +
+ test_vector->auth_iv.length +
+ test_vector->aead_iv.length +
+ options->aead_aad_sz;
+ }
+ uint16_t crypto_op_total_size = crypto_op_size +
+ crypto_op_private_size;
+ uint16_t crypto_op_total_size_padded =
+ RTE_CACHE_LINE_ROUNDUP(crypto_op_total_size);
+ uint32_t mbuf_size = sizeof(struct rte_mbuf) + options->segment_sz;
uint32_t max_size = options->max_buffer_size + options->digest_sz;
uint16_t segments_nb = (max_size % options->segment_sz) ?
(max_size / options->segment_sz) + 1 :
max_size / options->segment_sz;
-
- *pkt_mbuf_pool_in = rte_pktmbuf_pool_create(pool_name,
- options->pool_sz * segments_nb, 0, 0,
- RTE_PKTMBUF_HEADROOM + options->segment_sz,
- rte_socket_id());
-
- if (*pkt_mbuf_pool_in == NULL)
- return -1;
-
- /* Generate mbufs_in with plaintext populated for test */
- *mbufs_in = (struct rte_mbuf **)rte_malloc(NULL,
- (sizeof(struct rte_mbuf *) * options->pool_sz), 0);
-
- for (mbuf_idx = 0; mbuf_idx < options->pool_sz; mbuf_idx++) {
- (*mbufs_in)[mbuf_idx] = cperf_mbuf_create(
- *pkt_mbuf_pool_in,
- options->segment_sz,
- segments_nb,
- options, test_vector);
- if ((*mbufs_in)[mbuf_idx] == NULL)
- return -1;
- }
-
- *mbufs_out = (struct rte_mbuf **)rte_zmalloc(NULL,
- (sizeof(struct rte_mbuf *) *
- options->pool_sz), 0);
-
- if (options->out_of_place == 1) {
- snprintf(pool_name, sizeof(pool_name), "cperf_pool_out_cdev_%d",
- dev_id);
-
- *pkt_mbuf_pool_out = rte_pktmbuf_pool_create(
- pool_name, options->pool_sz, 0, 0,
- RTE_PKTMBUF_HEADROOM + max_size,
- rte_socket_id());
-
- if (*pkt_mbuf_pool_out == NULL)
- return -1;
-
- for (mbuf_idx = 0; mbuf_idx < options->pool_sz; mbuf_idx++) {
- (*mbufs_out)[mbuf_idx] = cperf_mbuf_create(
- *pkt_mbuf_pool_out, max_size,
- 1, options, test_vector);
- if ((*mbufs_out)[mbuf_idx] == NULL)
- return -1;
- }
+ uint32_t obj_size = crypto_op_total_size_padded +
+ (mbuf_size * segments_nb);
+
+ snprintf(pool_name, sizeof(pool_name), "pool_cdev_%u_qp_%u",
+ dev_id, qp_id);
+
+ *src_buf_offset = crypto_op_total_size_padded;
+
+ struct obj_params params = {
+ .segment_sz = options->segment_sz,
+ .headroom_sz = options->headroom_sz,
+ /* Data len = segment size - (headroom + tailroom) */
+ .data_len = options->segment_sz -
+ options->headroom_sz -
+ options->tailroom_sz,
+ .segments_nb = segments_nb,
+ .src_buf_offset = crypto_op_total_size_padded,
+ .dst_buf_offset = 0
+ };
+
+ if (options->out_of_place) {
+ *dst_buf_offset = *src_buf_offset +
+ (mbuf_size * segments_nb);
+ params.dst_buf_offset = *dst_buf_offset;
+ /* Destination buffer will be one segment only */
+ obj_size += max_size;
}
- snprintf(pool_name, sizeof(pool_name), "cperf_op_pool_cdev_%d",
+ *pool = rte_mempool_create_empty(pool_name,
+ options->pool_sz, obj_size, 512, 0,
+ rte_socket_id(), 0);
+ if (*pool == NULL) {
+ RTE_LOG(ERR, USER1,
+ "Cannot allocate mempool for device %u\n",
dev_id);
-
- uint16_t priv_size = RTE_ALIGN_CEIL(test_vector->cipher_iv.length +
- test_vector->auth_iv.length + test_vector->aead_iv.length +
- extra_op_priv_size, 16) +
- RTE_ALIGN_CEIL(options->aead_aad_sz, 16);
-
- *crypto_op_pool = rte_crypto_op_pool_create(pool_name,
- RTE_CRYPTO_OP_TYPE_SYMMETRIC, options->pool_sz,
- 512, priv_size, rte_socket_id());
- if (*crypto_op_pool == NULL)
return -1;
-
- return 0;
-}
-
-void
-cperf_free_common_memory(const struct cperf_options *options,
- struct rte_mempool *pkt_mbuf_pool_in,
- struct rte_mempool *pkt_mbuf_pool_out,
- struct rte_mbuf **mbufs_in,
- struct rte_mbuf **mbufs_out,
- struct rte_mempool *crypto_op_pool)
-{
- uint32_t i = 0;
-
- if (mbufs_in) {
- while (mbufs_in[i] != NULL &&
- i < options->pool_sz)
- rte_pktmbuf_free(mbufs_in[i++]);
-
- rte_free(mbufs_in);
}
- if (mbufs_out) {
- i = 0;
- while (mbufs_out[i] != NULL
- && i < options->pool_sz)
- rte_pktmbuf_free(mbufs_out[i++]);
+ mp_ops_name = rte_mbuf_best_mempool_ops();
- rte_free(mbufs_out);
+ ret = rte_mempool_set_ops_byname(*pool,
+ mp_ops_name, NULL);
+ if (ret != 0) {
+ RTE_LOG(ERR, USER1,
+ "Error setting mempool handler for device %u\n",
+ dev_id);
+ return -1;
}
- if (pkt_mbuf_pool_in)
- rte_mempool_free(pkt_mbuf_pool_in);
+ ret = rte_mempool_populate_default(*pool);
+ if (ret < 0) {
+ RTE_LOG(ERR, USER1,
+ "Error populating mempool for device %u\n",
+ dev_id);
+ return -1;
+ }
- if (pkt_mbuf_pool_out)
- rte_mempool_free(pkt_mbuf_pool_out);
+ rte_mempool_obj_iter(*pool, mempool_obj_init, (void *)¶ms);
- if (crypto_op_pool)
- rte_mempool_free(crypto_op_pool);
+ return 0;
}