/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2018 Cavium, Inc
*/
+#include <assert.h>
#include <string.h>
+#include <unistd.h>
#include <rte_branch_prediction.h>
#include <rte_common.h>
+#include <cryptodev_pmd.h>
+#include <rte_errno.h>
+#include <rte_mempool.h>
+#include <rte_memzone.h>
+#include <rte_string_fns.h>
#include "otx_cryptodev_hw_access.h"
#include "otx_cryptodev_mbox.h"
#include "cpt_pmd_logs.h"
+#include "cpt_pmd_ops_helper.h"
#include "cpt_hw_types.h"
+#define METABUF_POOL_CACHE_SIZE 512
+
/*
* VF HAL functions
* Access its own BAR0/4 registers by passing VF number as 0.
CPTX_VQX_MISC_INT(0, 0), vqx_misc_int.u);
}
+/* Write to VQX_CTL register
+ */
+static void
+otx_cpt_write_vq_ctl(struct cpt_vf *cptvf, bool val)
+{
+ cptx_vqx_ctl_t vqx_ctl;
+
+ vqx_ctl.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_CTL(0, 0));
+ vqx_ctl.s.ena = val;
+ CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_CTL(0, 0), vqx_ctl.u);
+}
+
+/* Write to VQX_INPROG register
+ */
+static void
+otx_cpt_write_vq_inprog(struct cpt_vf *cptvf, uint8_t val)
+{
+ cptx_vqx_inprog_t vqx_inprg;
+
+ vqx_inprg.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_INPROG(0, 0));
+ vqx_inprg.s.inflight = val;
+ CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_INPROG(0, 0), vqx_inprg.u);
+}
+
+/* Write to VQX_DONE_WAIT NUMWAIT register
+ */
+static void
+otx_cpt_write_vq_done_numwait(struct cpt_vf *cptvf, uint32_t val)
+{
+ cptx_vqx_done_wait_t vqx_dwait;
+
+ vqx_dwait.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_DONE_WAIT(0, 0));
+ vqx_dwait.s.num_wait = val;
+ CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_DONE_WAIT(0, 0), vqx_dwait.u);
+}
+
+/* Write to VQX_DONE_WAIT NUM_WAIT register
+ */
+static void
+otx_cpt_write_vq_done_timewait(struct cpt_vf *cptvf, uint16_t val)
+{
+ cptx_vqx_done_wait_t vqx_dwait;
+
+ vqx_dwait.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_DONE_WAIT(0, 0));
+ vqx_dwait.s.time_wait = val;
+ CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_DONE_WAIT(0, 0), vqx_dwait.u);
+}
+
+/* Write to VQX_SADDR register
+ */
+static void
+otx_cpt_write_vq_saddr(struct cpt_vf *cptvf, uint64_t val)
+{
+ cptx_vqx_saddr_t vqx_saddr;
+
+ vqx_saddr.u = val;
+ CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
+ CPTX_VQX_SADDR(0, 0), vqx_saddr.u);
+}
+
+static void
+otx_cpt_vfvq_init(struct cpt_vf *cptvf)
+{
+ uint64_t base_addr = 0;
+
+ /* Disable the VQ */
+ otx_cpt_write_vq_ctl(cptvf, 0);
+
+ /* Reset the doorbell */
+ otx_cpt_write_vq_doorbell(cptvf, 0);
+ /* Clear inflight */
+ otx_cpt_write_vq_inprog(cptvf, 0);
+
+ /* Write VQ SADDR */
+ base_addr = (uint64_t)(cptvf->cqueue.chead[0].dma_addr);
+ otx_cpt_write_vq_saddr(cptvf, base_addr);
+
+ /* Configure timerhold / coalescence */
+ otx_cpt_write_vq_done_timewait(cptvf, CPT_TIMER_THOLD);
+ otx_cpt_write_vq_done_numwait(cptvf, CPT_COUNT_THOLD);
+
+ /* Enable the VQ */
+ otx_cpt_write_vq_ctl(cptvf, 1);
+}
+
+static int
+cpt_vq_init(struct cpt_vf *cptvf, uint8_t group)
+{
+ int err;
+
+ /* Convey VQ LEN to PF */
+ err = otx_cpt_send_vq_size_msg(cptvf);
+ if (err) {
+ CPT_LOG_ERR("%s: PF not responding to QLEN msg",
+ cptvf->dev_name);
+ err = -EBUSY;
+ goto cleanup;
+ }
+
+ /* CPT VF device initialization */
+ otx_cpt_vfvq_init(cptvf);
+
+ /* Send msg to PF to assign currnet Q to required group */
+ cptvf->vfgrp = group;
+ err = otx_cpt_send_vf_grp_msg(cptvf, group);
+ if (err) {
+ CPT_LOG_ERR("%s: PF not responding to VF_GRP msg",
+ cptvf->dev_name);
+ err = -EBUSY;
+ goto cleanup;
+ }
+
+ CPT_LOG_DP_DEBUG("%s: %s done", cptvf->dev_name, __func__);
+ return 0;
+
+cleanup:
+ return err;
+}
+
void
otx_cpt_poll_misc(struct cpt_vf *cptvf)
{
/* Bar0 base address */
cptvf->reg_base = reg_base;
- strncpy(cptvf->dev_name, name, 32);
+
+ /* Save device name */
+ strlcpy(cptvf->dev_name, name, (sizeof(cptvf->dev_name)));
cptvf->pdev = pdev;
return -1;
}
+ /* Gets device type */
+ if (otx_cpt_get_dev_type(cptvf)) {
+ CPT_LOG_ERR("Failed to get device type");
+ return -1;
+ }
+
return 0;
}
return 0;
}
+
+static int
+otx_cpt_metabuf_mempool_create(const struct rte_cryptodev *dev,
+ struct cpt_instance *instance, uint8_t qp_id,
+ unsigned int nb_elements)
+{
+ char mempool_name[RTE_MEMPOOL_NAMESIZE];
+ struct cpt_qp_meta_info *meta_info;
+ struct rte_mempool *pool;
+ int max_mlen = 0;
+ int sg_mlen = 0;
+ int lb_mlen = 0;
+ int mb_pool_sz;
+ int ret;
+
+ /*
+ * Calculate metabuf length required. The 'crypto_octeontx' device
+ * would be either SYMMETRIC or ASYMMETRIC.
+ */
+
+ if (dev->feature_flags & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) {
+
+ /* Get meta len for scatter gather mode */
+ sg_mlen = cpt_pmd_ops_helper_get_mlen_sg_mode();
+
+ /* Extra 32B saved for future considerations */
+ sg_mlen += 4 * sizeof(uint64_t);
+
+ /* Get meta len for linear buffer (direct) mode */
+ lb_mlen = cpt_pmd_ops_helper_get_mlen_direct_mode();
+
+ /* Extra 32B saved for future considerations */
+ lb_mlen += 4 * sizeof(uint64_t);
+
+ /* Check max requirement for meta buffer */
+ max_mlen = RTE_MAX(lb_mlen, sg_mlen);
+ } else {
+
+ /* Asymmetric device */
+
+ /* Get meta len for asymmetric operations */
+ max_mlen = cpt_pmd_ops_helper_asym_get_mlen();
+ }
+
+ /* Allocate mempool */
+
+ snprintf(mempool_name, RTE_MEMPOOL_NAMESIZE, "otx_cpt_mb_%u:%u",
+ dev->data->dev_id, qp_id);
+
+ mb_pool_sz = RTE_MAX(nb_elements, (METABUF_POOL_CACHE_SIZE * rte_lcore_count()));
+
+ pool = rte_mempool_create_empty(mempool_name, mb_pool_sz, max_mlen,
+ METABUF_POOL_CACHE_SIZE, 0,
+ rte_socket_id(), 0);
+
+ if (pool == NULL) {
+ CPT_LOG_ERR("Could not create mempool for metabuf");
+ return rte_errno;
+ }
+
+ ret = rte_mempool_set_ops_byname(pool, RTE_MBUF_DEFAULT_MEMPOOL_OPS,
+ NULL);
+ if (ret) {
+ CPT_LOG_ERR("Could not set mempool ops");
+ goto mempool_free;
+ }
+
+ ret = rte_mempool_populate_default(pool);
+ if (ret <= 0) {
+ CPT_LOG_ERR("Could not populate metabuf pool");
+ goto mempool_free;
+ }
+
+ meta_info = &instance->meta_info;
+
+ meta_info->pool = pool;
+ meta_info->lb_mlen = lb_mlen;
+ meta_info->sg_mlen = sg_mlen;
+
+ return 0;
+
+mempool_free:
+ rte_mempool_free(pool);
+ return ret;
+}
+
+static void
+otx_cpt_metabuf_mempool_destroy(struct cpt_instance *instance)
+{
+ struct cpt_qp_meta_info *meta_info = &instance->meta_info;
+
+ rte_mempool_free(meta_info->pool);
+
+ meta_info->pool = NULL;
+ meta_info->lb_mlen = 0;
+ meta_info->sg_mlen = 0;
+}
+
+int
+otx_cpt_get_resource(const struct rte_cryptodev *dev, uint8_t group,
+ struct cpt_instance **instance, uint16_t qp_id)
+{
+ int ret = -ENOENT, len, qlen, i;
+ int chunk_len, chunks, chunk_size;
+ struct cpt_vf *cptvf = dev->data->dev_private;
+ struct cpt_instance *cpt_instance;
+ struct command_chunk *chunk_head = NULL, *chunk_prev = NULL;
+ struct command_chunk *chunk = NULL;
+ uint8_t *mem;
+ const struct rte_memzone *rz;
+ uint64_t dma_addr = 0, alloc_len, used_len;
+ uint64_t *next_ptr;
+ uint64_t pg_sz = sysconf(_SC_PAGESIZE);
+
+ CPT_LOG_DP_DEBUG("Initializing cpt resource %s", cptvf->dev_name);
+
+ cpt_instance = &cptvf->instance;
+
+ memset(&cptvf->cqueue, 0, sizeof(cptvf->cqueue));
+ memset(&cptvf->pqueue, 0, sizeof(cptvf->pqueue));
+
+ /* Chunks are of fixed size buffers */
+
+ qlen = DEFAULT_CMD_QLEN;
+ chunks = DEFAULT_CMD_QCHUNKS;
+ chunk_len = DEFAULT_CMD_QCHUNK_SIZE;
+ /* Chunk size includes 8 bytes of next chunk ptr */
+ chunk_size = chunk_len * CPT_INST_SIZE + CPT_NEXT_CHUNK_PTR_SIZE;
+
+ /* For command chunk structures */
+ len = chunks * RTE_ALIGN(sizeof(struct command_chunk), 8);
+
+ /* For pending queue */
+ len += qlen * RTE_ALIGN(sizeof(cptvf->pqueue.rid_queue[0]), 8);
+
+ /* So that instruction queues start as pg size aligned */
+ len = RTE_ALIGN(len, pg_sz);
+
+ /* For Instruction queues */
+ len += chunks * RTE_ALIGN(chunk_size, 128);
+
+ /* Wastage after instruction queues */
+ len = RTE_ALIGN(len, pg_sz);
+
+ rz = rte_memzone_reserve_aligned(cptvf->dev_name, len, cptvf->node,
+ RTE_MEMZONE_SIZE_HINT_ONLY |
+ RTE_MEMZONE_256MB,
+ RTE_CACHE_LINE_SIZE);
+ if (!rz) {
+ ret = rte_errno;
+ goto exit;
+ }
+
+ mem = rz->addr;
+ dma_addr = rz->iova;
+ alloc_len = len;
+
+ memset(mem, 0, len);
+
+ cpt_instance->rsvd = (uintptr_t)rz;
+
+ ret = otx_cpt_metabuf_mempool_create(dev, cpt_instance, qp_id, qlen);
+ if (ret) {
+ CPT_LOG_ERR("Could not create mempool for metabuf");
+ goto memzone_free;
+ }
+
+ /* Pending queue setup */
+ cptvf->pqueue.rid_queue = (void **)mem;
+
+ mem += qlen * RTE_ALIGN(sizeof(cptvf->pqueue.rid_queue[0]), 8);
+ len -= qlen * RTE_ALIGN(sizeof(cptvf->pqueue.rid_queue[0]), 8);
+ dma_addr += qlen * RTE_ALIGN(sizeof(cptvf->pqueue.rid_queue[0]), 8);
+
+ /* Alignment wastage */
+ used_len = alloc_len - len;
+ mem += RTE_ALIGN(used_len, pg_sz) - used_len;
+ len -= RTE_ALIGN(used_len, pg_sz) - used_len;
+ dma_addr += RTE_ALIGN(used_len, pg_sz) - used_len;
+
+ /* Init instruction queues */
+ chunk_head = &cptvf->cqueue.chead[0];
+ i = qlen;
+
+ chunk_prev = NULL;
+ for (i = 0; i < DEFAULT_CMD_QCHUNKS; i++) {
+ int csize;
+
+ chunk = &cptvf->cqueue.chead[i];
+ chunk->head = mem;
+ chunk->dma_addr = dma_addr;
+
+ csize = RTE_ALIGN(chunk_size, 128);
+ mem += csize;
+ dma_addr += csize;
+ len -= csize;
+
+ if (chunk_prev) {
+ next_ptr = (uint64_t *)(chunk_prev->head +
+ chunk_size - 8);
+ *next_ptr = (uint64_t)chunk->dma_addr;
+ }
+ chunk_prev = chunk;
+ }
+ /* Circular loop */
+ next_ptr = (uint64_t *)(chunk_prev->head + chunk_size - 8);
+ *next_ptr = (uint64_t)chunk_head->dma_addr;
+
+ assert(!len);
+
+ /* This is used for CPT(0)_PF_Q(0..15)_CTL.size config */
+ cptvf->qsize = chunk_size / 8;
+ cptvf->cqueue.qhead = chunk_head->head;
+ cptvf->cqueue.idx = 0;
+ cptvf->cqueue.cchunk = 0;
+
+ if (cpt_vq_init(cptvf, group)) {
+ CPT_LOG_ERR("Failed to initialize CPT VQ of device %s",
+ cptvf->dev_name);
+ ret = -EBUSY;
+ goto mempool_destroy;
+ }
+
+ *instance = cpt_instance;
+
+ CPT_LOG_DP_DEBUG("Crypto device (%s) initialized", cptvf->dev_name);
+
+ return 0;
+
+mempool_destroy:
+ otx_cpt_metabuf_mempool_destroy(cpt_instance);
+memzone_free:
+ rte_memzone_free(rz);
+exit:
+ *instance = NULL;
+ return ret;
+}
+
+int
+otx_cpt_put_resource(struct cpt_instance *instance)
+{
+ struct cpt_vf *cptvf = (struct cpt_vf *)instance;
+ struct rte_memzone *rz;
+
+ if (!cptvf) {
+ CPT_LOG_ERR("Invalid CPTVF handle");
+ return -EINVAL;
+ }
+
+ CPT_LOG_DP_DEBUG("Releasing cpt device %s", cptvf->dev_name);
+
+ otx_cpt_metabuf_mempool_destroy(instance);
+
+ rz = (struct rte_memzone *)instance->rsvd;
+ rte_memzone_free(rz);
+ return 0;
+}
+
+int
+otx_cpt_start_device(void *dev)
+{
+ int rc;
+ struct cpt_vf *cptvf = (struct cpt_vf *)dev;
+
+ rc = otx_cpt_send_vf_up(cptvf);
+ if (rc) {
+ CPT_LOG_ERR("Failed to mark CPT VF device %s UP, rc = %d",
+ cptvf->dev_name, rc);
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+void
+otx_cpt_stop_device(void *dev)
+{
+ int rc;
+ uint32_t pending, retries = 5;
+ struct cpt_vf *cptvf = (struct cpt_vf *)dev;
+
+ /* Wait for pending entries to complete */
+ pending = otx_cpt_read_vq_doorbell(cptvf);
+ while (pending) {
+ CPT_LOG_DP_DEBUG("%s: Waiting for pending %u cmds to complete",
+ cptvf->dev_name, pending);
+ sleep(1);
+ pending = otx_cpt_read_vq_doorbell(cptvf);
+ retries--;
+ if (!retries)
+ break;
+ }
+
+ if (!retries && pending) {
+ CPT_LOG_ERR("%s: Timeout waiting for commands(%u)",
+ cptvf->dev_name, pending);
+ return;
+ }
+
+ rc = otx_cpt_send_vf_down(cptvf);
+ if (rc) {
+ CPT_LOG_ERR("Failed to bring down vf %s, rc %d",
+ cptvf->dev_name, rc);
+ return;
+ }
+}
git.droids-corp.org / dpdk.git / blobdiff