X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fraw%2Fdpaa2_qdma%2Fdpaa2_qdma.c;h=c961e18d670aed2267c4e8ddc53c7fab68426a90;hb=2d084f69aa268634fa7ccefa52c980c1e17e17d0;hp=7b755cea72b9e5b22235ea032b24878f6ada041a;hpb=83a4b2d7fb21900955bf17386eed4c09085400d1;p=dpdk.git diff --git a/drivers/raw/dpaa2_qdma/dpaa2_qdma.c b/drivers/raw/dpaa2_qdma/dpaa2_qdma.c index 7b755cea72..c961e18d67 100644 --- a/drivers/raw/dpaa2_qdma/dpaa2_qdma.c +++ b/drivers/raw/dpaa2_qdma/dpaa2_qdma.c @@ -46,7 +46,7 @@ static struct qdma_per_core_info qdma_core_info[RTE_MAX_LCORE]; static inline int qdma_populate_fd_pci(phys_addr_t src, phys_addr_t dest, uint32_t len, struct qbman_fd *fd, - struct rte_qdma_rbp *rbp) + struct rte_qdma_rbp *rbp, int ser) { fd->simple_pci.saddr_lo = lower_32_bits((uint64_t) (src)); fd->simple_pci.saddr_hi = upper_32_bits((uint64_t) (src)); @@ -56,7 +56,7 @@ qdma_populate_fd_pci(phys_addr_t src, phys_addr_t dest, fd->simple_pci.bmt = 1; fd->simple_pci.fmt = 3; fd->simple_pci.sl = 1; - fd->simple_pci.ser = 1; + fd->simple_pci.ser = ser; fd->simple_pci.sportid = rbp->sportid; /*pcie 3 */ fd->simple_pci.srbp = rbp->srbp; @@ -81,7 +81,7 @@ qdma_populate_fd_pci(phys_addr_t src, phys_addr_t dest, static inline int qdma_populate_fd_ddr(phys_addr_t src, phys_addr_t dest, - uint32_t len, struct qbman_fd *fd) + uint32_t len, struct qbman_fd *fd, int ser) { fd->simple_ddr.saddr_lo = lower_32_bits((uint64_t) (src)); fd->simple_ddr.saddr_hi = upper_32_bits((uint64_t) (src)); @@ -91,7 +91,7 @@ qdma_populate_fd_ddr(phys_addr_t src, phys_addr_t dest, fd->simple_ddr.bmt = 1; fd->simple_ddr.fmt = 3; fd->simple_ddr.sl = 1; - fd->simple_ddr.ser = 1; + fd->simple_ddr.ser = ser; /** * src If RBP=0 {NS,RDTTYPE[3:0]}: 0_1011 * Coherent copy of cacheable memory, @@ -204,6 +204,8 @@ static inline int dpdmai_dev_set_fd_us( struct rte_qdma_job **ppjob; size_t iova; int ret = 0, loop; + int ser = (qdma_vq->flags & RTE_QDMA_VQ_NO_RESPONSE) ? + 0 : 1; for (loop = 0; loop < nb_jobs; loop++) { if (job[loop]->src & QDMA_RBP_UPPER_ADDRESS_MASK) @@ -218,12 +220,12 @@ static inline int dpdmai_dev_set_fd_us( if ((rbp->drbp == 1) || (rbp->srbp == 1)) ret = qdma_populate_fd_pci((phys_addr_t)job[loop]->src, - (phys_addr_t)job[loop]->dest, - job[loop]->len, &fd[loop], rbp); + (phys_addr_t)job[loop]->dest, + job[loop]->len, &fd[loop], rbp, ser); else ret = qdma_populate_fd_ddr((phys_addr_t)job[loop]->src, - (phys_addr_t)job[loop]->dest, - job[loop]->len, &fd[loop]); + (phys_addr_t)job[loop]->dest, + job[loop]->len, &fd[loop], ser); } return ret; @@ -290,6 +292,52 @@ static uint32_t qdma_populate_sg_entry( return total_len; } +static inline int dpdmai_dev_set_multi_fd_lf_no_rsp( + struct qdma_virt_queue *qdma_vq, + struct qbman_fd *fd, + struct rte_qdma_job **job, + uint16_t nb_jobs) +{ + struct rte_qdma_rbp *rbp = &qdma_vq->rbp; + struct rte_qdma_job **ppjob; + uint16_t i; + void *elem; + struct qbman_fle *fle; + uint64_t elem_iova, fle_iova; + + for (i = 0; i < nb_jobs; i++) { + elem = job[i]->usr_elem; +#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA + elem_iova = rte_mempool_virt2iova(elem); +#else + elem_iova = DPAA2_VADDR_TO_IOVA(elem); +#endif + + ppjob = (struct rte_qdma_job **) + ((uintptr_t)(uint64_t)elem + + QDMA_FLE_SINGLE_JOB_OFFSET); + *ppjob = job[i]; + + job[i]->vq_id = qdma_vq->vq_id; + + fle = (struct qbman_fle *) + ((uintptr_t)(uint64_t)elem + QDMA_FLE_FLE_OFFSET); + fle_iova = elem_iova + QDMA_FLE_FLE_OFFSET; + + DPAA2_SET_FD_ADDR(&fd[i], fle_iova); + DPAA2_SET_FD_COMPOUND_FMT(&fd[i]); + + memset(fle, 0, DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle) + + DPAA2_QDMA_MAX_SDD * sizeof(struct qdma_sdd)); + + dpaa2_qdma_populate_fle(fle, fle_iova, rbp, + job[i]->src, job[i]->dest, job[i]->len, + job[i]->flags, QBMAN_FLE_WORD4_FMT_SBF); + } + + return 0; +} + static inline int dpdmai_dev_set_multi_fd_lf( struct qdma_virt_queue *qdma_vq, struct qbman_fd *fd, @@ -300,12 +348,11 @@ static inline int dpdmai_dev_set_multi_fd_lf( struct rte_qdma_job **ppjob; uint16_t i; int ret; - struct qdma_device *qdma_dev = QDMA_DEV_OF_VQ(qdma_vq); void *elem[RTE_QDMA_BURST_NB_MAX]; struct qbman_fle *fle; uint64_t elem_iova, fle_iova; - ret = rte_mempool_get_bulk(qdma_dev->fle_pool, elem, nb_jobs); + ret = rte_mempool_get_bulk(qdma_vq->fle_pool, elem, nb_jobs); if (ret) { DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE"); return ret; @@ -318,11 +365,9 @@ static inline int dpdmai_dev_set_multi_fd_lf( elem_iova = DPAA2_VADDR_TO_IOVA(elem[i]); #endif - *((uint16_t *) - ((uintptr_t)(uint64_t)elem[i] + QDMA_FLE_JOB_NB_OFFSET)) = 1; - ppjob = (struct rte_qdma_job **) - ((uintptr_t)(uint64_t)elem[i] + QDMA_FLE_JOBS_OFFSET); + ((uintptr_t)(uint64_t)elem[i] + + QDMA_FLE_SINGLE_JOB_OFFSET); *ppjob = job[i]; job[i]->vq_id = qdma_vq->vq_id; @@ -360,16 +405,19 @@ static inline int dpdmai_dev_set_sg_fd_lf( int ret = 0, i; struct qdma_sg_entry *src_sge, *dst_sge; uint32_t len, fmt, flags; - struct qdma_device *qdma_dev = QDMA_DEV_OF_VQ(qdma_vq); /* * Get an FLE/SDD from FLE pool. * Note: IO metadata is before the FLE and SDD memory. */ - ret = rte_mempool_get(qdma_dev->fle_pool, (void **)(&elem)); - if (ret) { - DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE"); - return ret; + if (qdma_vq->flags & RTE_QDMA_VQ_NO_RESPONSE) { + elem = job[0]->usr_elem; + } else { + ret = rte_mempool_get(qdma_vq->fle_pool, &elem); + if (ret) { + DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE"); + return ret; + } } #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA @@ -383,7 +431,7 @@ static inline int dpdmai_dev_set_sg_fd_lf( *((uint16_t *) ((uintptr_t)(uint64_t)elem + QDMA_FLE_JOB_NB_OFFSET)) = nb_jobs; ppjob = (struct rte_qdma_job **) - ((uintptr_t)(uint64_t)elem + QDMA_FLE_JOBS_OFFSET); + ((uintptr_t)(uint64_t)elem + QDMA_FLE_SG_JOBS_OFFSET); for (i = 0; i < nb_jobs; i++) ppjob[i] = job[i]; @@ -395,7 +443,8 @@ static inline int dpdmai_dev_set_sg_fd_lf( DPAA2_SET_FD_ADDR(fd, fle_iova); DPAA2_SET_FD_COMPOUND_FMT(fd); - DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX); + if (!(qdma_vq->flags & RTE_QDMA_VQ_NO_RESPONSE)) + DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX); /* Populate FLE */ if (likely(nb_jobs > 1)) { @@ -451,7 +500,41 @@ static inline uint16_t dpdmai_dev_get_job_us( return vqid; } -static inline uint16_t dpdmai_dev_get_job_lf( +static inline uint16_t dpdmai_dev_get_single_job_lf( + struct qdma_virt_queue *qdma_vq, + const struct qbman_fd *fd, + struct rte_qdma_job **job, + uint16_t *nb_jobs) +{ + struct qbman_fle *fle; + struct rte_qdma_job **ppjob = NULL; + uint16_t status; + + /* + * Fetch metadata from FLE. job and vq_id were set + * in metadata in the enqueue operation. + */ + fle = (struct qbman_fle *) + DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); + + *nb_jobs = 1; + ppjob = (struct rte_qdma_job **)((uintptr_t)(uint64_t)fle - + QDMA_FLE_FLE_OFFSET + QDMA_FLE_SINGLE_JOB_OFFSET); + + status = (DPAA2_GET_FD_ERR(fd) << 8) | (DPAA2_GET_FD_FRC(fd) & 0xFF); + + *job = *ppjob; + (*job)->status = status; + + /* Free FLE to the pool */ + rte_mempool_put(qdma_vq->fle_pool, + (void *) + ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET)); + + return (*job)->vq_id; +} + +static inline uint16_t dpdmai_dev_get_sg_job_lf( struct qdma_virt_queue *qdma_vq, const struct qbman_fd *fd, struct rte_qdma_job **job, @@ -460,7 +543,6 @@ static inline uint16_t dpdmai_dev_get_job_lf( struct qbman_fle *fle; struct rte_qdma_job **ppjob = NULL; uint16_t i, status; - struct qdma_device *qdma_dev = QDMA_DEV_OF_VQ(qdma_vq); /* * Fetch metadata from FLE. job and vq_id were set @@ -470,10 +552,9 @@ static inline uint16_t dpdmai_dev_get_job_lf( DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); *nb_jobs = *((uint16_t *)((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET + QDMA_FLE_JOB_NB_OFFSET)); - status = (DPAA2_GET_FD_ERR(fd) << 8) | (DPAA2_GET_FD_FRC(fd) & 0xFF); - ppjob = (struct rte_qdma_job **)((uintptr_t)(uint64_t)fle - - QDMA_FLE_FLE_OFFSET + QDMA_FLE_JOBS_OFFSET); + QDMA_FLE_FLE_OFFSET + QDMA_FLE_SG_JOBS_OFFSET); + status = (DPAA2_GET_FD_ERR(fd) << 8) | (DPAA2_GET_FD_FRC(fd) & 0xFF); for (i = 0; i < (*nb_jobs); i++) { job[i] = ppjob[i]; @@ -481,7 +562,7 @@ static inline uint16_t dpdmai_dev_get_job_lf( } /* Free FLE to the pool */ - rte_mempool_put(qdma_dev->fle_pool, + rte_mempool_put(qdma_vq->fle_pool, (void *) ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET)); @@ -1045,14 +1126,9 @@ dpaa2_qdma_reset(struct rte_rawdev *rawdev) memset(&qdma_core_info, 0, sizeof(struct qdma_per_core_info) * RTE_MAX_LCORE); - /* Free the FLE pool */ - if (qdma_dev->fle_pool) - rte_mempool_free(qdma_dev->fle_pool); - /* Reset QDMA device structure */ qdma_dev->max_hw_queues_per_core = 0; - qdma_dev->fle_pool = NULL; - qdma_dev->fle_pool_count = 0; + qdma_dev->fle_queue_pool_cnt = 0; qdma_dev->max_vqs = 0; return 0; @@ -1099,23 +1175,7 @@ dpaa2_qdma_configure(const struct rte_rawdev *rawdev, return -ENOMEM; } qdma_dev->max_vqs = qdma_config->max_vqs; - - /* Allocate FLE pool; just append PID so that in case of - * multiprocess, the pool's don't collide. - */ - snprintf(name, sizeof(name), "qdma_fle_pool%u", - getpid()); - qdma_dev->fle_pool = rte_mempool_create(name, - qdma_config->fle_pool_count, QDMA_FLE_POOL_SIZE, - QDMA_FLE_CACHE_SIZE(qdma_config->fle_pool_count), 0, - NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0); - if (!qdma_dev->fle_pool) { - DPAA2_QDMA_ERR("qdma_fle_pool create failed"); - rte_free(qdma_dev->vqs); - qdma_dev->vqs = NULL; - return -ENOMEM; - } - qdma_dev->fle_pool_count = qdma_config->fle_pool_count; + qdma_dev->fle_queue_pool_cnt = qdma_config->fle_queue_pool_cnt; return 0; } @@ -1177,11 +1237,13 @@ dpaa2_qdma_queue_setup(struct rte_rawdev *rawdev, size_t conf_size) { char ring_name[32]; + char pool_name[64]; int i; struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private; struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev; struct rte_qdma_queue_config *q_config = (struct rte_qdma_queue_config *)queue_conf; + uint32_t pool_size; DPAA2_QDMA_FUNC_TRACE(); @@ -1216,6 +1278,9 @@ dpaa2_qdma_queue_setup(struct rte_rawdev *rawdev, rte_spinlock_unlock(&qdma_dev->lock); return -ENODEV; } + pool_size = QDMA_FLE_SG_POOL_SIZE; + } else { + pool_size = QDMA_FLE_SINGLE_POOL_SIZE; } if (q_config->flags & RTE_QDMA_VQ_EXCLUSIVE_PQ) { @@ -1226,7 +1291,7 @@ dpaa2_qdma_queue_setup(struct rte_rawdev *rawdev, /* Allocate a Ring for Virtual Queue in VQ mode */ snprintf(ring_name, sizeof(ring_name), "status ring %d", i); qdma_dev->vqs[i].status_ring = rte_ring_create(ring_name, - qdma_dev->fle_pool_count, rte_socket_id(), 0); + qdma_dev->fle_queue_pool_cnt, rte_socket_id(), 0); if (!qdma_dev->vqs[i].status_ring) { DPAA2_QDMA_ERR("Status ring creation failed for vq"); rte_spinlock_unlock(&qdma_dev->lock); @@ -1248,17 +1313,36 @@ dpaa2_qdma_queue_setup(struct rte_rawdev *rawdev, return -ENODEV; } + snprintf(pool_name, sizeof(pool_name), + "qdma_fle_pool%u_queue%d", getpid(), i); + qdma_dev->vqs[i].fle_pool = rte_mempool_create(pool_name, + qdma_dev->fle_queue_pool_cnt, pool_size, + QDMA_FLE_CACHE_SIZE(qdma_dev->fle_queue_pool_cnt), 0, + NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0); + if (!qdma_dev->vqs[i].fle_pool) { + DPAA2_QDMA_ERR("qdma_fle_pool create failed"); + rte_spinlock_unlock(&qdma_dev->lock); + return -ENOMEM; + } + qdma_dev->vqs[i].flags = q_config->flags; qdma_dev->vqs[i].in_use = 1; qdma_dev->vqs[i].lcore_id = q_config->lcore_id; memset(&qdma_dev->vqs[i].rbp, 0, sizeof(struct rte_qdma_rbp)); if (q_config->flags & RTE_QDMA_VQ_FD_LONG_FORMAT) { - if (q_config->flags & RTE_QDMA_VQ_FD_SG_FORMAT) + if (q_config->flags & RTE_QDMA_VQ_FD_SG_FORMAT) { qdma_dev->vqs[i].set_fd = dpdmai_dev_set_sg_fd_lf; - else - qdma_dev->vqs[i].set_fd = dpdmai_dev_set_multi_fd_lf; - qdma_dev->vqs[i].get_job = dpdmai_dev_get_job_lf; + qdma_dev->vqs[i].get_job = dpdmai_dev_get_sg_job_lf; + } else { + if (q_config->flags & RTE_QDMA_VQ_NO_RESPONSE) + qdma_dev->vqs[i].set_fd = + dpdmai_dev_set_multi_fd_lf_no_rsp; + else + qdma_dev->vqs[i].set_fd = + dpdmai_dev_set_multi_fd_lf; + qdma_dev->vqs[i].get_job = dpdmai_dev_get_single_job_lf; + } } else { qdma_dev->vqs[i].set_fd = dpdmai_dev_set_fd_us; qdma_dev->vqs[i].get_job = dpdmai_dev_get_job_us; @@ -1444,6 +1528,9 @@ dpaa2_qdma_queue_release(struct rte_rawdev *rawdev, put_hw_queue(qdma_vq->hw_queue); } + if (qdma_vq->fle_pool) + rte_mempool_free(qdma_vq->fle_pool); + memset(qdma_vq, 0, sizeof(struct qdma_virt_queue)); rte_spinlock_unlock(&qdma_dev->lock);