drivers/raw/octeontx2_ep/otx2_ep_enqdeq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <string.h>
   6 #include <unistd.h>
   7 #include <dirent.h>
   8 #include <fcntl.h>
   9
  10 #include <rte_bus.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_eal.h>
  13 #include <rte_lcore.h>
  14 #include <rte_mempool.h>
  15 #include <rte_pci.h>
  16
  17 #include <rte_common.h>
  18 #include <rte_rawdev.h>
  19 #include <rte_rawdev_pmd.h>
  20
  21 #include "otx2_common.h"
  22 #include "otx2_ep_enqdeq.h"
  23
  24 static void
  25 sdp_dmazone_free(const struct rte_memzone *mz)
  26 {
  27         const struct rte_memzone *mz_tmp;
  28         int ret = 0;
  29
  30         if (mz == NULL) {
  31                 otx2_err("Memzone %s : NULL", mz->name);
  32                 return;
  33         }
  34
  35         mz_tmp = rte_memzone_lookup(mz->name);
  36         if (mz_tmp == NULL) {
  37                 otx2_err("Memzone %s Not Found", mz->name);
  38                 return;
  39         }
  40
  41         ret = rte_memzone_free(mz);
  42         if (ret)
  43                 otx2_err("Memzone free failed : ret = %d", ret);
  44
  45 }
  46
  47 /* Free IQ resources */
  48 int
  49 sdp_delete_iqs(struct sdp_device *sdpvf, uint32_t iq_no)
  50 {
  51         struct sdp_instr_queue *iq;
  52
  53         iq = sdpvf->instr_queue[iq_no];
  54         if (iq == NULL) {
  55                 otx2_err("Invalid IQ[%d]\n", iq_no);
  56                 return -ENOMEM;
  57         }
  58
  59         rte_free(iq->req_list);
  60         iq->req_list = NULL;
  61
  62         if (iq->iq_mz) {
  63                 sdp_dmazone_free(iq->iq_mz);
  64                 iq->iq_mz = NULL;
  65         }
  66
  67         rte_free(sdpvf->instr_queue[iq_no]);
  68         sdpvf->instr_queue[iq_no] = NULL;
  69
  70         sdpvf->num_iqs--;
  71
  72         otx2_info("IQ[%d] is deleted", iq_no);
  73
  74         return 0;
  75 }
  76
  77 /* IQ initialization */
  78 static int
  79 sdp_init_instr_queue(struct sdp_device *sdpvf, int iq_no)
  80 {
  81         const struct sdp_config *conf;
  82         struct sdp_instr_queue *iq;
  83         uint32_t q_size;
  84
  85         conf = sdpvf->conf;
  86         iq = sdpvf->instr_queue[iq_no];
  87         q_size = conf->iq.instr_type * conf->num_iqdef_descs;
  88
  89         /* IQ memory creation for Instruction submission to OCTEON TX2 */
  90         iq->iq_mz = rte_memzone_reserve_aligned("iqmz",
  91                                         q_size,
  92                                         rte_socket_id(),
  93                                         RTE_MEMZONE_IOVA_CONTIG,
  94                                         RTE_CACHE_LINE_SIZE);
  95         if (iq->iq_mz == NULL) {
  96                 otx2_err("IQ[%d] memzone alloc failed", iq_no);
  97                 goto iq_init_fail;
  98         }
  99
 100         iq->base_addr_dma = iq->iq_mz->iova;
 101         iq->base_addr = (uint8_t *)iq->iq_mz->addr;
 102
 103         if (conf->num_iqdef_descs & (conf->num_iqdef_descs - 1)) {
 104                 otx2_err("IQ[%d] descs not in power of 2", iq_no);
 105                 goto iq_init_fail;
 106         }
 107
 108         iq->nb_desc = conf->num_iqdef_descs;
 109
 110         /* Create a IQ request list to hold requests that have been
 111          * posted to OCTEON TX2. This list will be used for freeing the IQ
 112          * data buffer(s) later once the OCTEON TX2 fetched the requests.
 113          */
 114         iq->req_list = rte_zmalloc_socket("request_list",
 115                         (iq->nb_desc * SDP_IQREQ_LIST_SIZE),
 116                         RTE_CACHE_LINE_SIZE,
 117                         rte_socket_id());
 118         if (iq->req_list == NULL) {
 119                 otx2_err("IQ[%d] req_list alloc failed", iq_no);
 120                 goto iq_init_fail;
 121         }
 122
 123         otx2_info("IQ[%d]: base: %p basedma: %lx count: %d",
 124                      iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma,
 125                      iq->nb_desc);
 126
 127         iq->sdp_dev = sdpvf;
 128         iq->q_no = iq_no;
 129         iq->fill_cnt = 0;
 130         iq->host_write_index = 0;
 131         iq->otx_read_index = 0;
 132         iq->flush_index = 0;
 133
 134         /* Initialize the spinlock for this instruction queue */
 135         rte_spinlock_init(&iq->lock);
 136         rte_spinlock_init(&iq->post_lock);
 137
 138         rte_atomic64_clear(&iq->iq_flush_running);
 139
 140         sdpvf->io_qmask.iq |= (1ull << iq_no);
 141
 142         /* Set 32B/64B mode for each input queue */
 143         if (conf->iq.instr_type == 64)
 144                 sdpvf->io_qmask.iq64B |= (1ull << iq_no);
 145
 146         iq->iqcmd_64B = (conf->iq.instr_type == 64);
 147
 148         /* Set up IQ registers */
 149         sdpvf->fn_list.setup_iq_regs(sdpvf, iq_no);
 150
 151         return 0;
 152
 153 iq_init_fail:
 154         return -ENOMEM;
 155
 156 }
 157
 158 int
 159 sdp_setup_iqs(struct sdp_device *sdpvf, uint32_t iq_no)
 160 {
 161         struct sdp_instr_queue *iq;
 162
 163         iq = (struct sdp_instr_queue *)rte_zmalloc("sdp_IQ", sizeof(*iq),
 164                                                 RTE_CACHE_LINE_SIZE);
 165         if (iq == NULL)
 166                 return -ENOMEM;
 167
 168         sdpvf->instr_queue[iq_no] = iq;
 169
 170         if (sdp_init_instr_queue(sdpvf, iq_no)) {
 171                 otx2_err("IQ init is failed");
 172                 goto delete_IQ;
 173         }
 174         otx2_info("IQ[%d] is created.", sdpvf->num_iqs);
 175
 176         sdpvf->num_iqs++;
 177
 178
 179         return 0;
 180
 181 delete_IQ:
 182         sdp_delete_iqs(sdpvf, iq_no);
 183         return -ENOMEM;
 184 }
 185
 186 static void
 187 sdp_droq_reset_indices(struct sdp_droq *droq)
 188 {
 189         droq->read_idx  = 0;
 190         droq->write_idx = 0;
 191         droq->refill_idx = 0;
 192         droq->refill_count = 0;
 193         rte_atomic64_set(&droq->pkts_pending, 0);
 194 }
 195
 196 static void
 197 sdp_droq_destroy_ring_buffers(struct sdp_device *sdpvf,
 198                                 struct sdp_droq *droq)
 199 {
 200         uint32_t idx;
 201
 202         for (idx = 0; idx < droq->nb_desc; idx++) {
 203                 if (droq->recv_buf_list[idx].buffer) {
 204                         rte_mempool_put(sdpvf->enqdeq_mpool,
 205                                 droq->recv_buf_list[idx].buffer);
 206
 207                         droq->recv_buf_list[idx].buffer = NULL;
 208                 }
 209         }
 210
 211         sdp_droq_reset_indices(droq);
 212 }
 213
 214 /* Free OQs resources */
 215 int
 216 sdp_delete_oqs(struct sdp_device *sdpvf, uint32_t oq_no)
 217 {
 218         struct sdp_droq *droq;
 219
 220         droq = sdpvf->droq[oq_no];
 221         if (droq == NULL) {
 222                 otx2_err("Invalid droq[%d]", oq_no);
 223                 return -ENOMEM;
 224         }
 225
 226         sdp_droq_destroy_ring_buffers(sdpvf, droq);
 227         rte_free(droq->recv_buf_list);
 228         droq->recv_buf_list = NULL;
 229
 230         if (droq->info_mz) {
 231                 sdp_dmazone_free(droq->info_mz);
 232                 droq->info_mz = NULL;
 233         }
 234
 235         if (droq->desc_ring_mz) {
 236                 sdp_dmazone_free(droq->desc_ring_mz);
 237                 droq->desc_ring_mz = NULL;
 238         }
 239
 240         memset(droq, 0, SDP_DROQ_SIZE);
 241
 242         rte_free(sdpvf->droq[oq_no]);
 243         sdpvf->droq[oq_no] = NULL;
 244
 245         sdpvf->num_oqs--;
 246
 247         otx2_info("OQ[%d] is deleted", oq_no);
 248         return 0;
 249 }
 250
 251 static int
 252 sdp_droq_setup_ring_buffers(struct sdp_device *sdpvf,
 253                 struct sdp_droq *droq)
 254 {
 255         struct sdp_droq_desc *desc_ring = droq->desc_ring;
 256         uint32_t idx;
 257         void *buf;
 258
 259         for (idx = 0; idx < droq->nb_desc; idx++) {
 260                 rte_mempool_get(sdpvf->enqdeq_mpool, &buf);
 261                 if (buf == NULL) {
 262                         otx2_err("OQ buffer alloc failed");
 263                         /* sdp_droq_destroy_ring_buffers(droq);*/
 264                         return -ENOMEM;
 265                 }
 266
 267                 droq->recv_buf_list[idx].buffer = buf;
 268                 droq->info_list[idx].length = 0;
 269
 270                 /* Map ring buffers into memory */
 271                 desc_ring[idx].info_ptr = (uint64_t)(droq->info_list_dma +
 272                         (idx * SDP_DROQ_INFO_SIZE));
 273
 274                 desc_ring[idx].buffer_ptr = rte_mem_virt2iova(buf);
 275         }
 276
 277         sdp_droq_reset_indices(droq);
 278
 279         return 0;
 280 }
 281
 282 static void *
 283 sdp_alloc_info_buffer(struct sdp_device *sdpvf __rte_unused,
 284         struct sdp_droq *droq)
 285 {
 286         droq->info_mz = rte_memzone_reserve_aligned("OQ_info_list",
 287                                 (droq->nb_desc * SDP_DROQ_INFO_SIZE),
 288                                 rte_socket_id(),
 289                                 RTE_MEMZONE_IOVA_CONTIG,
 290                                 RTE_CACHE_LINE_SIZE);
 291
 292         if (droq->info_mz == NULL)
 293                 return NULL;
 294
 295         droq->info_list_dma = droq->info_mz->iova;
 296         droq->info_alloc_size = droq->info_mz->len;
 297         droq->info_base_addr = (size_t)droq->info_mz->addr;
 298
 299         return droq->info_mz->addr;
 300 }
 301
 302 /* OQ initialization */
 303 static int
 304 sdp_init_droq(struct sdp_device *sdpvf, uint32_t q_no)
 305 {
 306         const struct sdp_config *conf = sdpvf->conf;
 307         uint32_t c_refill_threshold;
 308         uint32_t desc_ring_size;
 309         struct sdp_droq *droq;
 310
 311         otx2_info("OQ[%d] Init start", q_no);
 312
 313         droq = sdpvf->droq[q_no];
 314         droq->sdp_dev = sdpvf;
 315         droq->q_no = q_no;
 316
 317         c_refill_threshold = conf->oq.refill_threshold;
 318         droq->nb_desc      = conf->num_oqdef_descs;
 319         droq->buffer_size  = conf->oqdef_buf_size;
 320
 321         /* OQ desc_ring set up */
 322         desc_ring_size = droq->nb_desc * SDP_DROQ_DESC_SIZE;
 323         droq->desc_ring_mz = rte_memzone_reserve_aligned("sdp_oqmz",
 324                                                 desc_ring_size,
 325                                                 rte_socket_id(),
 326                                                 RTE_MEMZONE_IOVA_CONTIG,
 327                                                 RTE_CACHE_LINE_SIZE);
 328
 329         if (droq->desc_ring_mz == NULL) {
 330                 otx2_err("OQ:%d desc_ring allocation failed", q_no);
 331                 goto init_droq_fail;
 332         }
 333
 334         droq->desc_ring_dma = droq->desc_ring_mz->iova;
 335         droq->desc_ring = (struct sdp_droq_desc *)droq->desc_ring_mz->addr;
 336
 337         otx2_sdp_dbg("OQ[%d]: desc_ring: virt: 0x%p, dma: %lx",
 338                     q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma);
 339         otx2_sdp_dbg("OQ[%d]: num_desc: %d", q_no, droq->nb_desc);
 340
 341
 342         /* OQ info_list set up */
 343         droq->info_list = sdp_alloc_info_buffer(sdpvf, droq);
 344         if (droq->info_list == NULL) {
 345                 otx2_err("memory allocation failed for OQ[%d] info_list", q_no);
 346                 goto init_droq_fail;
 347         }
 348
 349         /* OQ buf_list set up */
 350         droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list",
 351                                 (droq->nb_desc * SDP_DROQ_RECVBUF_SIZE),
 352                                  RTE_CACHE_LINE_SIZE, rte_socket_id());
 353         if (droq->recv_buf_list == NULL) {
 354                 otx2_err("OQ recv_buf_list alloc failed");
 355                 goto init_droq_fail;
 356         }
 357
 358         if (sdp_droq_setup_ring_buffers(sdpvf, droq))
 359                 goto init_droq_fail;
 360
 361         droq->refill_threshold = c_refill_threshold;
 362         rte_spinlock_init(&droq->lock);
 363
 364
 365         /* Set up OQ registers */
 366         sdpvf->fn_list.setup_oq_regs(sdpvf, q_no);
 367
 368         sdpvf->io_qmask.oq |= (1ull << q_no);
 369
 370         return 0;
 371
 372 init_droq_fail:
 373         return -ENOMEM;
 374 }
 375
 376 /* OQ configuration and setup */
 377 int
 378 sdp_setup_oqs(struct sdp_device *sdpvf, uint32_t oq_no)
 379 {
 380         struct sdp_droq *droq;
 381
 382         /* Allocate new droq. */
 383         droq = (struct sdp_droq *)rte_zmalloc("sdp_OQ",
 384                                 sizeof(*droq), RTE_CACHE_LINE_SIZE);
 385         if (droq == NULL) {
 386                 otx2_err("Droq[%d] Creation Failed", oq_no);
 387                 return -ENOMEM;
 388         }
 389         sdpvf->droq[oq_no] = droq;
 390
 391         if (sdp_init_droq(sdpvf, oq_no)) {
 392                 otx2_err("Droq[%d] Initialization failed", oq_no);
 393                 goto delete_OQ;
 394         }
 395         otx2_info("OQ[%d] is created.", oq_no);
 396
 397         sdpvf->num_oqs++;
 398
 399         return 0;
 400
 401 delete_OQ:
 402         sdp_delete_oqs(sdpvf, oq_no);
 403         return -ENOMEM;
 404 }
 405