drivers/net/octeontx_ep/otx_ep_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include <unistd.h>
   6
   7 #include <rte_eal.h>
   8 #include <rte_mempool.h>
   9 #include <rte_mbuf.h>
  10 #include <rte_io.h>
  11 #include <rte_net.h>
  12 #include <ethdev_pci.h>
  13
  14 #include "otx_ep_common.h"
  15 #include "otx_ep_vf.h"
  16 #include "otx2_ep_vf.h"
  17 #include "otx_ep_rxtx.h"
  18
  19 /* SDP_LENGTH_S specifies packet length and is of 8-byte size */
  20 #define INFO_SIZE 8
  21 #define DROQ_REFILL_THRESHOLD 16
  22
  23 static void
  24 otx_ep_dmazone_free(const struct rte_memzone *mz)
  25 {
  26         const struct rte_memzone *mz_tmp;
  27         int ret = 0;
  28
  29         if (mz == NULL) {
  30                 otx_ep_err("Memzone: NULL\n");
  31                 return;
  32         }
  33
  34         mz_tmp = rte_memzone_lookup(mz->name);
  35         if (mz_tmp == NULL) {
  36                 otx_ep_err("Memzone %s Not Found\n", mz->name);
  37                 return;
  38         }
  39
  40         ret = rte_memzone_free(mz);
  41         if (ret)
  42                 otx_ep_err("Memzone free failed : ret = %d\n", ret);
  43 }
  44
  45 /* Free IQ resources */
  46 int
  47 otx_ep_delete_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no)
  48 {
  49         struct otx_ep_instr_queue *iq;
  50
  51         iq = otx_ep->instr_queue[iq_no];
  52         if (iq == NULL) {
  53                 otx_ep_err("Invalid IQ[%d]\n", iq_no);
  54                 return -EINVAL;
  55         }
  56
  57         rte_free(iq->req_list);
  58         iq->req_list = NULL;
  59
  60         if (iq->iq_mz) {
  61                 otx_ep_dmazone_free(iq->iq_mz);
  62                 iq->iq_mz = NULL;
  63         }
  64
  65         rte_free(otx_ep->instr_queue[iq_no]);
  66         otx_ep->instr_queue[iq_no] = NULL;
  67
  68         otx_ep->nb_tx_queues--;
  69
  70         otx_ep_info("IQ[%d] is deleted\n", iq_no);
  71
  72         return 0;
  73 }
  74
  75 /* IQ initialization */
  76 static int
  77 otx_ep_init_instr_queue(struct otx_ep_device *otx_ep, int iq_no, int num_descs,
  78                      unsigned int socket_id)
  79 {
  80         const struct otx_ep_config *conf;
  81         struct otx_ep_instr_queue *iq;
  82         uint32_t q_size;
  83
  84         conf = otx_ep->conf;
  85         iq = otx_ep->instr_queue[iq_no];
  86         q_size = conf->iq.instr_type * num_descs;
  87
  88         /* IQ memory creation for Instruction submission to OCTEON TX2 */
  89         iq->iq_mz = rte_eth_dma_zone_reserve(otx_ep->eth_dev,
  90                                              "instr_queue", iq_no, q_size,
  91                                              OTX_EP_PCI_RING_ALIGN,
  92                                              socket_id);
  93         if (iq->iq_mz == NULL) {
  94                 otx_ep_err("IQ[%d] memzone alloc failed\n", iq_no);
  95                 goto iq_init_fail;
  96         }
  97
  98         iq->base_addr_dma = iq->iq_mz->iova;
  99         iq->base_addr = (uint8_t *)iq->iq_mz->addr;
 100
 101         if (num_descs & (num_descs - 1)) {
 102                 otx_ep_err("IQ[%d] descs not in power of 2\n", iq_no);
 103                 goto iq_init_fail;
 104         }
 105
 106         iq->nb_desc = num_descs;
 107
 108         /* Create a IQ request list to hold requests that have been
 109          * posted to OCTEON TX2. This list will be used for freeing the IQ
 110          * data buffer(s) later once the OCTEON TX2 fetched the requests.
 111          */
 112         iq->req_list = rte_zmalloc_socket("request_list",
 113                         (iq->nb_desc * OTX_EP_IQREQ_LIST_SIZE),
 114                         RTE_CACHE_LINE_SIZE,
 115                         rte_socket_id());
 116         if (iq->req_list == NULL) {
 117                 otx_ep_err("IQ[%d] req_list alloc failed\n", iq_no);
 118                 goto iq_init_fail;
 119         }
 120
 121         otx_ep_info("IQ[%d]: base: %p basedma: %lx count: %d\n",
 122                      iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma,
 123                      iq->nb_desc);
 124
 125         iq->otx_ep_dev = otx_ep;
 126         iq->q_no = iq_no;
 127         iq->fill_cnt = 0;
 128         iq->host_write_index = 0;
 129         iq->otx_read_index = 0;
 130         iq->flush_index = 0;
 131         iq->instr_pending = 0;
 132
 133         otx_ep->io_qmask.iq |= (1ull << iq_no);
 134
 135         /* Set 32B/64B mode for each input queue */
 136         if (conf->iq.instr_type == 64)
 137                 otx_ep->io_qmask.iq64B |= (1ull << iq_no);
 138
 139         iq->iqcmd_64B = (conf->iq.instr_type == 64);
 140
 141         /* Set up IQ registers */
 142         otx_ep->fn_list.setup_iq_regs(otx_ep, iq_no);
 143
 144         return 0;
 145
 146 iq_init_fail:
 147         return -ENOMEM;
 148 }
 149
 150 int
 151 otx_ep_setup_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no, int num_descs,
 152                  unsigned int socket_id)
 153 {
 154         struct otx_ep_instr_queue *iq;
 155
 156         iq = (struct otx_ep_instr_queue *)rte_zmalloc("otx_ep_IQ", sizeof(*iq),
 157                                                 RTE_CACHE_LINE_SIZE);
 158         if (iq == NULL)
 159                 return -ENOMEM;
 160
 161         otx_ep->instr_queue[iq_no] = iq;
 162
 163         if (otx_ep_init_instr_queue(otx_ep, iq_no, num_descs, socket_id)) {
 164                 otx_ep_err("IQ init is failed\n");
 165                 goto delete_IQ;
 166         }
 167         otx_ep->nb_tx_queues++;
 168
 169         otx_ep_info("IQ[%d] is created.\n", iq_no);
 170
 171         return 0;
 172
 173 delete_IQ:
 174         otx_ep_delete_iqs(otx_ep, iq_no);
 175         return -ENOMEM;
 176 }
 177
 178 static void
 179 otx_ep_droq_reset_indices(struct otx_ep_droq *droq)
 180 {
 181         droq->read_idx  = 0;
 182         droq->write_idx = 0;
 183         droq->refill_idx = 0;
 184         droq->refill_count = 0;
 185         droq->last_pkt_count = 0;
 186         droq->pkts_pending = 0;
 187 }
 188
 189 static void
 190 otx_ep_droq_destroy_ring_buffers(struct otx_ep_droq *droq)
 191 {
 192         uint32_t idx;
 193
 194         for (idx = 0; idx < droq->nb_desc; idx++) {
 195                 if (droq->recv_buf_list[idx]) {
 196                         rte_pktmbuf_free(droq->recv_buf_list[idx]);
 197                         droq->recv_buf_list[idx] = NULL;
 198                 }
 199         }
 200
 201         otx_ep_droq_reset_indices(droq);
 202 }
 203
 204 /* Free OQs resources */
 205 int
 206 otx_ep_delete_oqs(struct otx_ep_device *otx_ep, uint32_t oq_no)
 207 {
 208         struct otx_ep_droq *droq;
 209
 210         droq = otx_ep->droq[oq_no];
 211         if (droq == NULL) {
 212                 otx_ep_err("Invalid droq[%d]\n", oq_no);
 213                 return -EINVAL;
 214         }
 215
 216         otx_ep_droq_destroy_ring_buffers(droq);
 217         rte_free(droq->recv_buf_list);
 218         droq->recv_buf_list = NULL;
 219
 220         if (droq->desc_ring_mz) {
 221                 otx_ep_dmazone_free(droq->desc_ring_mz);
 222                 droq->desc_ring_mz = NULL;
 223         }
 224
 225         memset(droq, 0, OTX_EP_DROQ_SIZE);
 226
 227         rte_free(otx_ep->droq[oq_no]);
 228         otx_ep->droq[oq_no] = NULL;
 229
 230         otx_ep->nb_rx_queues--;
 231
 232         otx_ep_info("OQ[%d] is deleted\n", oq_no);
 233         return 0;
 234 }
 235
 236 static int
 237 otx_ep_droq_setup_ring_buffers(struct otx_ep_droq *droq)
 238 {
 239         struct otx_ep_droq_desc *desc_ring = droq->desc_ring;
 240         struct otx_ep_droq_info *info;
 241         struct rte_mbuf *buf;
 242         uint32_t idx;
 243
 244         for (idx = 0; idx < droq->nb_desc; idx++) {
 245                 buf = rte_pktmbuf_alloc(droq->mpool);
 246                 if (buf == NULL) {
 247                         otx_ep_err("OQ buffer alloc failed\n");
 248                         droq->stats.rx_alloc_failure++;
 249                         return -ENOMEM;
 250                 }
 251
 252                 droq->recv_buf_list[idx] = buf;
 253                 info = rte_pktmbuf_mtod(buf, struct otx_ep_droq_info *);
 254                 memset(info, 0, sizeof(*info));
 255                 desc_ring[idx].buffer_ptr = rte_mbuf_data_iova_default(buf);
 256         }
 257
 258         otx_ep_droq_reset_indices(droq);
 259
 260         return 0;
 261 }
 262
 263 /* OQ initialization */
 264 static int
 265 otx_ep_init_droq(struct otx_ep_device *otx_ep, uint32_t q_no,
 266               uint32_t num_descs, uint32_t desc_size,
 267               struct rte_mempool *mpool, unsigned int socket_id)
 268 {
 269         const struct otx_ep_config *conf = otx_ep->conf;
 270         uint32_t c_refill_threshold;
 271         struct otx_ep_droq *droq;
 272         uint32_t desc_ring_size;
 273
 274         otx_ep_info("OQ[%d] Init start\n", q_no);
 275
 276         droq = otx_ep->droq[q_no];
 277         droq->otx_ep_dev = otx_ep;
 278         droq->q_no = q_no;
 279         droq->mpool = mpool;
 280
 281         droq->nb_desc      = num_descs;
 282         droq->buffer_size  = desc_size;
 283         c_refill_threshold = RTE_MAX(conf->oq.refill_threshold,
 284                                      droq->nb_desc / 2);
 285
 286         /* OQ desc_ring set up */
 287         desc_ring_size = droq->nb_desc * OTX_EP_DROQ_DESC_SIZE;
 288         droq->desc_ring_mz = rte_eth_dma_zone_reserve(otx_ep->eth_dev, "droq",
 289                                                       q_no, desc_ring_size,
 290                                                       OTX_EP_PCI_RING_ALIGN,
 291                                                       socket_id);
 292
 293         if (droq->desc_ring_mz == NULL) {
 294                 otx_ep_err("OQ:%d desc_ring allocation failed\n", q_no);
 295                 goto init_droq_fail;
 296         }
 297
 298         droq->desc_ring_dma = droq->desc_ring_mz->iova;
 299         droq->desc_ring = (struct otx_ep_droq_desc *)droq->desc_ring_mz->addr;
 300
 301         otx_ep_dbg("OQ[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
 302                     q_no, droq->desc_ring, (unsigned long)droq->desc_ring_dma);
 303         otx_ep_dbg("OQ[%d]: num_desc: %d\n", q_no, droq->nb_desc);
 304
 305         /* OQ buf_list set up */
 306         droq->recv_buf_list = rte_zmalloc_socket("recv_buf_list",
 307                                 (droq->nb_desc * sizeof(struct rte_mbuf *)),
 308                                  RTE_CACHE_LINE_SIZE, socket_id);
 309         if (droq->recv_buf_list == NULL) {
 310                 otx_ep_err("OQ recv_buf_list alloc failed\n");
 311                 goto init_droq_fail;
 312         }
 313
 314         if (otx_ep_droq_setup_ring_buffers(droq))
 315                 goto init_droq_fail;
 316
 317         droq->refill_threshold = c_refill_threshold;
 318
 319         /* Set up OQ registers */
 320         otx_ep->fn_list.setup_oq_regs(otx_ep, q_no);
 321
 322         otx_ep->io_qmask.oq |= (1ull << q_no);
 323
 324         return 0;
 325
 326 init_droq_fail:
 327         return -ENOMEM;
 328 }
 329
 330 /* OQ configuration and setup */
 331 int
 332 otx_ep_setup_oqs(struct otx_ep_device *otx_ep, int oq_no, int num_descs,
 333                  int desc_size, struct rte_mempool *mpool,
 334                  unsigned int socket_id)
 335 {
 336         struct otx_ep_droq *droq;
 337
 338         /* Allocate new droq. */
 339         droq = (struct otx_ep_droq *)rte_zmalloc("otx_ep_OQ",
 340                                 sizeof(*droq), RTE_CACHE_LINE_SIZE);
 341         if (droq == NULL) {
 342                 otx_ep_err("Droq[%d] Creation Failed\n", oq_no);
 343                 return -ENOMEM;
 344         }
 345         otx_ep->droq[oq_no] = droq;
 346
 347         if (otx_ep_init_droq(otx_ep, oq_no, num_descs, desc_size, mpool,
 348                              socket_id)) {
 349                 otx_ep_err("Droq[%d] Initialization failed\n", oq_no);
 350                 goto delete_OQ;
 351         }
 352         otx_ep_info("OQ[%d] is created.\n", oq_no);
 353
 354         otx_ep->nb_rx_queues++;
 355
 356         return 0;
 357
 358 delete_OQ:
 359         otx_ep_delete_oqs(otx_ep, oq_no);
 360         return -ENOMEM;
 361 }
 362
 363 static inline void
 364 otx_ep_iqreq_delete(struct otx_ep_instr_queue *iq, uint32_t idx)
 365 {
 366         uint32_t reqtype;
 367         void *buf;
 368         struct otx_ep_buf_free_info *finfo;
 369
 370         buf     = iq->req_list[idx].buf;
 371         reqtype = iq->req_list[idx].reqtype;
 372
 373         switch (reqtype) {
 374         case OTX_EP_REQTYPE_NORESP_NET:
 375                 rte_pktmbuf_free((struct rte_mbuf *)buf);
 376                 otx_ep_dbg("IQ buffer freed at idx[%d]\n", idx);
 377                 break;
 378
 379         case OTX_EP_REQTYPE_NORESP_GATHER:
 380                 finfo = (struct  otx_ep_buf_free_info *)buf;
 381                 /* This will take care of multiple segments also */
 382                 rte_pktmbuf_free(finfo->mbuf);
 383                 rte_free(finfo->g.sg);
 384                 rte_free(finfo);
 385                 break;
 386
 387         case OTX_EP_REQTYPE_NONE:
 388         default:
 389                 otx_ep_info("This iqreq mode is not supported:%d\n", reqtype);
 390         }
 391
 392         /* Reset the request list at this index */
 393         iq->req_list[idx].buf = NULL;
 394         iq->req_list[idx].reqtype = 0;
 395 }
 396
 397 static inline void
 398 otx_ep_iqreq_add(struct otx_ep_instr_queue *iq, void *buf,
 399                 uint32_t reqtype, int index)
 400 {
 401         iq->req_list[index].buf = buf;
 402         iq->req_list[index].reqtype = reqtype;
 403 }
 404
 405 static uint32_t
 406 otx_vf_update_read_index(struct otx_ep_instr_queue *iq)
 407 {
 408         uint32_t new_idx = rte_read32(iq->inst_cnt_reg);
 409         if (unlikely(new_idx == 0xFFFFFFFFU))
 410                 rte_write32(new_idx, iq->inst_cnt_reg);
 411         /* Modulo of the new index with the IQ size will give us
 412          * the new index.
 413          */
 414         new_idx &= (iq->nb_desc - 1);
 415
 416         return new_idx;
 417 }
 418
 419 static void
 420 otx_ep_flush_iq(struct otx_ep_instr_queue *iq)
 421 {
 422         uint32_t instr_processed = 0;
 423
 424         iq->otx_read_index = otx_vf_update_read_index(iq);
 425         while (iq->flush_index != iq->otx_read_index) {
 426                 /* Free the IQ data buffer to the pool */
 427                 otx_ep_iqreq_delete(iq, iq->flush_index);
 428                 iq->flush_index =
 429                         otx_ep_incr_index(iq->flush_index, 1, iq->nb_desc);
 430
 431                 instr_processed++;
 432         }
 433
 434         iq->stats.instr_processed = instr_processed;
 435         iq->instr_pending -= instr_processed;
 436 }
 437
 438 static inline void
 439 otx_ep_ring_doorbell(struct otx_ep_device *otx_ep __rte_unused,
 440                 struct otx_ep_instr_queue *iq)
 441 {
 442         rte_wmb();
 443         rte_write64(iq->fill_cnt, iq->doorbell_reg);
 444         iq->fill_cnt = 0;
 445 }
 446
 447 static inline int
 448 post_iqcmd(struct otx_ep_instr_queue *iq, uint8_t *iqcmd)
 449 {
 450         uint8_t *iqptr, cmdsize;
 451
 452         /* This ensures that the read index does not wrap around to
 453          * the same position if queue gets full before OCTEON TX2 could
 454          * fetch any instr.
 455          */
 456         if (iq->instr_pending > (iq->nb_desc - 1))
 457                 return OTX_EP_IQ_SEND_FAILED;
 458
 459         /* Copy cmd into iq */
 460         cmdsize = 64;
 461         iqptr   = iq->base_addr + (iq->host_write_index << 6);
 462
 463         rte_memcpy(iqptr, iqcmd, cmdsize);
 464
 465         /* Increment the host write index */
 466         iq->host_write_index =
 467                 otx_ep_incr_index(iq->host_write_index, 1, iq->nb_desc);
 468
 469         iq->fill_cnt++;
 470
 471         /* Flush the command into memory. We need to be sure the data
 472          * is in memory before indicating that the instruction is
 473          * pending.
 474          */
 475         iq->instr_pending++;
 476         /* OTX_EP_IQ_SEND_SUCCESS */
 477         return 0;
 478 }
 479
 480
 481 static int
 482 otx_ep_send_data(struct otx_ep_device *otx_ep, struct otx_ep_instr_queue *iq,
 483                  void *cmd, int dbell)
 484 {
 485         uint32_t ret;
 486
 487         /* Submit IQ command */
 488         ret = post_iqcmd(iq, cmd);
 489
 490         if (ret == OTX_EP_IQ_SEND_SUCCESS) {
 491                 if (dbell)
 492                         otx_ep_ring_doorbell(otx_ep, iq);
 493                 iq->stats.instr_posted++;
 494
 495         } else {
 496                 iq->stats.instr_dropped++;
 497                 if (iq->fill_cnt)
 498                         otx_ep_ring_doorbell(otx_ep, iq);
 499         }
 500         return ret;
 501 }
 502
 503 static inline void
 504 set_sg_size(struct otx_ep_sg_entry *sg_entry, uint16_t size, uint32_t pos)
 505 {
 506 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
 507         sg_entry->u.size[pos] = size;
 508 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
 509         sg_entry->u.size[3 - pos] = size;
 510 #endif
 511 }
 512
 513 /* Enqueue requests/packets to OTX_EP IQ queue.
 514  * returns number of requests enqueued successfully
 515  */
 516 uint16_t
 517 otx_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
 518 {
 519         struct otx_ep_instr_64B iqcmd;
 520         struct otx_ep_instr_queue *iq;
 521         struct otx_ep_device *otx_ep;
 522         struct rte_mbuf *m;
 523
 524         uint32_t iqreq_type, sgbuf_sz;
 525         int dbell, index, count = 0;
 526         unsigned int pkt_len, i;
 527         int gather, gsz;
 528         void *iqreq_buf;
 529         uint64_t dptr;
 530
 531         iq = (struct otx_ep_instr_queue *)tx_queue;
 532         otx_ep = iq->otx_ep_dev;
 533
 534         iqcmd.ih.u64 = 0;
 535         iqcmd.pki_ih3.u64 = 0;
 536         iqcmd.irh.u64 = 0;
 537
 538         /* ih invars */
 539         iqcmd.ih.s.fsz = OTX_EP_FSZ;
 540         iqcmd.ih.s.pkind = otx_ep->pkind; /* The SDK decided PKIND value */
 541
 542         /* pki ih3 invars */
 543         iqcmd.pki_ih3.s.w = 1;
 544         iqcmd.pki_ih3.s.utt = 1;
 545         iqcmd.pki_ih3.s.tagtype = ORDERED_TAG;
 546         /* sl will be sizeof(pki_ih3) */
 547         iqcmd.pki_ih3.s.sl = OTX_EP_FSZ + OTX_CUST_DATA_LEN;
 548
 549         /* irh invars */
 550         iqcmd.irh.s.opcode = OTX_EP_NW_PKT_OP;
 551
 552         for (i = 0; i < nb_pkts; i++) {
 553                 m = pkts[i];
 554                 if (m->nb_segs == 1) {
 555                         /* dptr */
 556                         dptr = rte_mbuf_data_iova(m);
 557                         pkt_len = rte_pktmbuf_data_len(m);
 558                         iqreq_buf = m;
 559                         iqreq_type = OTX_EP_REQTYPE_NORESP_NET;
 560                         gather = 0;
 561                         gsz = 0;
 562                 } else {
 563                         struct otx_ep_buf_free_info *finfo;
 564                         int j, frags, num_sg;
 565
 566                         if (!(otx_ep->tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS))
 567                                 goto xmit_fail;
 568
 569                         finfo = (struct otx_ep_buf_free_info *)rte_malloc(NULL,
 570                                                         sizeof(*finfo), 0);
 571                         if (finfo == NULL) {
 572                                 otx_ep_err("free buffer alloc failed\n");
 573                                 goto xmit_fail;
 574                         }
 575                         num_sg = (m->nb_segs + 3) / 4;
 576                         sgbuf_sz = sizeof(struct otx_ep_sg_entry) * num_sg;
 577                         finfo->g.sg =
 578                                 rte_zmalloc(NULL, sgbuf_sz, OTX_EP_SG_ALIGN);
 579                         if (finfo->g.sg == NULL) {
 580                                 rte_free(finfo);
 581                                 otx_ep_err("sg entry alloc failed\n");
 582                                 goto xmit_fail;
 583                         }
 584                         gather = 1;
 585                         gsz = m->nb_segs;
 586                         finfo->g.num_sg = num_sg;
 587                         finfo->g.sg[0].ptr[0] = rte_mbuf_data_iova(m);
 588                         set_sg_size(&finfo->g.sg[0], m->data_len, 0);
 589                         pkt_len = m->data_len;
 590                         finfo->mbuf = m;
 591
 592                         frags = m->nb_segs - 1;
 593                         j = 1;
 594                         m = m->next;
 595                         while (frags--) {
 596                                 finfo->g.sg[(j >> 2)].ptr[(j & 3)] =
 597                                                 rte_mbuf_data_iova(m);
 598                                 set_sg_size(&finfo->g.sg[(j >> 2)],
 599                                                 m->data_len, (j & 3));
 600                                 pkt_len += m->data_len;
 601                                 j++;
 602                                 m = m->next;
 603                         }
 604                         dptr = rte_mem_virt2iova(finfo->g.sg);
 605                         iqreq_buf = finfo;
 606                         iqreq_type = OTX_EP_REQTYPE_NORESP_GATHER;
 607                         if (pkt_len > OTX_EP_MAX_PKT_SZ) {
 608                                 rte_free(finfo->g.sg);
 609                                 rte_free(finfo);
 610                                 otx_ep_err("failed\n");
 611                                 goto xmit_fail;
 612                         }
 613                 }
 614                 /* ih vars */
 615                 iqcmd.ih.s.tlen = pkt_len + iqcmd.ih.s.fsz;
 616                 iqcmd.ih.s.gather = gather;
 617                 iqcmd.ih.s.gsz = gsz;
 618
 619                 iqcmd.dptr = dptr;
 620                 otx_ep_swap_8B_data(&iqcmd.irh.u64, 1);
 621
 622 #ifdef OTX_EP_IO_DEBUG
 623                 otx_ep_dbg("After swapping\n");
 624                 otx_ep_dbg("Word0 [dptr]: 0x%016lx\n",
 625                            (unsigned long)iqcmd.dptr);
 626                 otx_ep_dbg("Word1 [ihtx]: 0x%016lx\n", (unsigned long)iqcmd.ih);
 627                 otx_ep_dbg("Word2 [pki_ih3]: 0x%016lx\n",
 628                            (unsigned long)iqcmd.pki_ih3);
 629                 otx_ep_dbg("Word3 [rptr]: 0x%016lx\n",
 630                            (unsigned long)iqcmd.rptr);
 631                 otx_ep_dbg("Word4 [irh]: 0x%016lx\n", (unsigned long)iqcmd.irh);
 632                 otx_ep_dbg("Word5 [exhdr[0]]: 0x%016lx\n",
 633                                 (unsigned long)iqcmd.exhdr[0]);
 634                 rte_pktmbuf_dump(stdout, m, rte_pktmbuf_pkt_len(m));
 635 #endif
 636                 dbell = (i == (unsigned int)(nb_pkts - 1)) ? 1 : 0;
 637                 index = iq->host_write_index;
 638                 if (otx_ep_send_data(otx_ep, iq, &iqcmd, dbell))
 639                         goto xmit_fail;
 640                 otx_ep_iqreq_add(iq, iqreq_buf, iqreq_type, index);
 641                 iq->stats.tx_pkts++;
 642                 iq->stats.tx_bytes += pkt_len;
 643                 count++;
 644         }
 645
 646 xmit_fail:
 647         if (iq->instr_pending >= OTX_EP_MAX_INSTR)
 648                 otx_ep_flush_iq(iq);
 649
 650         /* Return no# of instructions posted successfully. */
 651         return count;
 652 }
 653
 654 /* Enqueue requests/packets to OTX_EP IQ queue.
 655  * returns number of requests enqueued successfully
 656  */
 657 uint16_t
 658 otx2_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
 659 {
 660         struct otx2_ep_instr_64B iqcmd2;
 661         struct otx_ep_instr_queue *iq;
 662         struct otx_ep_device *otx_ep;
 663         uint64_t dptr;
 664         int count = 0;
 665         unsigned int i;
 666         struct rte_mbuf *m;
 667         unsigned int pkt_len;
 668         void *iqreq_buf;
 669         uint32_t iqreq_type, sgbuf_sz;
 670         int gather, gsz;
 671         int dbell;
 672         int index;
 673
 674         iq = (struct otx_ep_instr_queue *)tx_queue;
 675         otx_ep = iq->otx_ep_dev;
 676
 677         iqcmd2.ih.u64 = 0;
 678         iqcmd2.irh.u64 = 0;
 679
 680         /* ih invars */
 681         iqcmd2.ih.s.fsz = OTX2_EP_FSZ;
 682         iqcmd2.ih.s.pkind = otx_ep->pkind; /* The SDK decided PKIND value */
 683         /* irh invars */
 684         iqcmd2.irh.s.opcode = OTX_EP_NW_PKT_OP;
 685
 686         for (i = 0; i < nb_pkts; i++) {
 687                 m = pkts[i];
 688                 if (m->nb_segs == 1) {
 689                         /* dptr */
 690                         dptr = rte_mbuf_data_iova(m);
 691                         pkt_len = rte_pktmbuf_data_len(m);
 692                         iqreq_buf = m;
 693                         iqreq_type = OTX_EP_REQTYPE_NORESP_NET;
 694                         gather = 0;
 695                         gsz = 0;
 696                 } else {
 697                         struct otx_ep_buf_free_info *finfo;
 698                         int j, frags, num_sg;
 699
 700                         if (!(otx_ep->tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS))
 701                                 goto xmit_fail;
 702
 703                         finfo = (struct otx_ep_buf_free_info *)
 704                                         rte_malloc(NULL, sizeof(*finfo), 0);
 705                         if (finfo == NULL) {
 706                                 otx_ep_err("free buffer alloc failed\n");
 707                                 goto xmit_fail;
 708                         }
 709                         num_sg = (m->nb_segs + 3) / 4;
 710                         sgbuf_sz = sizeof(struct otx_ep_sg_entry) * num_sg;
 711                         finfo->g.sg =
 712                                 rte_zmalloc(NULL, sgbuf_sz, OTX_EP_SG_ALIGN);
 713                         if (finfo->g.sg == NULL) {
 714                                 rte_free(finfo);
 715                                 otx_ep_err("sg entry alloc failed\n");
 716                                 goto xmit_fail;
 717                         }
 718                         gather = 1;
 719                         gsz = m->nb_segs;
 720                         finfo->g.num_sg = num_sg;
 721                         finfo->g.sg[0].ptr[0] = rte_mbuf_data_iova(m);
 722                         set_sg_size(&finfo->g.sg[0], m->data_len, 0);
 723                         pkt_len = m->data_len;
 724                         finfo->mbuf = m;
 725
 726                         frags = m->nb_segs - 1;
 727                         j = 1;
 728                         m = m->next;
 729                         while (frags--) {
 730                                 finfo->g.sg[(j >> 2)].ptr[(j & 3)] =
 731                                                 rte_mbuf_data_iova(m);
 732                                 set_sg_size(&finfo->g.sg[(j >> 2)],
 733                                                 m->data_len, (j & 3));
 734                                 pkt_len += m->data_len;
 735                                 j++;
 736                                 m = m->next;
 737                         }
 738                         dptr = rte_mem_virt2iova(finfo->g.sg);
 739                         iqreq_buf = finfo;
 740                         iqreq_type = OTX_EP_REQTYPE_NORESP_GATHER;
 741                         if (pkt_len > OTX_EP_MAX_PKT_SZ) {
 742                                 rte_free(finfo->g.sg);
 743                                 rte_free(finfo);
 744                                 otx_ep_err("failed\n");
 745                                 goto xmit_fail;
 746                         }
 747                 }
 748                 /* ih vars */
 749                 iqcmd2.ih.s.tlen = pkt_len + iqcmd2.ih.s.fsz;
 750                 iqcmd2.ih.s.gather = gather;
 751                 iqcmd2.ih.s.gsz = gsz;
 752                 iqcmd2.dptr = dptr;
 753                 otx_ep_swap_8B_data(&iqcmd2.irh.u64, 1);
 754
 755 #ifdef OTX_EP_IO_DEBUG
 756                 otx_ep_dbg("After swapping\n");
 757                 otx_ep_dbg("Word0 [dptr]: 0x%016lx\n",
 758                            (unsigned long)iqcmd.dptr);
 759                 otx_ep_dbg("Word1 [ihtx]: 0x%016lx\n", (unsigned long)iqcmd.ih);
 760                 otx_ep_dbg("Word2 [pki_ih3]: 0x%016lx\n",
 761                            (unsigned long)iqcmd.pki_ih3);
 762                 otx_ep_dbg("Word3 [rptr]: 0x%016lx\n",
 763                            (unsigned long)iqcmd.rptr);
 764                 otx_ep_dbg("Word4 [irh]: 0x%016lx\n", (unsigned long)iqcmd.irh);
 765                 otx_ep_dbg("Word5 [exhdr[0]]: 0x%016lx\n",
 766                            (unsigned long)iqcmd.exhdr[0]);
 767 #endif
 768                 index = iq->host_write_index;
 769                 dbell = (i == (unsigned int)(nb_pkts - 1)) ? 1 : 0;
 770                 if (otx_ep_send_data(otx_ep, iq, &iqcmd2, dbell))
 771                         goto xmit_fail;
 772                 otx_ep_iqreq_add(iq, iqreq_buf, iqreq_type, index);
 773                 iq->stats.tx_pkts++;
 774                 iq->stats.tx_bytes += pkt_len;
 775                 count++;
 776         }
 777
 778 xmit_fail:
 779         if (iq->instr_pending >= OTX_EP_MAX_INSTR)
 780                 otx_ep_flush_iq(iq);
 781
 782         /* Return no# of instructions posted successfully. */
 783         return count;
 784 }
 785
 786 static uint32_t
 787 otx_ep_droq_refill(struct otx_ep_droq *droq)
 788 {
 789         struct otx_ep_droq_desc *desc_ring;
 790         struct otx_ep_droq_info *info;
 791         struct rte_mbuf *buf = NULL;
 792         uint32_t desc_refilled = 0;
 793
 794         desc_ring = droq->desc_ring;
 795
 796         while (droq->refill_count && (desc_refilled < droq->nb_desc)) {
 797                 /* If a valid buffer exists (happens if there is no dispatch),
 798                  * reuse the buffer, else allocate.
 799                  */
 800                 if (droq->recv_buf_list[droq->refill_idx] != NULL)
 801                         break;
 802
 803                 buf = rte_pktmbuf_alloc(droq->mpool);
 804                 /* If a buffer could not be allocated, no point in
 805                  * continuing
 806                  */
 807                 if (buf == NULL) {
 808                         droq->stats.rx_alloc_failure++;
 809                         break;
 810                 }
 811                 info = rte_pktmbuf_mtod(buf, struct otx_ep_droq_info *);
 812                 memset(info, 0, sizeof(*info));
 813
 814                 droq->recv_buf_list[droq->refill_idx] = buf;
 815                 desc_ring[droq->refill_idx].buffer_ptr =
 816                                         rte_mbuf_data_iova_default(buf);
 817
 818
 819                 droq->refill_idx = otx_ep_incr_index(droq->refill_idx, 1,
 820                                 droq->nb_desc);
 821
 822                 desc_refilled++;
 823                 droq->refill_count--;
 824         }
 825
 826         return desc_refilled;
 827 }
 828
 829 static struct rte_mbuf *
 830 otx_ep_droq_read_packet(struct otx_ep_device *otx_ep,
 831                         struct otx_ep_droq *droq, int next_fetch)
 832 {
 833         volatile struct otx_ep_droq_info *info;
 834         struct rte_mbuf *droq_pkt2 = NULL;
 835         struct rte_mbuf *droq_pkt = NULL;
 836         struct rte_net_hdr_lens hdr_lens;
 837         struct otx_ep_droq_info *info2;
 838         uint64_t total_pkt_len;
 839         uint32_t pkt_len = 0;
 840         int next_idx;
 841
 842         droq_pkt  = droq->recv_buf_list[droq->read_idx];
 843         droq_pkt2  = droq->recv_buf_list[droq->read_idx];
 844         info = rte_pktmbuf_mtod(droq_pkt, struct otx_ep_droq_info *);
 845         /* make sure info is available */
 846         rte_rmb();
 847         if (unlikely(!info->length)) {
 848                 int retry = OTX_EP_MAX_DELAYED_PKT_RETRIES;
 849                 /* otx_ep_dbg("OCTEON DROQ[%d]: read_idx: %d; Data not ready "
 850                  * "yet, Retry; pending=%lu\n", droq->q_no, droq->read_idx,
 851                  * droq->pkts_pending);
 852                  */
 853                 droq->stats.pkts_delayed_data++;
 854                 while (retry && !info->length)
 855                         retry--;
 856                 if (!retry && !info->length) {
 857                         otx_ep_err("OCTEON DROQ[%d]: read_idx: %d; Retry failed !!\n",
 858                                    droq->q_no, droq->read_idx);
 859                         /* May be zero length packet; drop it */
 860                         rte_pktmbuf_free(droq_pkt);
 861                         droq->recv_buf_list[droq->read_idx] = NULL;
 862                         droq->read_idx = otx_ep_incr_index(droq->read_idx, 1,
 863                                                            droq->nb_desc);
 864                         droq->stats.dropped_zlp++;
 865                         droq->refill_count++;
 866                         goto oq_read_fail;
 867                 }
 868         }
 869         if (next_fetch) {
 870                 next_idx = otx_ep_incr_index(droq->read_idx, 1, droq->nb_desc);
 871                 droq_pkt2  = droq->recv_buf_list[next_idx];
 872                 info2 = rte_pktmbuf_mtod(droq_pkt2, struct otx_ep_droq_info *);
 873                 rte_prefetch_non_temporal((const void *)info2);
 874         }
 875
 876         info->length = rte_bswap64(info->length);
 877         /* Deduce the actual data size */
 878         total_pkt_len = info->length + INFO_SIZE;
 879         if (total_pkt_len <= droq->buffer_size) {
 880                 info->length -=  OTX_EP_RH_SIZE;
 881                 droq_pkt  = droq->recv_buf_list[droq->read_idx];
 882                 if (likely(droq_pkt != NULL)) {
 883                         droq_pkt->data_off += OTX_EP_DROQ_INFO_SIZE;
 884                         /* otx_ep_dbg("OQ: pkt_len[%ld], buffer_size %d\n",
 885                          * (long)info->length, droq->buffer_size);
 886                          */
 887                         pkt_len = (uint32_t)info->length;
 888                         droq_pkt->pkt_len  = pkt_len;
 889                         droq_pkt->data_len  = pkt_len;
 890                         droq_pkt->port = otx_ep->port_id;
 891                         droq->recv_buf_list[droq->read_idx] = NULL;
 892                         droq->read_idx = otx_ep_incr_index(droq->read_idx, 1,
 893                                                            droq->nb_desc);
 894                         droq->refill_count++;
 895                 }
 896         } else {
 897                 struct rte_mbuf *first_buf = NULL;
 898                 struct rte_mbuf *last_buf = NULL;
 899
 900                 while (pkt_len < total_pkt_len) {
 901                         int cpy_len = 0;
 902
 903                         cpy_len = ((pkt_len + droq->buffer_size) >
 904                                         total_pkt_len)
 905                                         ? ((uint32_t)total_pkt_len -
 906                                                 pkt_len)
 907                                         : droq->buffer_size;
 908
 909                         droq_pkt = droq->recv_buf_list[droq->read_idx];
 910                         droq->recv_buf_list[droq->read_idx] = NULL;
 911
 912                         if (likely(droq_pkt != NULL)) {
 913                                 /* Note the first seg */
 914                                 if (!pkt_len)
 915                                         first_buf = droq_pkt;
 916
 917                                 droq_pkt->port = otx_ep->port_id;
 918                                 if (!pkt_len) {
 919                                         droq_pkt->data_off +=
 920                                                 OTX_EP_DROQ_INFO_SIZE;
 921                                         droq_pkt->pkt_len =
 922                                                 cpy_len - OTX_EP_DROQ_INFO_SIZE;
 923                                         droq_pkt->data_len =
 924                                                 cpy_len - OTX_EP_DROQ_INFO_SIZE;
 925                                 } else {
 926                                         droq_pkt->pkt_len = cpy_len;
 927                                         droq_pkt->data_len = cpy_len;
 928                                 }
 929
 930                                 if (pkt_len) {
 931                                         first_buf->nb_segs++;
 932                                         first_buf->pkt_len += droq_pkt->pkt_len;
 933                                 }
 934
 935                                 if (last_buf)
 936                                         last_buf->next = droq_pkt;
 937
 938                                 last_buf = droq_pkt;
 939                         } else {
 940                                 otx_ep_err("no buf\n");
 941                         }
 942
 943                         pkt_len += cpy_len;
 944                         droq->read_idx = otx_ep_incr_index(droq->read_idx, 1,
 945                                                            droq->nb_desc);
 946                         droq->refill_count++;
 947                 }
 948                 droq_pkt = first_buf;
 949         }
 950         droq_pkt->packet_type = rte_net_get_ptype(droq_pkt, &hdr_lens,
 951                                         RTE_PTYPE_ALL_MASK);
 952         droq_pkt->l2_len = hdr_lens.l2_len;
 953         droq_pkt->l3_len = hdr_lens.l3_len;
 954         droq_pkt->l4_len = hdr_lens.l4_len;
 955
 956         if (droq_pkt->nb_segs > 1 &&
 957             !(otx_ep->rx_offloads & DEV_RX_OFFLOAD_SCATTER)) {
 958                 rte_pktmbuf_free(droq_pkt);
 959                 goto oq_read_fail;
 960         }
 961
 962         return droq_pkt;
 963
 964 oq_read_fail:
 965         return NULL;
 966 }
 967
 968 static inline uint32_t
 969 otx_ep_check_droq_pkts(struct otx_ep_droq *droq)
 970 {
 971         volatile uint64_t pkt_count;
 972         uint32_t new_pkts;
 973
 974         /* Latest available OQ packets */
 975         pkt_count = rte_read32(droq->pkts_sent_reg);
 976         rte_write32(pkt_count, droq->pkts_sent_reg);
 977         new_pkts = pkt_count;
 978         droq->pkts_pending += new_pkts;
 979         return new_pkts;
 980 }
 981
 982 /* Check for response arrival from OCTEON TX2
 983  * returns number of requests completed
 984  */
 985 uint16_t
 986 otx_ep_recv_pkts(void *rx_queue,
 987                   struct rte_mbuf **rx_pkts,
 988                   uint16_t budget)
 989 {
 990         struct otx_ep_droq *droq = rx_queue;
 991         struct otx_ep_device *otx_ep;
 992         struct rte_mbuf *oq_pkt;
 993
 994         uint32_t pkts = 0;
 995         uint32_t new_pkts = 0;
 996         int next_fetch;
 997
 998         otx_ep = droq->otx_ep_dev;
 999
1000         if (droq->pkts_pending > budget) {
1001                 new_pkts = budget;
1002         } else {
1003                 new_pkts = droq->pkts_pending;
1004                 new_pkts += otx_ep_check_droq_pkts(droq);
1005                 if (new_pkts > budget)
1006                         new_pkts = budget;
1007         }
1008
1009         if (!new_pkts)
1010                 goto update_credit; /* No pkts at this moment */
1011
1012         for (pkts = 0; pkts < new_pkts; pkts++) {
1013                 /* Push the received pkt to application */
1014                 next_fetch = (pkts == new_pkts - 1) ? 0 : 1;
1015                 oq_pkt = otx_ep_droq_read_packet(otx_ep, droq, next_fetch);
1016                 if (!oq_pkt) {
1017                         RTE_LOG_DP(ERR, PMD,
1018                                    "DROQ read pkt failed pending %" PRIu64
1019                                     "last_pkt_count %" PRIu64 "new_pkts %d.\n",
1020                                    droq->pkts_pending, droq->last_pkt_count,
1021                                    new_pkts);
1022                         droq->pkts_pending -= pkts;
1023                         droq->stats.rx_err++;
1024                         goto finish;
1025                 }
1026                 rx_pkts[pkts] = oq_pkt;
1027                 /* Stats */
1028                 droq->stats.pkts_received++;
1029                 droq->stats.bytes_received += oq_pkt->pkt_len;
1030         }
1031         droq->pkts_pending -= pkts;
1032
1033         /* Refill DROQ buffers */
1034 update_credit:
1035         if (droq->refill_count >= DROQ_REFILL_THRESHOLD) {
1036                 int desc_refilled = otx_ep_droq_refill(droq);
1037
1038                 /* Flush the droq descriptor data to memory to be sure
1039                  * that when we update the credits the data in memory is
1040                  * accurate.
1041                  */
1042                 rte_wmb();
1043                 rte_write32(desc_refilled, droq->pkts_credit_reg);
1044         } else {
1045                 /*
1046                  * SDP output goes into DROP state when output doorbell count
1047                  * goes below drop count. When door bell count is written with
1048                  * a value greater than drop count SDP output should come out
1049                  * of DROP state. Due to a race condition this is not happening.
1050                  * Writing doorbell register with 0 again may make SDP output
1051                  * come out of this state.
1052                  */
1053
1054                 rte_write32(0, droq->pkts_credit_reg);
1055         }
1056 finish:
1057         return pkts;
1058 }