drivers/net/ark/ark_ethdev_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2015-2018 Atomic Rules LLC
   3  */
   4
   5 #include <unistd.h>
   6
   7 #include "ark_ethdev_rx.h"
   8 #include "ark_global.h"
   9 #include "ark_logs.h"
  10 #include "ark_mpu.h"
  11 #include "ark_udm.h"
  12
  13 #define ARK_RX_META_SIZE 32
  14 #define ARK_RX_META_OFFSET (RTE_PKTMBUF_HEADROOM - ARK_RX_META_SIZE)
  15 #define ARK_RX_MPU_CHUNK (64U)
  16
  17 /* Forward declarations */
  18 struct ark_rx_queue;
  19 struct ark_rx_meta;
  20
  21 static void dump_mbuf_data(struct rte_mbuf *mbuf, uint16_t lo, uint16_t hi);
  22 static void ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue);
  23 static uint32_t eth_ark_rx_jumbo(struct ark_rx_queue *queue,
  24                                  struct ark_rx_meta *meta,
  25                                  struct rte_mbuf *mbuf0,
  26                                  uint32_t cons_index);
  27 static inline int eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue);
  28
  29 /* ************************************************************************* */
  30 struct ark_rx_queue {
  31         /* array of mbufs to populate */
  32         struct rte_mbuf **reserve_q;
  33         /* array of physical addresses of the mbuf data pointer */
  34         /* This point is a virtual address */
  35         rte_iova_t *paddress_q;
  36         struct rte_mempool *mb_pool;
  37
  38         struct ark_udm_t *udm;
  39         struct ark_mpu_t *mpu;
  40
  41         rx_user_meta_hook_fn rx_user_meta_hook;
  42         void *ext_user_data;
  43
  44         uint32_t dataroom;
  45         uint32_t headroom;
  46
  47         uint32_t queue_size;
  48         uint32_t queue_mask;
  49
  50         uint32_t seed_index;            /* step 1 set with empty mbuf */
  51         uint32_t cons_index;            /* step 3 consumed by driver */
  52
  53         /* The queue Id is used to identify the HW Q */
  54         uint16_t phys_qid;
  55
  56         /* The queue Index is used within the dpdk device structures */
  57         uint16_t queue_index;
  58
  59         uint32_t unused;
  60
  61         /* next cache line - fields written by device */
  62         RTE_MARKER cacheline1 __rte_cache_min_aligned;
  63
  64         volatile uint32_t prod_index;   /* step 2 filled by FPGA */
  65 } __rte_cache_aligned;
  66
  67 /* ************************************************************************* */
  68 static int
  69 eth_ark_rx_hw_setup(struct rte_eth_dev *dev,
  70                     struct ark_rx_queue *queue,
  71                     uint16_t rx_queue_id __rte_unused, uint16_t rx_queue_idx)
  72 {
  73         rte_iova_t queue_base;
  74         rte_iova_t phys_addr_q_base;
  75         rte_iova_t phys_addr_prod_index;
  76
  77         queue_base = rte_malloc_virt2iova(queue);
  78         phys_addr_prod_index = queue_base +
  79                 offsetof(struct ark_rx_queue, prod_index);
  80
  81         phys_addr_q_base = rte_malloc_virt2iova(queue->paddress_q);
  82
  83         /* Verify HW */
  84         if (ark_mpu_verify(queue->mpu, sizeof(rte_iova_t))) {
  85                 ARK_PMD_LOG(ERR, "Illegal configuration rx queue\n");
  86                 return -1;
  87         }
  88
  89         /* Stop and Reset and configure MPU */
  90         ark_mpu_configure(queue->mpu, phys_addr_q_base, queue->queue_size, 0);
  91
  92         ark_udm_write_addr(queue->udm, phys_addr_prod_index);
  93
  94         /* advance the valid pointer, but don't start until the queue starts */
  95         ark_mpu_reset_stats(queue->mpu);
  96
  97         /* The seed is the producer index for the HW */
  98         ark_mpu_set_producer(queue->mpu, queue->seed_index);
  99         dev->data->rx_queue_state[rx_queue_idx] = RTE_ETH_QUEUE_STATE_STOPPED;
 100
 101         return 0;
 102 }
 103
 104 static inline void
 105 eth_ark_rx_update_cons_index(struct ark_rx_queue *queue, uint32_t cons_index)
 106 {
 107         queue->cons_index = cons_index;
 108         if ((cons_index + queue->queue_size - queue->seed_index) >= ARK_RX_MPU_CHUNK) {
 109                 eth_ark_rx_seed_mbufs(queue);
 110                 ark_mpu_set_producer(queue->mpu, queue->seed_index);
 111         }
 112 }
 113
 114 /* ************************************************************************* */
 115 int
 116 eth_ark_dev_rx_queue_setup(struct rte_eth_dev *dev,
 117                            uint16_t queue_idx,
 118                            uint16_t nb_desc,
 119                            unsigned int socket_id,
 120                            const struct rte_eth_rxconf *rx_conf,
 121                            struct rte_mempool *mb_pool)
 122 {
 123         static int warning1;            /* = 0 */
 124         struct ark_adapter *ark = dev->data->dev_private;
 125
 126         struct ark_rx_queue *queue;
 127         uint32_t i;
 128         int status;
 129
 130         int qidx = queue_idx;
 131
 132         /* We may already be setup, free memory prior to re-allocation */
 133         if (dev->data->rx_queues[queue_idx] != NULL) {
 134                 eth_ark_dev_rx_queue_release(dev->data->rx_queues[queue_idx]);
 135                 dev->data->rx_queues[queue_idx] = NULL;
 136         }
 137
 138         if (rx_conf != NULL && warning1 == 0) {
 139                 warning1 = 1;
 140                 ARK_PMD_LOG(NOTICE,
 141                             "Arkville ignores rte_eth_rxconf argument.\n");
 142         }
 143
 144         if (RTE_PKTMBUF_HEADROOM < ARK_RX_META_SIZE) {
 145                 ARK_PMD_LOG(ERR,
 146                             "Error: DPDK Arkville requires head room > %d bytes (%s)\n",
 147                             ARK_RX_META_SIZE, __func__);
 148                 return -1;              /* ERROR CODE */
 149         }
 150
 151         if (!rte_is_power_of_2(nb_desc)) {
 152                 ARK_PMD_LOG(ERR,
 153                             "DPDK Arkville configuration queue size must be power of two %u (%s)\n",
 154                             nb_desc, __func__);
 155                 return -1;              /* ERROR CODE */
 156         }
 157
 158         /* Allocate queue struct */
 159         queue = rte_zmalloc_socket("Ark_rxqueue",
 160                                    sizeof(struct ark_rx_queue),
 161                                    64,
 162                                    socket_id);
 163         if (queue == 0) {
 164                 ARK_PMD_LOG(ERR, "Failed to allocate memory in %s\n", __func__);
 165                 return -ENOMEM;
 166         }
 167
 168         /* NOTE zmalloc is used, no need to 0 indexes, etc. */
 169         queue->mb_pool = mb_pool;
 170         queue->dataroom = rte_pktmbuf_data_room_size(mb_pool) -
 171                 RTE_PKTMBUF_HEADROOM;
 172         queue->headroom = RTE_PKTMBUF_HEADROOM;
 173         queue->phys_qid = qidx;
 174         queue->queue_index = queue_idx;
 175         queue->queue_size = nb_desc;
 176         queue->queue_mask = nb_desc - 1;
 177         queue->rx_user_meta_hook = ark->user_ext.rx_user_meta_hook;
 178         queue->ext_user_data = ark->user_data[dev->data->port_id];
 179
 180         queue->reserve_q =
 181                 rte_zmalloc_socket("Ark_rx_queue mbuf",
 182                                    nb_desc * sizeof(struct rte_mbuf *),
 183                                    512,
 184                                    socket_id);
 185         queue->paddress_q =
 186                 rte_zmalloc_socket("Ark_rx_queue paddr",
 187                                    nb_desc * sizeof(rte_iova_t),
 188                                    512,
 189                                    socket_id);
 190
 191         if (queue->reserve_q == 0 || queue->paddress_q == 0) {
 192                 ARK_PMD_LOG(ERR,
 193                             "Failed to allocate queue memory in %s\n",
 194                             __func__);
 195                 rte_free(queue->reserve_q);
 196                 rte_free(queue->paddress_q);
 197                 rte_free(queue);
 198                 return -ENOMEM;
 199         }
 200
 201         dev->data->rx_queues[queue_idx] = queue;
 202         queue->udm = RTE_PTR_ADD(ark->udm.v, qidx * ARK_UDM_QOFFSET);
 203         queue->mpu = RTE_PTR_ADD(ark->mpurx.v, qidx * ARK_MPU_QOFFSET);
 204
 205         /* Configure UDM per queue */
 206         ark_udm_stop(queue->udm, 0);
 207         ark_udm_configure(queue->udm,
 208                           RTE_PKTMBUF_HEADROOM,
 209                           queue->dataroom,
 210                           ARK_RX_WRITE_TIME_NS);
 211         ark_udm_stats_reset(queue->udm);
 212         ark_udm_stop(queue->udm, 0);
 213
 214         /* populate mbuf reserve */
 215         status = eth_ark_rx_seed_mbufs(queue);
 216
 217         if (queue->seed_index != nb_desc) {
 218                 ARK_PMD_LOG(ERR, "Failed to allocate %u mbufs for RX queue %d\n",
 219                             nb_desc, qidx);
 220                 status = -1;
 221         }
 222         /* MPU Setup */
 223         if (status == 0)
 224                 status = eth_ark_rx_hw_setup(dev, queue, qidx, queue_idx);
 225
 226         if (unlikely(status != 0)) {
 227                 struct rte_mbuf **mbuf;
 228
 229                 ARK_PMD_LOG(ERR, "Failed to initialize RX queue %d %s\n",
 230                             qidx,
 231                             __func__);
 232                 /* Free the mbufs allocated */
 233                 for (i = 0, mbuf = queue->reserve_q;
 234                      i < queue->seed_index; ++i, mbuf++) {
 235                         rte_pktmbuf_free(*mbuf);
 236                 }
 237                 rte_free(queue->reserve_q);
 238                 rte_free(queue->paddress_q);
 239                 rte_free(queue);
 240                 return -1;              /* ERROR CODE */
 241         }
 242
 243         return 0;
 244 }
 245
 246 /* ************************************************************************* */
 247 uint16_t
 248 eth_ark_recv_pkts(void *rx_queue,
 249                   struct rte_mbuf **rx_pkts,
 250                   uint16_t nb_pkts)
 251 {
 252         struct ark_rx_queue *queue;
 253         register uint32_t cons_index, prod_index;
 254         uint16_t nb;
 255         uint16_t i;
 256         struct rte_mbuf *mbuf;
 257         struct rte_mbuf **pmbuf;
 258         struct ark_rx_meta *meta;
 259         rx_user_meta_hook_fn rx_user_meta_hook;
 260
 261         queue = (struct ark_rx_queue *)rx_queue;
 262         if (unlikely(queue == 0))
 263                 return 0;
 264         if (unlikely(nb_pkts == 0))
 265                 return 0;
 266         prod_index = queue->prod_index;
 267         cons_index = queue->cons_index;
 268         if (prod_index == cons_index)
 269                 return 0;
 270         nb = 0;
 271
 272         while (prod_index != cons_index) {
 273                 mbuf = queue->reserve_q[cons_index & queue->queue_mask];
 274                 /* prefetch mbuf */
 275                 rte_mbuf_prefetch_part1(mbuf);
 276                 rte_mbuf_prefetch_part2(mbuf);
 277
 278                 /* META DATA embedded in headroom */
 279                 meta = RTE_PTR_ADD(mbuf->buf_addr, ARK_RX_META_OFFSET);
 280
 281                 mbuf->pkt_len = meta->pkt_len;
 282                 mbuf->data_len = meta->pkt_len;
 283
 284                 if (ARK_DEBUG_CORE) {   /* debug sanity checks */
 285
 286                         if ((meta->pkt_len > (1024 * 16)) ||
 287                             (meta->pkt_len == 0)) {
 288                                 ARK_PMD_LOG(DEBUG, "RX: Bad Meta Q: %u"
 289                                            " cons: %" PRIU32
 290                                            " prod: %" PRIU32
 291                                            " seed_index %" PRIU32
 292                                            "\n",
 293                                            queue->phys_qid,
 294                                            cons_index,
 295                                            queue->prod_index,
 296                                            queue->seed_index);
 297
 298
 299                                 ARK_PMD_LOG(DEBUG, "       :  UDM"
 300                                            " prod: %" PRIU32
 301                                            " len: %u\n",
 302                                            queue->udm->rt_cfg.prod_idx,
 303                                            meta->pkt_len);
 304                                 ark_mpu_dump(queue->mpu,
 305                                              "    ",
 306                                              queue->phys_qid);
 307                                 dump_mbuf_data(mbuf, 0, 256);
 308                                 /* its FUBAR so fix it */
 309                                 mbuf->pkt_len = 63;
 310                                 meta->pkt_len = 63;
 311                         }
 312                 }
 313
 314                 if (unlikely(meta->pkt_len > queue->dataroom))
 315                         cons_index = eth_ark_rx_jumbo
 316                                 (queue, meta, mbuf, cons_index + 1);
 317                 else
 318                         cons_index += 1;
 319
 320                 rx_pkts[nb] = mbuf;
 321                 nb++;
 322                 if (nb >= nb_pkts)
 323                         break;
 324         }
 325
 326         rx_user_meta_hook = queue->rx_user_meta_hook;
 327         for (pmbuf = rx_pkts, i = 0; rx_user_meta_hook && i < nb; i++) {
 328                 mbuf = *pmbuf++;
 329                 meta = RTE_PTR_ADD(mbuf->buf_addr, ARK_RX_META_OFFSET);
 330                 rx_user_meta_hook(mbuf, meta->user_meta, queue->ext_user_data);
 331         }
 332
 333         eth_ark_rx_update_cons_index(queue, cons_index);
 334
 335         return nb;
 336 }
 337
 338 /* ************************************************************************* */
 339 static uint32_t
 340 eth_ark_rx_jumbo(struct ark_rx_queue *queue,
 341                  struct ark_rx_meta *meta,
 342                  struct rte_mbuf *mbuf0,
 343                  uint32_t cons_index)
 344 {
 345         struct rte_mbuf *mbuf_prev;
 346         struct rte_mbuf *mbuf;
 347
 348         uint16_t remaining;
 349         uint16_t data_len;
 350         uint16_t segments;
 351
 352         /* first buf populated by called */
 353         mbuf_prev = mbuf0;
 354         segments = 1;
 355         data_len = RTE_MIN(meta->pkt_len, queue->dataroom);
 356         remaining = meta->pkt_len - data_len;
 357         mbuf0->data_len = data_len;
 358
 359         /* HW guarantees that the data does not exceed prod_index! */
 360         while (remaining != 0) {
 361                 data_len = RTE_MIN(remaining,
 362                                    queue->dataroom);
 363
 364                 remaining -= data_len;
 365                 segments += 1;
 366
 367                 mbuf = queue->reserve_q[cons_index & queue->queue_mask];
 368                 mbuf_prev->next = mbuf;
 369                 mbuf_prev = mbuf;
 370                 mbuf->data_len = data_len;
 371
 372                 cons_index += 1;
 373         }
 374
 375         mbuf0->nb_segs = segments;
 376         return cons_index;
 377 }
 378
 379 /* Drain the internal queue allowing hw to clear out. */
 380 static void
 381 eth_ark_rx_queue_drain(struct ark_rx_queue *queue)
 382 {
 383         register uint32_t cons_index;
 384         struct rte_mbuf *mbuf;
 385
 386         cons_index = queue->cons_index;
 387
 388         /* NOT performance optimized, since this is a one-shot call */
 389         while ((cons_index ^ queue->prod_index) & queue->queue_mask) {
 390                 mbuf = queue->reserve_q[cons_index & queue->queue_mask];
 391                 rte_pktmbuf_free(mbuf);
 392                 cons_index++;
 393                 eth_ark_rx_update_cons_index(queue, cons_index);
 394         }
 395 }
 396
 397 uint32_t
 398 eth_ark_dev_rx_queue_count(void *rx_queue)
 399 {
 400         struct ark_rx_queue *queue;
 401
 402         queue = rx_queue;
 403         return (queue->prod_index - queue->cons_index); /* mod arith */
 404 }
 405
 406 /* ************************************************************************* */
 407 int
 408 eth_ark_rx_start_queue(struct rte_eth_dev *dev, uint16_t queue_id)
 409 {
 410         struct ark_rx_queue *queue;
 411
 412         queue = dev->data->rx_queues[queue_id];
 413         if (queue == 0)
 414                 return -1;
 415
 416         dev->data->rx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 417
 418         ark_mpu_set_producer(queue->mpu, queue->seed_index);
 419         ark_mpu_start(queue->mpu);
 420
 421         ark_udm_queue_enable(queue->udm, 1);
 422
 423         return 0;
 424 }
 425
 426 /* ************************************************************************* */
 427
 428 /* Queue can be restarted.   data remains
 429  */
 430 int
 431 eth_ark_rx_stop_queue(struct rte_eth_dev *dev, uint16_t queue_id)
 432 {
 433         struct ark_rx_queue *queue;
 434
 435         queue = dev->data->rx_queues[queue_id];
 436         if (queue == 0)
 437                 return -1;
 438
 439         ark_udm_queue_enable(queue->udm, 0);
 440
 441         dev->data->rx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 442
 443         return 0;
 444 }
 445
 446 /* ************************************************************************* */
 447 static inline int
 448 eth_ark_rx_seed_mbufs(struct ark_rx_queue *queue)
 449 {
 450         uint32_t limit = (queue->cons_index & ~(ARK_RX_MPU_CHUNK - 1)) +
 451                 queue->queue_size;
 452         uint32_t seed_index = queue->seed_index;
 453
 454         uint32_t count = 0;
 455         uint32_t seed_m = queue->seed_index & queue->queue_mask;
 456
 457         uint32_t nb = limit - seed_index;
 458
 459         /* Handle wrap around -- remainder is filled on the next call */
 460         if (unlikely(seed_m + nb > queue->queue_size))
 461                 nb = queue->queue_size - seed_m;
 462
 463         struct rte_mbuf **mbufs = &queue->reserve_q[seed_m];
 464         int status = rte_pktmbuf_alloc_bulk(queue->mb_pool, mbufs, nb);
 465
 466         if (unlikely(status != 0)) {
 467                 ARK_PMD_LOG(NOTICE,
 468                             "Could not allocate %u mbufs from pool"
 469                             " for RX queue %u;"
 470                             " %u free buffers remaining in queue\n",
 471                             nb, queue->queue_index,
 472                             queue->seed_index - queue->cons_index);
 473                 return -1;
 474         }
 475
 476         if (ARK_DEBUG_CORE) {           /* DEBUG */
 477                 while (count != nb) {
 478                         struct rte_mbuf *mbuf_init =
 479                                 queue->reserve_q[seed_m + count];
 480
 481                         memset(mbuf_init->buf_addr, -1, 512);
 482                         *((uint32_t *)mbuf_init->buf_addr) =
 483                                 seed_index + count;
 484                         *(uint16_t *)RTE_PTR_ADD(mbuf_init->buf_addr, 4) =
 485                                 queue->phys_qid;
 486                         count++;
 487                 }
 488                 count = 0;
 489         } /* DEBUG */
 490         queue->seed_index += nb;
 491
 492         /* Duff's device https://en.wikipedia.org/wiki/Duff's_device */
 493         switch (nb % 4) {
 494         case 0:
 495                 while (count != nb) {
 496                         queue->paddress_q[seed_m++] =
 497                                 (*mbufs++)->buf_iova;
 498                         count++;
 499                 /* FALLTHROUGH */
 500         case 3:
 501                 queue->paddress_q[seed_m++] =
 502                         (*mbufs++)->buf_iova;
 503                 count++;
 504                 /* FALLTHROUGH */
 505         case 2:
 506                 queue->paddress_q[seed_m++] =
 507                         (*mbufs++)->buf_iova;
 508                 count++;
 509                 /* FALLTHROUGH */
 510         case 1:
 511                 queue->paddress_q[seed_m++] =
 512                         (*mbufs++)->buf_iova;
 513                 count++;
 514                 /* FALLTHROUGH */
 515
 516                 } /* while (count != nb) */
 517         } /* switch */
 518
 519         return 0;
 520 }
 521
 522 void
 523 eth_ark_rx_dump_queue(struct rte_eth_dev *dev, uint16_t queue_id,
 524                       const char *msg)
 525 {
 526         struct ark_rx_queue *queue;
 527
 528         queue = dev->data->rx_queues[queue_id];
 529
 530         ark_ethdev_rx_dump(msg, queue);
 531 }
 532
 533 /* ************************************************************************* */
 534 /* Call on device closed no user API, queue is stopped */
 535 void
 536 eth_ark_dev_rx_queue_release(void *vqueue)
 537 {
 538         struct ark_rx_queue *queue;
 539         uint32_t i;
 540
 541         queue = (struct ark_rx_queue *)vqueue;
 542         if (queue == 0)
 543                 return;
 544
 545         ark_udm_queue_enable(queue->udm, 0);
 546         /* Stop the MPU since pointer are going away */
 547         ark_mpu_stop(queue->mpu);
 548
 549         /* Need to clear out mbufs here, dropping packets along the way */
 550         eth_ark_rx_queue_drain(queue);
 551
 552         for (i = 0; i < queue->queue_size; ++i)
 553                 rte_pktmbuf_free(queue->reserve_q[i]);
 554
 555         rte_free(queue->reserve_q);
 556         rte_free(queue->paddress_q);
 557         rte_free(queue);
 558 }
 559
 560 void
 561 eth_rx_queue_stats_get(void *vqueue, struct rte_eth_stats *stats)
 562 {
 563         struct ark_rx_queue *queue;
 564         struct ark_udm_t *udm;
 565
 566         queue = vqueue;
 567         if (queue == 0)
 568                 return;
 569         udm = queue->udm;
 570
 571         uint64_t ibytes = ark_udm_bytes(udm);
 572         uint64_t ipackets = ark_udm_packets(udm);
 573         uint64_t idropped = ark_udm_dropped(queue->udm);
 574
 575         stats->q_ipackets[queue->queue_index] = ipackets;
 576         stats->q_ibytes[queue->queue_index] = ibytes;
 577         stats->q_errors[queue->queue_index] = idropped;
 578         stats->ipackets += ipackets;
 579         stats->ibytes += ibytes;
 580         stats->imissed += idropped;
 581 }
 582
 583 void
 584 eth_rx_queue_stats_reset(void *vqueue)
 585 {
 586         struct ark_rx_queue *queue;
 587
 588         queue = vqueue;
 589         if (queue == 0)
 590                 return;
 591
 592         ark_mpu_reset_stats(queue->mpu);
 593         ark_udm_queue_stats_reset(queue->udm);
 594 }
 595
 596 void
 597 eth_ark_udm_force_close(struct rte_eth_dev *dev)
 598 {
 599         struct ark_adapter *ark = dev->data->dev_private;
 600         struct ark_rx_queue *queue;
 601         uint32_t index;
 602         uint16_t i;
 603
 604         if (!ark_udm_is_flushed(ark->udm.v)) {
 605                 /* restart the MPUs */
 606                 ARK_PMD_LOG(NOTICE, "UDM not flushed -- forcing flush\n");
 607                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
 608                         queue = (struct ark_rx_queue *)dev->data->rx_queues[i];
 609                         if (queue == 0)
 610                                 continue;
 611
 612                         ark_mpu_start(queue->mpu);
 613                         /* Add some buffers */
 614                         index = ARK_RX_MPU_CHUNK + queue->seed_index;
 615                         ark_mpu_set_producer(queue->mpu, index);
 616                 }
 617                 /* Wait to allow data to pass */
 618                 usleep(100);
 619
 620                 ARK_PMD_LOG(NOTICE, "UDM forced flush attempt, stopped = %d\n",
 621                             ark_udm_is_flushed(ark->udm.v));
 622         }
 623         ark_udm_reset(ark->udm.v);
 624 }
 625
 626 static void
 627 ark_ethdev_rx_dump(const char *name, struct ark_rx_queue *queue)
 628 {
 629         if (queue == NULL)
 630                 return;
 631         ARK_PMD_LOG(DEBUG, "RX QUEUE %d -- %s", queue->phys_qid, name);
 632         ARK_PMD_LOG(DEBUG, ARK_SU32 ARK_SU32 ARK_SU32 ARK_SU32 "\n",
 633                         "queue_size", queue->queue_size,
 634                         "seed_index", queue->seed_index,
 635                         "prod_index", queue->prod_index,
 636                         "cons_index", queue->cons_index);
 637
 638         ark_mpu_dump(queue->mpu, name, queue->phys_qid);
 639         ark_mpu_dump_setup(queue->mpu, queue->phys_qid);
 640         ark_udm_dump(queue->udm, name);
 641         ark_udm_dump_setup(queue->udm, queue->phys_qid);
 642 }
 643
 644 /* Only used in debug.
 645  * This function is a raw memory dump of a portion of an mbuf's memory
 646  * region.  The usual function, rte_pktmbuf_dump() only shows data
 647  * with respect to the data_off field.  This function show data
 648  * anywhere in the mbuf's buffer.  This is useful for examining
 649  * data in the headroom or tailroom portion of an mbuf.
 650  */
 651 static void
 652 dump_mbuf_data(struct rte_mbuf *mbuf, uint16_t lo, uint16_t hi)
 653 {
 654         uint16_t i, j;
 655
 656         ARK_PMD_LOG(DEBUG, " MBUF: %p len %d, off: %d\n",
 657                     mbuf, mbuf->pkt_len, mbuf->data_off);
 658         for (i = lo; i < hi; i += 16) {
 659                 uint8_t *dp = RTE_PTR_ADD(mbuf->buf_addr, i);
 660
 661                 ARK_PMD_LOG(DEBUG, "  %6d:  ", i);
 662                 for (j = 0; j < 16; j++)
 663                         ARK_PMD_LOG(DEBUG, " %02x", dp[j]);
 664
 665                 ARK_PMD_LOG(DEBUG, "\n");
 666         }
 667 }