drivers/net/pfe/pfe_hif.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 NXP
   3  */
   4
   5 #include "pfe_logs.h"
   6 #include "pfe_mod.h"
   7 #include <sys/ioctl.h>
   8 #include <sys/epoll.h>
   9 #include <sys/eventfd.h>
  10 #include <arpa/inet.h>
  11
  12 static int
  13 pfe_hif_alloc_descr(struct pfe_hif *hif)
  14 {
  15         void *addr;
  16         int err = 0;
  17
  18         PMD_INIT_FUNC_TRACE();
  19
  20         addr = rte_zmalloc(NULL, HIF_RX_DESC_NT * sizeof(struct hif_desc) +
  21                 HIF_TX_DESC_NT * sizeof(struct hif_desc), RTE_CACHE_LINE_SIZE);
  22         if (!addr) {
  23                 PFE_PMD_ERR("Could not allocate buffer descriptors!");
  24                 err = -ENOMEM;
  25                 goto err0;
  26         }
  27
  28         hif->descr_baseaddr_p = pfe_mem_vtop((uintptr_t)addr);
  29         hif->descr_baseaddr_v = addr;
  30         hif->rx_ring_size = HIF_RX_DESC_NT;
  31         hif->tx_ring_size = HIF_TX_DESC_NT;
  32
  33         return 0;
  34
  35 err0:
  36         return err;
  37 }
  38
  39 static void
  40 pfe_hif_free_descr(struct pfe_hif *hif)
  41 {
  42         PMD_INIT_FUNC_TRACE();
  43
  44         rte_free(hif->descr_baseaddr_v);
  45 }
  46
  47 /* pfe_hif_release_buffers */
  48 static void
  49 pfe_hif_release_buffers(struct pfe_hif *hif)
  50 {
  51         struct hif_desc *desc;
  52         uint32_t i = 0;
  53         struct rte_mbuf *mbuf;
  54         struct rte_pktmbuf_pool_private *mb_priv;
  55
  56         hif->rx_base = hif->descr_baseaddr_v;
  57
  58         /*Free Rx buffers */
  59         desc = hif->rx_base;
  60         mb_priv = rte_mempool_get_priv(hif->shm->pool);
  61         for (i = 0; i < hif->rx_ring_size; i++) {
  62                 if (readl(&desc->data)) {
  63                         if (i < hif->shm->rx_buf_pool_cnt &&
  64                             !hif->shm->rx_buf_pool[i]) {
  65                                 mbuf = hif->rx_buf_vaddr[i] + PFE_PKT_HEADER_SZ
  66                                         - sizeof(struct rte_mbuf)
  67                                         - RTE_PKTMBUF_HEADROOM
  68                                         - mb_priv->mbuf_priv_size;
  69                                 hif->shm->rx_buf_pool[i] = mbuf;
  70                         }
  71                 }
  72                 writel(0, &desc->data);
  73                 writel(0, &desc->status);
  74                 writel(0, &desc->ctrl);
  75                 desc++;
  76         }
  77 }
  78
  79 /*
  80  * pfe_hif_init_buffers
  81  * This function initializes the HIF Rx/Tx ring descriptors and
  82  * initialize Rx queue with buffers.
  83  */
  84 int
  85 pfe_hif_init_buffers(struct pfe_hif *hif)
  86 {
  87         struct hif_desc *desc, *first_desc_p;
  88         uint32_t i = 0;
  89
  90         PMD_INIT_FUNC_TRACE();
  91
  92         /* Check enough Rx buffers available in the shared memory */
  93         if (hif->shm->rx_buf_pool_cnt < hif->rx_ring_size)
  94                 return -ENOMEM;
  95
  96         hif->rx_base = hif->descr_baseaddr_v;
  97         memset(hif->rx_base, 0, hif->rx_ring_size * sizeof(struct hif_desc));
  98
  99         /*Initialize Rx descriptors */
 100         desc = hif->rx_base;
 101         first_desc_p = (struct hif_desc *)hif->descr_baseaddr_p;
 102
 103         for (i = 0; i < hif->rx_ring_size; i++) {
 104                 /* Initialize Rx buffers from the shared memory */
 105                 struct rte_mbuf *mbuf =
 106                         (struct rte_mbuf *)hif->shm->rx_buf_pool[i];
 107
 108                 /* PFE mbuf structure is as follow:
 109                  * ----------------------------------------------------------+
 110                  * | mbuf  | priv | headroom (annotation + PFE data) | data  |
 111                  * ----------------------------------------------------------+
 112                  *
 113                  * As we are expecting additional information like parse
 114                  * results, eth id, queue id from PFE block along with data.
 115                  * so we have to provide additional memory for each packet to
 116                  * HIF rx rings so that PFE block can write its headers.
 117                  * so, we are giving the data pointor to HIF rings whose
 118                  * calculation is as below:
 119                  * mbuf->data_pointor - Required_header_size
 120                  *
 121                  * We are utilizing the HEADROOM area to receive the PFE
 122                  * block headers. On packet reception, HIF driver will use
 123                  * PFE headers information based on which it will decide
 124                  * the clients and fill the parse results.
 125                  * after that application can use/overwrite the HEADROOM area.
 126                  */
 127                 hif->rx_buf_vaddr[i] =
 128                         (void *)((size_t)mbuf->buf_addr + mbuf->data_off -
 129                                         PFE_PKT_HEADER_SZ);
 130                 hif->rx_buf_addr[i] =
 131                         (void *)(size_t)(rte_pktmbuf_iova(mbuf) -
 132                                         PFE_PKT_HEADER_SZ);
 133                 hif->rx_buf_len[i] =  mbuf->buf_len - RTE_PKTMBUF_HEADROOM;
 134
 135                 hif->shm->rx_buf_pool[i] = NULL;
 136
 137                 writel(DDR_PHYS_TO_PFE(hif->rx_buf_addr[i]),
 138                                         &desc->data);
 139                 writel(0, &desc->status);
 140
 141                 /*
 142                  * Ensure everything else is written to DDR before
 143                  * writing bd->ctrl
 144                  */
 145                 rte_wmb();
 146
 147                 writel((BD_CTRL_PKT_INT_EN | BD_CTRL_LIFM
 148                         | BD_CTRL_DIR | BD_CTRL_DESC_EN
 149                         | BD_BUF_LEN(hif->rx_buf_len[i])), &desc->ctrl);
 150
 151                 /* Chain descriptors */
 152                 writel((u32)DDR_PHYS_TO_PFE(first_desc_p + i + 1), &desc->next);
 153                 desc++;
 154         }
 155
 156         /* Overwrite last descriptor to chain it to first one*/
 157         desc--;
 158         writel((u32)DDR_PHYS_TO_PFE(first_desc_p), &desc->next);
 159
 160         hif->rxtoclean_index = 0;
 161
 162         /*Initialize Rx buffer descriptor ring base address */
 163         writel(DDR_PHYS_TO_PFE(hif->descr_baseaddr_p), HIF_RX_BDP_ADDR);
 164
 165         hif->tx_base = hif->rx_base + hif->rx_ring_size;
 166         first_desc_p = (struct hif_desc *)hif->descr_baseaddr_p +
 167                                 hif->rx_ring_size;
 168         memset(hif->tx_base, 0, hif->tx_ring_size * sizeof(struct hif_desc));
 169
 170         /*Initialize tx descriptors */
 171         desc = hif->tx_base;
 172
 173         for (i = 0; i < hif->tx_ring_size; i++) {
 174                 /* Chain descriptors */
 175                 writel((u32)DDR_PHYS_TO_PFE(first_desc_p + i + 1), &desc->next);
 176                 writel(0, &desc->ctrl);
 177                 desc++;
 178         }
 179
 180         /* Overwrite last descriptor to chain it to first one */
 181         desc--;
 182         writel((u32)DDR_PHYS_TO_PFE(first_desc_p), &desc->next);
 183         hif->txavail = hif->tx_ring_size;
 184         hif->txtosend = 0;
 185         hif->txtoclean = 0;
 186         hif->txtoflush = 0;
 187
 188         /*Initialize Tx buffer descriptor ring base address */
 189         writel((u32)DDR_PHYS_TO_PFE(first_desc_p), HIF_TX_BDP_ADDR);
 190
 191         return 0;
 192 }
 193
 194 /*
 195  * pfe_hif_client_register
 196  *
 197  * This function used to register a client driver with the HIF driver.
 198  *
 199  * Return value:
 200  * 0 - on Successful registration
 201  */
 202 static int
 203 pfe_hif_client_register(struct pfe_hif *hif, u32 client_id,
 204                         struct hif_client_shm *client_shm)
 205 {
 206         struct hif_client *client = &hif->client[client_id];
 207         u32 i, cnt;
 208         struct rx_queue_desc *rx_qbase;
 209         struct tx_queue_desc *tx_qbase;
 210         struct hif_rx_queue *rx_queue;
 211         struct hif_tx_queue *tx_queue;
 212         int err = 0;
 213
 214         PMD_INIT_FUNC_TRACE();
 215
 216         rte_spinlock_lock(&hif->tx_lock);
 217
 218         if (test_bit(client_id, &hif->shm->g_client_status[0])) {
 219                 PFE_PMD_ERR("client %d already registered", client_id);
 220                 err = -1;
 221                 goto unlock;
 222         }
 223
 224         memset(client, 0, sizeof(struct hif_client));
 225
 226         /* Initialize client Rx queues baseaddr, size */
 227
 228         cnt = CLIENT_CTRL_RX_Q_CNT(client_shm->ctrl);
 229         /* Check if client is requesting for more queues than supported */
 230         if (cnt > HIF_CLIENT_QUEUES_MAX)
 231                 cnt = HIF_CLIENT_QUEUES_MAX;
 232
 233         client->rx_qn = cnt;
 234         rx_qbase = (struct rx_queue_desc *)client_shm->rx_qbase;
 235         for (i = 0; i < cnt; i++) {
 236                 rx_queue = &client->rx_q[i];
 237                 rx_queue->base = rx_qbase + i * client_shm->rx_qsize;
 238                 rx_queue->size = client_shm->rx_qsize;
 239                 rx_queue->write_idx = 0;
 240         }
 241
 242         /* Initialize client Tx queues baseaddr, size */
 243         cnt = CLIENT_CTRL_TX_Q_CNT(client_shm->ctrl);
 244
 245         /* Check if client is requesting for more queues than supported */
 246         if (cnt > HIF_CLIENT_QUEUES_MAX)
 247                 cnt = HIF_CLIENT_QUEUES_MAX;
 248
 249         client->tx_qn = cnt;
 250         tx_qbase = (struct tx_queue_desc *)client_shm->tx_qbase;
 251         for (i = 0; i < cnt; i++) {
 252                 tx_queue = &client->tx_q[i];
 253                 tx_queue->base = tx_qbase + i * client_shm->tx_qsize;
 254                 tx_queue->size = client_shm->tx_qsize;
 255                 tx_queue->ack_idx = 0;
 256         }
 257
 258         set_bit(client_id, &hif->shm->g_client_status[0]);
 259
 260 unlock:
 261         rte_spinlock_unlock(&hif->tx_lock);
 262
 263         return err;
 264 }
 265
 266 /*
 267  * pfe_hif_client_unregister
 268  *
 269  * This function used to unregister a client  from the HIF driver.
 270  *
 271  */
 272 static void
 273 pfe_hif_client_unregister(struct pfe_hif *hif, u32 client_id)
 274 {
 275         PMD_INIT_FUNC_TRACE();
 276
 277         /*
 278          * Mark client as no longer available (which prevents further packet
 279          * receive for this client)
 280          */
 281         rte_spinlock_lock(&hif->tx_lock);
 282
 283         if (!test_bit(client_id, &hif->shm->g_client_status[0])) {
 284                 PFE_PMD_ERR("client %d not registered", client_id);
 285
 286                 rte_spinlock_unlock(&hif->tx_lock);
 287                 return;
 288         }
 289
 290         clear_bit(client_id, &hif->shm->g_client_status[0]);
 291
 292         rte_spinlock_unlock(&hif->tx_lock);
 293 }
 294
 295 /*
 296  * client_put_rxpacket-
 297  */
 298 static struct rte_mbuf *
 299 client_put_rxpacket(struct hif_rx_queue *queue,
 300                 void *pkt, u32 len,
 301                 u32 flags, u32 client_ctrl,
 302                 struct rte_mempool *pool,
 303                 u32 *rem_len)
 304 {
 305         struct rx_queue_desc *desc = queue->base + queue->write_idx;
 306         struct rte_mbuf *mbuf = NULL;
 307
 308
 309         if (readl(&desc->ctrl) & CL_DESC_OWN) {
 310                 mbuf = rte_cpu_to_le_64(rte_pktmbuf_alloc(pool));
 311                 if (unlikely(!mbuf)) {
 312                         PFE_PMD_WARN("Buffer allocation failure\n");
 313                         return NULL;
 314                 }
 315
 316                 desc->data = pkt;
 317                 desc->client_ctrl = client_ctrl;
 318                 /*
 319                  * Ensure everything else is written to DDR before
 320                  * writing bd->ctrl
 321                  */
 322                 rte_wmb();
 323                 writel(CL_DESC_BUF_LEN(len) | flags, &desc->ctrl);
 324                 queue->write_idx = (queue->write_idx + 1)
 325                                     & (queue->size - 1);
 326
 327                 *rem_len = mbuf->buf_len;
 328         }
 329
 330         return mbuf;
 331 }
 332
 333 /*
 334  * pfe_hif_rx_process-
 335  * This function does pfe hif rx queue processing.
 336  * Dequeue packet from Rx queue and send it to corresponding client queue
 337  */
 338 int
 339 pfe_hif_rx_process(struct pfe *pfe, int budget)
 340 {
 341         struct hif_desc *desc;
 342         struct hif_hdr *pkt_hdr;
 343         struct __hif_hdr hif_hdr;
 344         void *free_buf;
 345         int rtc, len, rx_processed = 0;
 346         struct __hif_desc local_desc;
 347         int flags = 0, wait_for_last = 0, retry = 0;
 348         unsigned int buf_size = 0;
 349         struct rte_mbuf *mbuf = NULL;
 350         struct pfe_hif *hif = &pfe->hif;
 351
 352         rte_spinlock_lock(&hif->lock);
 353
 354         rtc = hif->rxtoclean_index;
 355
 356         while (rx_processed < budget) {
 357                 desc = hif->rx_base + rtc;
 358
 359                 __memcpy12(&local_desc, desc);
 360
 361                 /* ACK pending Rx interrupt */
 362                 if (local_desc.ctrl & BD_CTRL_DESC_EN) {
 363                         if (unlikely(wait_for_last))
 364                                 continue;
 365                         else
 366                                 break;
 367                 }
 368
 369                 len = BD_BUF_LEN(local_desc.ctrl);
 370                 pkt_hdr = (struct hif_hdr *)hif->rx_buf_vaddr[rtc];
 371
 372                 /* Track last HIF header received */
 373                 if (!hif->started) {
 374                         hif->started = 1;
 375
 376                         __memcpy8(&hif_hdr, pkt_hdr);
 377
 378                         hif->qno = hif_hdr.hdr.q_num;
 379                         hif->client_id = hif_hdr.hdr.client_id;
 380                         hif->client_ctrl = (hif_hdr.hdr.client_ctrl1 << 16) |
 381                                                 hif_hdr.hdr.client_ctrl;
 382                         flags = CL_DESC_FIRST;
 383
 384                 } else {
 385                         flags = 0;
 386                 }
 387
 388                 if (local_desc.ctrl & BD_CTRL_LIFM) {
 389                         flags |= CL_DESC_LAST;
 390                         wait_for_last = 0;
 391                 } else {
 392                         wait_for_last = 1;
 393                 }
 394
 395                 /* Check for valid client id and still registered */
 396                 if (hif->client_id >= HIF_CLIENTS_MAX ||
 397                     !(test_bit(hif->client_id,
 398                         &hif->shm->g_client_status[0]))) {
 399                         PFE_PMD_INFO("packet with invalid client id %d qnum %d",
 400                                 hif->client_id, hif->qno);
 401
 402                         free_buf = hif->rx_buf_addr[rtc];
 403
 404                         goto pkt_drop;
 405                 }
 406
 407                 /* Check to valid queue number */
 408                 if (hif->client[hif->client_id].rx_qn <= hif->qno) {
 409                         PFE_DP_LOG(DEBUG, "packet with invalid queue: %d",
 410                                         hif->qno);
 411                         hif->qno = 0;
 412                 }
 413
 414 retry:
 415                 mbuf =
 416                 client_put_rxpacket(&hif->client[hif->client_id].rx_q[hif->qno],
 417                                     (void *)pkt_hdr, len, flags,
 418                                     hif->client_ctrl, hif->shm->pool,
 419                                     &buf_size);
 420
 421                 if (unlikely(!mbuf)) {
 422                         if (!retry) {
 423                                 pfe_tx_do_cleanup(pfe);
 424                                 retry = 1;
 425                                 goto retry;
 426                         }
 427                         rx_processed = budget;
 428
 429                         if (flags & CL_DESC_FIRST)
 430                                 hif->started = 0;
 431
 432                         PFE_DP_LOG(DEBUG, "No buffers");
 433                         break;
 434                 }
 435
 436                 retry = 0;
 437
 438                 free_buf = (void *)(size_t)rte_pktmbuf_iova(mbuf);
 439                 free_buf = free_buf - PFE_PKT_HEADER_SZ;
 440
 441                 /*Fill free buffer in the descriptor */
 442                 hif->rx_buf_addr[rtc] = free_buf;
 443                 hif->rx_buf_vaddr[rtc] = (void *)((size_t)mbuf->buf_addr +
 444                                 mbuf->data_off - PFE_PKT_HEADER_SZ);
 445                 hif->rx_buf_len[rtc] = buf_size - RTE_PKTMBUF_HEADROOM;
 446
 447 pkt_drop:
 448                 writel(DDR_PHYS_TO_PFE(free_buf), &desc->data);
 449                 /*
 450                  * Ensure everything else is written to DDR before
 451                  * writing bd->ctrl
 452                  */
 453                 rte_wmb();
 454                 writel((BD_CTRL_PKT_INT_EN | BD_CTRL_LIFM | BD_CTRL_DIR |
 455                         BD_CTRL_DESC_EN | BD_BUF_LEN(hif->rx_buf_len[rtc])),
 456                         &desc->ctrl);
 457
 458                 rtc = (rtc + 1) & (hif->rx_ring_size - 1);
 459
 460                 if (local_desc.ctrl & BD_CTRL_LIFM) {
 461                         if (!(hif->client_ctrl & HIF_CTRL_RX_CONTINUED))
 462                                 rx_processed++;
 463
 464                         hif->started = 0;
 465                 }
 466         }
 467
 468
 469         hif->rxtoclean_index = rtc;
 470         rte_spinlock_unlock(&hif->lock);
 471
 472         /* we made some progress, re-start rx dma in case it stopped */
 473         hif_rx_dma_start();
 474
 475         return rx_processed;
 476 }
 477
 478 /*
 479  * client_ack_txpacket-
 480  * This function ack the Tx packet in the give client Tx queue by resetting
 481  * ownership bit in the descriptor.
 482  */
 483 static int
 484 client_ack_txpacket(struct pfe_hif *hif, unsigned int client_id,
 485                     unsigned int q_no)
 486 {
 487         struct hif_tx_queue *queue = &hif->client[client_id].tx_q[q_no];
 488         struct tx_queue_desc *desc = queue->base + queue->ack_idx;
 489
 490         if (readl(&desc->ctrl) & CL_DESC_OWN) {
 491                 writel((readl(&desc->ctrl) & ~CL_DESC_OWN), &desc->ctrl);
 492                 queue->ack_idx = (queue->ack_idx + 1) & (queue->size - 1);
 493
 494                 return 0;
 495
 496         } else {
 497                 /*This should not happen */
 498                 PFE_PMD_ERR("%d %d %d %d %d %p %d",
 499                        hif->txtosend, hif->txtoclean, hif->txavail,
 500                         client_id, q_no, queue, queue->ack_idx);
 501                 return 1;
 502         }
 503 }
 504
 505 static void
 506 __hif_tx_done_process(struct pfe *pfe, int count)
 507 {
 508         struct hif_desc *desc;
 509         struct hif_desc_sw *desc_sw;
 510         unsigned int ttc, tx_avl;
 511         int pkts_done[HIF_CLIENTS_MAX] = {0, 0};
 512         struct pfe_hif *hif = &pfe->hif;
 513
 514         ttc = hif->txtoclean;
 515         tx_avl = hif->txavail;
 516
 517         while ((tx_avl < hif->tx_ring_size) && count--) {
 518                 desc = hif->tx_base + ttc;
 519
 520                 if (readl(&desc->ctrl) & BD_CTRL_DESC_EN)
 521                         break;
 522
 523                 desc_sw = &hif->tx_sw_queue[ttc];
 524
 525                 if (desc_sw->client_id > HIF_CLIENTS_MAX)
 526                         PFE_PMD_ERR("Invalid cl id %d", desc_sw->client_id);
 527
 528                 pkts_done[desc_sw->client_id]++;
 529
 530                 client_ack_txpacket(hif, desc_sw->client_id, desc_sw->q_no);
 531
 532                 ttc = (ttc + 1) & (hif->tx_ring_size - 1);
 533                 tx_avl++;
 534         }
 535
 536         if (pkts_done[0])
 537                 hif_lib_indicate_client(pfe->hif_client[0], EVENT_TXDONE_IND,
 538                                 0);
 539         if (pkts_done[1])
 540                 hif_lib_indicate_client(pfe->hif_client[1], EVENT_TXDONE_IND,
 541                                 0);
 542         hif->txtoclean = ttc;
 543         hif->txavail = tx_avl;
 544 }
 545
 546 static inline void
 547 hif_tx_done_process(struct pfe *pfe, int count)
 548 {
 549         struct pfe_hif *hif = &pfe->hif;
 550         rte_spinlock_lock(&hif->tx_lock);
 551         __hif_tx_done_process(pfe, count);
 552         rte_spinlock_unlock(&hif->tx_lock);
 553 }
 554
 555 void
 556 pfe_tx_do_cleanup(struct pfe *pfe)
 557 {
 558         hif_tx_done_process(pfe, HIF_TX_DESC_NT);
 559 }
 560
 561 /*
 562  * __hif_xmit_pkt -
 563  * This function puts one packet in the HIF Tx queue
 564  */
 565 void
 566 hif_xmit_pkt(struct pfe_hif *hif, unsigned int client_id, unsigned int
 567              q_no, void *data, u32 len, unsigned int flags)
 568 {
 569         struct hif_desc *desc;
 570         struct hif_desc_sw *desc_sw;
 571
 572         desc = hif->tx_base + hif->txtosend;
 573         desc_sw = &hif->tx_sw_queue[hif->txtosend];
 574
 575         desc_sw->len = len;
 576         desc_sw->client_id = client_id;
 577         desc_sw->q_no = q_no;
 578         desc_sw->flags = flags;
 579
 580         writel((u32)DDR_PHYS_TO_PFE(data), &desc->data);
 581
 582         hif->txtosend = (hif->txtosend + 1) & (hif->tx_ring_size - 1);
 583         hif->txavail--;
 584
 585         if ((!((flags & HIF_DATA_VALID) && (flags &
 586                                 HIF_LAST_BUFFER))))
 587                 goto skip_tx;
 588
 589         /*
 590          * Ensure everything else is written to DDR before
 591          * writing bd->ctrl
 592          */
 593         rte_wmb();
 594
 595         do {
 596                 desc_sw = &hif->tx_sw_queue[hif->txtoflush];
 597                 desc = hif->tx_base + hif->txtoflush;
 598
 599                 if (desc_sw->flags & HIF_LAST_BUFFER) {
 600                         writel((BD_CTRL_LIFM |
 601                                BD_CTRL_BRFETCH_DISABLE | BD_CTRL_RTFETCH_DISABLE
 602                                | BD_CTRL_PARSE_DISABLE | BD_CTRL_DESC_EN |
 603                                  BD_BUF_LEN(desc_sw->len)),
 604                                 &desc->ctrl);
 605                 } else {
 606                         writel((BD_CTRL_DESC_EN |
 607                                 BD_BUF_LEN(desc_sw->len)), &desc->ctrl);
 608                 }
 609                 hif->txtoflush = (hif->txtoflush + 1) & (hif->tx_ring_size - 1);
 610         }
 611         while (hif->txtoflush != hif->txtosend)
 612                 ;
 613
 614 skip_tx:
 615         return;
 616 }
 617
 618 void
 619 hif_process_client_req(struct pfe_hif *hif, int req,
 620                             int data1, __rte_unused int data2)
 621 {
 622         unsigned int client_id = data1;
 623
 624         if (client_id >= HIF_CLIENTS_MAX) {
 625                 PFE_PMD_ERR("client id %d out of bounds", client_id);
 626                 return;
 627         }
 628
 629         switch (req) {
 630         case REQUEST_CL_REGISTER:
 631                         /* Request for register a client */
 632                         PFE_PMD_INFO("register client_id %d", client_id);
 633                         pfe_hif_client_register(hif, client_id, (struct
 634                                 hif_client_shm *)&hif->shm->client[client_id]);
 635                         break;
 636
 637         case REQUEST_CL_UNREGISTER:
 638                         PFE_PMD_INFO("unregister client_id %d", client_id);
 639
 640                         /* Request for unregister a client */
 641                         pfe_hif_client_unregister(hif, client_id);
 642
 643                         break;
 644
 645         default:
 646                         PFE_PMD_ERR("unsupported request %d", req);
 647                         break;
 648         }
 649
 650         /*
 651          * Process client Tx queues
 652          * Currently we don't have checking for tx pending
 653          */
 654 }
 655
 656 #if defined(LS1012A_PFE_RESET_WA)
 657 static void
 658 pfe_hif_disable_rx_desc(struct pfe_hif *hif)
 659 {
 660         u32 ii;
 661         struct hif_desc *desc = hif->rx_base;
 662
 663         /*Mark all descriptors as LAST_BD */
 664         for (ii = 0; ii < hif->rx_ring_size; ii++) {
 665                 desc->ctrl |= BD_CTRL_LAST_BD;
 666                 desc++;
 667         }
 668 }
 669
 670 struct class_rx_hdr_t {
 671         u32     next_ptr;       /* ptr to the start of the first DDR buffer */
 672         u16     length;         /* total packet length */
 673         u16     phyno;          /* input physical port number */
 674         u32     status;         /* gemac status bits */
 675         u32     status2;            /* reserved for software usage */
 676 };
 677
 678 /* STATUS_BAD_FRAME_ERR is set for all errors (including checksums if enabled)
 679  * except overflow
 680  */
 681 #define STATUS_BAD_FRAME_ERR            BIT(16)
 682 #define STATUS_LENGTH_ERR               BIT(17)
 683 #define STATUS_CRC_ERR                  BIT(18)
 684 #define STATUS_TOO_SHORT_ERR            BIT(19)
 685 #define STATUS_TOO_LONG_ERR             BIT(20)
 686 #define STATUS_CODE_ERR                 BIT(21)
 687 #define STATUS_MC_HASH_MATCH            BIT(22)
 688 #define STATUS_CUMULATIVE_ARC_HIT       BIT(23)
 689 #define STATUS_UNICAST_HASH_MATCH       BIT(24)
 690 #define STATUS_IP_CHECKSUM_CORRECT      BIT(25)
 691 #define STATUS_TCP_CHECKSUM_CORRECT     BIT(26)
 692 #define STATUS_UDP_CHECKSUM_CORRECT     BIT(27)
 693 #define STATUS_OVERFLOW_ERR             BIT(28) /* GPI error */
 694 #define MIN_PKT_SIZE                    64
 695 #define DUMMY_PKT_COUNT                 128
 696
 697 static inline void
 698 copy_to_lmem(u32 *dst, u32 *src, int len)
 699 {
 700         int i;
 701
 702         for (i = 0; i < len; i += sizeof(u32))  {
 703                 *dst = htonl(*src);
 704                 dst++; src++;
 705         }
 706 }
 707 #if defined(RTE_TOOLCHAIN_GCC)
 708 __attribute__ ((optimize(1)))
 709 #endif
 710 static void
 711 send_dummy_pkt_to_hif(void)
 712 {
 713         void *lmem_ptr, *ddr_ptr, *lmem_virt_addr;
 714         u64 physaddr;
 715         struct class_rx_hdr_t local_hdr;
 716         static u32 dummy_pkt[] =  {
 717                 0x33221100, 0x2b785544, 0xd73093cb, 0x01000608,
 718                 0x04060008, 0x2b780200, 0xd73093cb, 0x0a01a8c0,
 719                 0x33221100, 0xa8c05544, 0x00000301, 0x00000000,
 720                 0x00000000, 0x00000000, 0x00000000, 0xbe86c51f };
 721
 722         ddr_ptr = (void *)(size_t)readl(BMU2_BASE_ADDR + BMU_ALLOC_CTRL);
 723         if (!ddr_ptr)
 724                 return;
 725
 726         lmem_ptr = (void *)(size_t)readl(BMU1_BASE_ADDR + BMU_ALLOC_CTRL);
 727         if (!lmem_ptr)
 728                 return;
 729
 730         PFE_PMD_INFO("Sending a dummy pkt to HIF %p %p", ddr_ptr, lmem_ptr);
 731         physaddr = DDR_VIRT_TO_PFE(ddr_ptr);
 732
 733         lmem_virt_addr = (void *)CBUS_PFE_TO_VIRT((unsigned long)lmem_ptr);
 734
 735         local_hdr.phyno = htons(0); /* RX_PHY_0 */
 736         local_hdr.length = htons(MIN_PKT_SIZE);
 737
 738         local_hdr.next_ptr = htonl((u32)physaddr);
 739         /*Mark checksum is correct */
 740         local_hdr.status = htonl((STATUS_IP_CHECKSUM_CORRECT |
 741                                 STATUS_UDP_CHECKSUM_CORRECT |
 742                                 STATUS_TCP_CHECKSUM_CORRECT |
 743                                 STATUS_UNICAST_HASH_MATCH |
 744                                 STATUS_CUMULATIVE_ARC_HIT));
 745         copy_to_lmem((u32 *)lmem_virt_addr, (u32 *)&local_hdr,
 746                      sizeof(local_hdr));
 747
 748         copy_to_lmem((u32 *)(lmem_virt_addr + LMEM_HDR_SIZE), (u32 *)dummy_pkt,
 749                      0x40);
 750
 751         writel((unsigned long)lmem_ptr, CLASS_INQ_PKTPTR);
 752 }
 753
 754 void
 755 pfe_hif_rx_idle(struct pfe_hif *hif)
 756 {
 757         int hif_stop_loop = DUMMY_PKT_COUNT;
 758         u32 rx_status;
 759
 760         pfe_hif_disable_rx_desc(hif);
 761         PFE_PMD_INFO("Bringing hif to idle state...");
 762         writel(0, HIF_INT_ENABLE);
 763         /*If HIF Rx BDP is busy send a dummy packet */
 764         do {
 765                 rx_status = readl(HIF_RX_STATUS);
 766                 if (rx_status & BDP_CSR_RX_DMA_ACTV)
 767                         send_dummy_pkt_to_hif();
 768
 769                 sleep(1);
 770         } while (--hif_stop_loop);
 771
 772         if (readl(HIF_RX_STATUS) & BDP_CSR_RX_DMA_ACTV)
 773                 PFE_PMD_ERR("Failed\n");
 774         else
 775                 PFE_PMD_INFO("Done\n");
 776 }
 777 #endif
 778
 779 /*
 780  * pfe_hif_init
 781  * This function initializes the baseaddresses and irq, etc.
 782  */
 783 int
 784 pfe_hif_init(struct pfe *pfe)
 785 {
 786         struct pfe_hif *hif = &pfe->hif;
 787         int err;
 788
 789         PMD_INIT_FUNC_TRACE();
 790
 791 #if defined(LS1012A_PFE_RESET_WA)
 792         pfe_hif_rx_idle(hif);
 793 #endif
 794
 795         err = pfe_hif_alloc_descr(hif);
 796         if (err)
 797                 goto err0;
 798
 799         rte_spinlock_init(&hif->tx_lock);
 800         rte_spinlock_init(&hif->lock);
 801
 802         gpi_enable(HGPI_BASE_ADDR);
 803         if (getenv("PFE_INTR_SUPPORT")) {
 804                 struct epoll_event epoll_ev;
 805                 int event_fd = -1, epoll_fd, pfe_cdev_fd;
 806
 807                 pfe_cdev_fd = open(PFE_CDEV_PATH, O_RDWR);
 808                 if (pfe_cdev_fd < 0) {
 809                         PFE_PMD_WARN("Unable to open PFE device file (%s).\n",
 810                                      PFE_CDEV_PATH);
 811                         pfe->cdev_fd = PFE_CDEV_INVALID_FD;
 812                         return -1;
 813                 }
 814                 pfe->cdev_fd = pfe_cdev_fd;
 815
 816                 event_fd = eventfd(0, EFD_NONBLOCK);
 817                 /* hif interrupt enable */
 818                 err = ioctl(pfe->cdev_fd, PFE_CDEV_HIF_INTR_EN, &event_fd);
 819                 if (err) {
 820                         PFE_PMD_ERR("\nioctl failed for intr enable err: %d\n",
 821                                         errno);
 822                         goto err0;
 823                 }
 824                 epoll_fd = epoll_create(1);
 825                 epoll_ev.events = EPOLLIN | EPOLLPRI | EPOLLET;
 826                 epoll_ev.data.fd = event_fd;
 827                 err = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event_fd, &epoll_ev);
 828                 if (err < 0) {
 829                         PFE_PMD_ERR("epoll_ctl failed with err = %d\n", errno);
 830                         goto err0;
 831                 }
 832                 pfe->hif.epoll_fd = epoll_fd;
 833         }
 834         return 0;
 835 err0:
 836         return err;
 837 }
 838
 839 /* pfe_hif_exit- */
 840 void
 841 pfe_hif_exit(struct pfe *pfe)
 842 {
 843         struct pfe_hif *hif = &pfe->hif;
 844
 845         PMD_INIT_FUNC_TRACE();
 846
 847         rte_spinlock_lock(&hif->lock);
 848         hif->shm->g_client_status[0] = 0;
 849         /* Make sure all clients are disabled*/
 850         hif->shm->g_client_status[1] = 0;
 851
 852         rte_spinlock_unlock(&hif->lock);
 853
 854         if (hif->setuped) {
 855 #if defined(LS1012A_PFE_RESET_WA)
 856                 pfe_hif_rx_idle(hif);
 857 #endif
 858                 /*Disable Rx/Tx */
 859                 hif_rx_disable();
 860                 hif_tx_disable();
 861
 862                 pfe_hif_release_buffers(hif);
 863                 pfe_hif_shm_clean(hif->shm);
 864
 865                 pfe_hif_free_descr(hif);
 866                 pfe->hif.setuped = 0;
 867         }
 868         gpi_disable(HGPI_BASE_ADDR);
 869 }