drivers/net/pfe/pfe_hif.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 NXP
   3  */
   4
   5 #include "pfe_logs.h"
   6 #include "pfe_mod.h"
   7 #include <sys/ioctl.h>
   8 #include <sys/epoll.h>
   9 #include <sys/eventfd.h>
  10
  11 static int
  12 pfe_hif_alloc_descr(struct pfe_hif *hif)
  13 {
  14         void *addr;
  15         int err = 0;
  16
  17         PMD_INIT_FUNC_TRACE();
  18
  19         addr = rte_zmalloc(NULL, HIF_RX_DESC_NT * sizeof(struct hif_desc) +
  20                 HIF_TX_DESC_NT * sizeof(struct hif_desc), RTE_CACHE_LINE_SIZE);
  21         if (!addr) {
  22                 PFE_PMD_ERR("Could not allocate buffer descriptors!");
  23                 err = -ENOMEM;
  24                 goto err0;
  25         }
  26
  27         hif->descr_baseaddr_p = pfe_mem_vtop((uintptr_t)addr);
  28         hif->descr_baseaddr_v = addr;
  29         hif->rx_ring_size = HIF_RX_DESC_NT;
  30         hif->tx_ring_size = HIF_TX_DESC_NT;
  31
  32         return 0;
  33
  34 err0:
  35         return err;
  36 }
  37
  38 static void
  39 pfe_hif_free_descr(struct pfe_hif *hif)
  40 {
  41         PMD_INIT_FUNC_TRACE();
  42
  43         rte_free(hif->descr_baseaddr_v);
  44 }
  45
  46 /* pfe_hif_release_buffers */
  47 static void
  48 pfe_hif_release_buffers(struct pfe_hif *hif)
  49 {
  50         struct hif_desc *desc;
  51         uint32_t i = 0;
  52         struct rte_mbuf *mbuf;
  53         struct rte_pktmbuf_pool_private *mb_priv;
  54
  55         hif->rx_base = hif->descr_baseaddr_v;
  56
  57         /*Free Rx buffers */
  58         desc = hif->rx_base;
  59         mb_priv = rte_mempool_get_priv(hif->shm->pool);
  60         for (i = 0; i < hif->rx_ring_size; i++) {
  61                 if (readl(&desc->data)) {
  62                         if (i < hif->shm->rx_buf_pool_cnt &&
  63                             !hif->shm->rx_buf_pool[i]) {
  64                                 mbuf = hif->rx_buf_vaddr[i] + PFE_PKT_HEADER_SZ
  65                                         - sizeof(struct rte_mbuf)
  66                                         - RTE_PKTMBUF_HEADROOM
  67                                         - mb_priv->mbuf_priv_size;
  68                                 hif->shm->rx_buf_pool[i] = mbuf;
  69                         }
  70                 }
  71                 writel(0, &desc->data);
  72                 writel(0, &desc->status);
  73                 writel(0, &desc->ctrl);
  74                 desc++;
  75         }
  76 }
  77
  78 /*
  79  * pfe_hif_init_buffers
  80  * This function initializes the HIF Rx/Tx ring descriptors and
  81  * initialize Rx queue with buffers.
  82  */
  83 int
  84 pfe_hif_init_buffers(struct pfe_hif *hif)
  85 {
  86         struct hif_desc *desc, *first_desc_p;
  87         uint32_t i = 0;
  88
  89         PMD_INIT_FUNC_TRACE();
  90
  91         /* Check enough Rx buffers available in the shared memory */
  92         if (hif->shm->rx_buf_pool_cnt < hif->rx_ring_size)
  93                 return -ENOMEM;
  94
  95         hif->rx_base = hif->descr_baseaddr_v;
  96         memset(hif->rx_base, 0, hif->rx_ring_size * sizeof(struct hif_desc));
  97
  98         /*Initialize Rx descriptors */
  99         desc = hif->rx_base;
 100         first_desc_p = (struct hif_desc *)hif->descr_baseaddr_p;
 101
 102         for (i = 0; i < hif->rx_ring_size; i++) {
 103                 /* Initialize Rx buffers from the shared memory */
 104                 struct rte_mbuf *mbuf =
 105                         (struct rte_mbuf *)hif->shm->rx_buf_pool[i];
 106
 107                 /* PFE mbuf structure is as follow:
 108                  * ----------------------------------------------------------+
 109                  * | mbuf  | priv | headroom (annotation + PFE data) | data  |
 110                  * ----------------------------------------------------------+
 111                  *
 112                  * As we are expecting additional information like parse
 113                  * results, eth id, queue id from PFE block along with data.
 114                  * so we have to provide additional memory for each packet to
 115                  * HIF rx rings so that PFE block can write its headers.
 116                  * so, we are giving the data pointor to HIF rings whose
 117                  * calculation is as below:
 118                  * mbuf->data_pointor - Required_header_size
 119                  *
 120                  * We are utilizing the HEADROOM area to receive the PFE
 121                  * block headers. On packet reception, HIF driver will use
 122                  * PFE headers information based on which it will decide
 123                  * the clients and fill the parse results.
 124                  * after that application can use/overwrite the HEADROOM area.
 125                  */
 126                 hif->rx_buf_vaddr[i] =
 127                         (void *)((size_t)mbuf->buf_addr + mbuf->data_off -
 128                                         PFE_PKT_HEADER_SZ);
 129                 hif->rx_buf_addr[i] =
 130                         (void *)(size_t)(rte_pktmbuf_iova(mbuf) -
 131                                         PFE_PKT_HEADER_SZ);
 132                 hif->rx_buf_len[i] =  mbuf->buf_len - RTE_PKTMBUF_HEADROOM;
 133
 134                 hif->shm->rx_buf_pool[i] = NULL;
 135
 136                 writel(DDR_PHYS_TO_PFE(hif->rx_buf_addr[i]),
 137                                         &desc->data);
 138                 writel(0, &desc->status);
 139
 140                 /*
 141                  * Ensure everything else is written to DDR before
 142                  * writing bd->ctrl
 143                  */
 144                 rte_wmb();
 145
 146                 writel((BD_CTRL_PKT_INT_EN | BD_CTRL_LIFM
 147                         | BD_CTRL_DIR | BD_CTRL_DESC_EN
 148                         | BD_BUF_LEN(hif->rx_buf_len[i])), &desc->ctrl);
 149
 150                 /* Chain descriptors */
 151                 writel((u32)DDR_PHYS_TO_PFE(first_desc_p + i + 1), &desc->next);
 152                 desc++;
 153         }
 154
 155         /* Overwrite last descriptor to chain it to first one*/
 156         desc--;
 157         writel((u32)DDR_PHYS_TO_PFE(first_desc_p), &desc->next);
 158
 159         hif->rxtoclean_index = 0;
 160
 161         /*Initialize Rx buffer descriptor ring base address */
 162         writel(DDR_PHYS_TO_PFE(hif->descr_baseaddr_p), HIF_RX_BDP_ADDR);
 163
 164         hif->tx_base = hif->rx_base + hif->rx_ring_size;
 165         first_desc_p = (struct hif_desc *)hif->descr_baseaddr_p +
 166                                 hif->rx_ring_size;
 167         memset(hif->tx_base, 0, hif->tx_ring_size * sizeof(struct hif_desc));
 168
 169         /*Initialize tx descriptors */
 170         desc = hif->tx_base;
 171
 172         for (i = 0; i < hif->tx_ring_size; i++) {
 173                 /* Chain descriptors */
 174                 writel((u32)DDR_PHYS_TO_PFE(first_desc_p + i + 1), &desc->next);
 175                 writel(0, &desc->ctrl);
 176                 desc++;
 177         }
 178
 179         /* Overwrite last descriptor to chain it to first one */
 180         desc--;
 181         writel((u32)DDR_PHYS_TO_PFE(first_desc_p), &desc->next);
 182         hif->txavail = hif->tx_ring_size;
 183         hif->txtosend = 0;
 184         hif->txtoclean = 0;
 185         hif->txtoflush = 0;
 186
 187         /*Initialize Tx buffer descriptor ring base address */
 188         writel((u32)DDR_PHYS_TO_PFE(first_desc_p), HIF_TX_BDP_ADDR);
 189
 190         return 0;
 191 }
 192
 193 /*
 194  * pfe_hif_client_register
 195  *
 196  * This function used to register a client driver with the HIF driver.
 197  *
 198  * Return value:
 199  * 0 - on Successful registration
 200  */
 201 static int
 202 pfe_hif_client_register(struct pfe_hif *hif, u32 client_id,
 203                         struct hif_client_shm *client_shm)
 204 {
 205         struct hif_client *client = &hif->client[client_id];
 206         u32 i, cnt;
 207         struct rx_queue_desc *rx_qbase;
 208         struct tx_queue_desc *tx_qbase;
 209         struct hif_rx_queue *rx_queue;
 210         struct hif_tx_queue *tx_queue;
 211         int err = 0;
 212
 213         PMD_INIT_FUNC_TRACE();
 214
 215         rte_spinlock_lock(&hif->tx_lock);
 216
 217         if (test_bit(client_id, &hif->shm->g_client_status[0])) {
 218                 PFE_PMD_ERR("client %d already registered", client_id);
 219                 err = -1;
 220                 goto unlock;
 221         }
 222
 223         memset(client, 0, sizeof(struct hif_client));
 224
 225         /* Initialize client Rx queues baseaddr, size */
 226
 227         cnt = CLIENT_CTRL_RX_Q_CNT(client_shm->ctrl);
 228         /* Check if client is requesting for more queues than supported */
 229         if (cnt > HIF_CLIENT_QUEUES_MAX)
 230                 cnt = HIF_CLIENT_QUEUES_MAX;
 231
 232         client->rx_qn = cnt;
 233         rx_qbase = (struct rx_queue_desc *)client_shm->rx_qbase;
 234         for (i = 0; i < cnt; i++) {
 235                 rx_queue = &client->rx_q[i];
 236                 rx_queue->base = rx_qbase + i * client_shm->rx_qsize;
 237                 rx_queue->size = client_shm->rx_qsize;
 238                 rx_queue->write_idx = 0;
 239         }
 240
 241         /* Initialize client Tx queues baseaddr, size */
 242         cnt = CLIENT_CTRL_TX_Q_CNT(client_shm->ctrl);
 243
 244         /* Check if client is requesting for more queues than supported */
 245         if (cnt > HIF_CLIENT_QUEUES_MAX)
 246                 cnt = HIF_CLIENT_QUEUES_MAX;
 247
 248         client->tx_qn = cnt;
 249         tx_qbase = (struct tx_queue_desc *)client_shm->tx_qbase;
 250         for (i = 0; i < cnt; i++) {
 251                 tx_queue = &client->tx_q[i];
 252                 tx_queue->base = tx_qbase + i * client_shm->tx_qsize;
 253                 tx_queue->size = client_shm->tx_qsize;
 254                 tx_queue->ack_idx = 0;
 255         }
 256
 257         set_bit(client_id, &hif->shm->g_client_status[0]);
 258
 259 unlock:
 260         rte_spinlock_unlock(&hif->tx_lock);
 261
 262         return err;
 263 }
 264
 265 /*
 266  * pfe_hif_client_unregister
 267  *
 268  * This function used to unregister a client  from the HIF driver.
 269  *
 270  */
 271 static void
 272 pfe_hif_client_unregister(struct pfe_hif *hif, u32 client_id)
 273 {
 274         PMD_INIT_FUNC_TRACE();
 275
 276         /*
 277          * Mark client as no longer available (which prevents further packet
 278          * receive for this client)
 279          */
 280         rte_spinlock_lock(&hif->tx_lock);
 281
 282         if (!test_bit(client_id, &hif->shm->g_client_status[0])) {
 283                 PFE_PMD_ERR("client %d not registered", client_id);
 284
 285                 rte_spinlock_unlock(&hif->tx_lock);
 286                 return;
 287         }
 288
 289         clear_bit(client_id, &hif->shm->g_client_status[0]);
 290
 291         rte_spinlock_unlock(&hif->tx_lock);
 292 }
 293
 294 /*
 295  * client_put_rxpacket-
 296  */
 297 static struct rte_mbuf *
 298 client_put_rxpacket(struct hif_rx_queue *queue,
 299                 void *pkt, u32 len,
 300                 u32 flags, u32 client_ctrl,
 301                 struct rte_mempool *pool,
 302                 u32 *rem_len)
 303 {
 304         struct rx_queue_desc *desc = queue->base + queue->write_idx;
 305         struct rte_mbuf *mbuf = NULL;
 306
 307
 308         if (readl(&desc->ctrl) & CL_DESC_OWN) {
 309                 mbuf = rte_cpu_to_le_64(rte_pktmbuf_alloc(pool));
 310                 if (unlikely(!mbuf)) {
 311                         PFE_PMD_WARN("Buffer allocation failure\n");
 312                         return NULL;
 313                 }
 314
 315                 desc->data = pkt;
 316                 desc->client_ctrl = client_ctrl;
 317                 /*
 318                  * Ensure everything else is written to DDR before
 319                  * writing bd->ctrl
 320                  */
 321                 rte_wmb();
 322                 writel(CL_DESC_BUF_LEN(len) | flags, &desc->ctrl);
 323                 queue->write_idx = (queue->write_idx + 1)
 324                                     & (queue->size - 1);
 325
 326                 *rem_len = mbuf->buf_len;
 327         }
 328
 329         return mbuf;
 330 }
 331
 332 /*
 333  * pfe_hif_rx_process-
 334  * This function does pfe hif rx queue processing.
 335  * Dequeue packet from Rx queue and send it to corresponding client queue
 336  */
 337 int
 338 pfe_hif_rx_process(struct pfe *pfe, int budget)
 339 {
 340         struct hif_desc *desc;
 341         struct hif_hdr *pkt_hdr;
 342         struct __hif_hdr hif_hdr;
 343         void *free_buf;
 344         int rtc, len, rx_processed = 0;
 345         struct __hif_desc local_desc;
 346         int flags = 0, wait_for_last = 0, retry = 0;
 347         unsigned int buf_size = 0;
 348         struct rte_mbuf *mbuf = NULL;
 349         struct pfe_hif *hif = &pfe->hif;
 350
 351         rte_spinlock_lock(&hif->lock);
 352
 353         rtc = hif->rxtoclean_index;
 354
 355         while (rx_processed < budget) {
 356                 desc = hif->rx_base + rtc;
 357
 358                 __memcpy12(&local_desc, desc);
 359
 360                 /* ACK pending Rx interrupt */
 361                 if (local_desc.ctrl & BD_CTRL_DESC_EN) {
 362                         if (unlikely(wait_for_last))
 363                                 continue;
 364                         else
 365                                 break;
 366                 }
 367
 368                 len = BD_BUF_LEN(local_desc.ctrl);
 369                 pkt_hdr = (struct hif_hdr *)hif->rx_buf_vaddr[rtc];
 370
 371                 /* Track last HIF header received */
 372                 if (!hif->started) {
 373                         hif->started = 1;
 374
 375                         __memcpy8(&hif_hdr, pkt_hdr);
 376
 377                         hif->qno = hif_hdr.hdr.q_num;
 378                         hif->client_id = hif_hdr.hdr.client_id;
 379                         hif->client_ctrl = (hif_hdr.hdr.client_ctrl1 << 16) |
 380                                                 hif_hdr.hdr.client_ctrl;
 381                         flags = CL_DESC_FIRST;
 382
 383                 } else {
 384                         flags = 0;
 385                 }
 386
 387                 if (local_desc.ctrl & BD_CTRL_LIFM) {
 388                         flags |= CL_DESC_LAST;
 389                         wait_for_last = 0;
 390                 } else {
 391                         wait_for_last = 1;
 392                 }
 393
 394                 /* Check for valid client id and still registered */
 395                 if (hif->client_id >= HIF_CLIENTS_MAX ||
 396                     !(test_bit(hif->client_id,
 397                         &hif->shm->g_client_status[0]))) {
 398                         PFE_PMD_INFO("packet with invalid client id %d qnum %d",
 399                                 hif->client_id, hif->qno);
 400
 401                         free_buf = hif->rx_buf_addr[rtc];
 402
 403                         goto pkt_drop;
 404                 }
 405
 406                 /* Check to valid queue number */
 407                 if (hif->client[hif->client_id].rx_qn <= hif->qno) {
 408                         PFE_DP_LOG(DEBUG, "packet with invalid queue: %d",
 409                                         hif->qno);
 410                         hif->qno = 0;
 411                 }
 412
 413 retry:
 414                 mbuf =
 415                 client_put_rxpacket(&hif->client[hif->client_id].rx_q[hif->qno],
 416                                     (void *)pkt_hdr, len, flags,
 417                                     hif->client_ctrl, hif->shm->pool,
 418                                     &buf_size);
 419
 420                 if (unlikely(!mbuf)) {
 421                         if (!retry) {
 422                                 pfe_tx_do_cleanup(pfe);
 423                                 retry = 1;
 424                                 goto retry;
 425                         }
 426                         rx_processed = budget;
 427
 428                         if (flags & CL_DESC_FIRST)
 429                                 hif->started = 0;
 430
 431                         PFE_DP_LOG(DEBUG, "No buffers");
 432                         break;
 433                 }
 434
 435                 retry = 0;
 436
 437                 free_buf = (void *)(size_t)rte_pktmbuf_iova(mbuf);
 438                 free_buf = free_buf - PFE_PKT_HEADER_SZ;
 439
 440                 /*Fill free buffer in the descriptor */
 441                 hif->rx_buf_addr[rtc] = free_buf;
 442                 hif->rx_buf_vaddr[rtc] = (void *)((size_t)mbuf->buf_addr +
 443                                 mbuf->data_off - PFE_PKT_HEADER_SZ);
 444                 hif->rx_buf_len[rtc] = buf_size - RTE_PKTMBUF_HEADROOM;
 445
 446 pkt_drop:
 447                 writel(DDR_PHYS_TO_PFE(free_buf), &desc->data);
 448                 /*
 449                  * Ensure everything else is written to DDR before
 450                  * writing bd->ctrl
 451                  */
 452                 rte_wmb();
 453                 writel((BD_CTRL_PKT_INT_EN | BD_CTRL_LIFM | BD_CTRL_DIR |
 454                         BD_CTRL_DESC_EN | BD_BUF_LEN(hif->rx_buf_len[rtc])),
 455                         &desc->ctrl);
 456
 457                 rtc = (rtc + 1) & (hif->rx_ring_size - 1);
 458
 459                 if (local_desc.ctrl & BD_CTRL_LIFM) {
 460                         if (!(hif->client_ctrl & HIF_CTRL_RX_CONTINUED))
 461                                 rx_processed++;
 462
 463                         hif->started = 0;
 464                 }
 465         }
 466
 467
 468         hif->rxtoclean_index = rtc;
 469         rte_spinlock_unlock(&hif->lock);
 470
 471         /* we made some progress, re-start rx dma in case it stopped */
 472         hif_rx_dma_start();
 473
 474         return rx_processed;
 475 }
 476
 477 /*
 478  * client_ack_txpacket-
 479  * This function ack the Tx packet in the give client Tx queue by resetting
 480  * ownership bit in the descriptor.
 481  */
 482 static int
 483 client_ack_txpacket(struct pfe_hif *hif, unsigned int client_id,
 484                     unsigned int q_no)
 485 {
 486         struct hif_tx_queue *queue = &hif->client[client_id].tx_q[q_no];
 487         struct tx_queue_desc *desc = queue->base + queue->ack_idx;
 488
 489         if (readl(&desc->ctrl) & CL_DESC_OWN) {
 490                 writel((readl(&desc->ctrl) & ~CL_DESC_OWN), &desc->ctrl);
 491                 queue->ack_idx = (queue->ack_idx + 1) & (queue->size - 1);
 492
 493                 return 0;
 494
 495         } else {
 496                 /*This should not happen */
 497                 PFE_PMD_ERR("%d %d %d %d %d %p %d",
 498                        hif->txtosend, hif->txtoclean, hif->txavail,
 499                         client_id, q_no, queue, queue->ack_idx);
 500                 return 1;
 501         }
 502 }
 503
 504 static void
 505 __hif_tx_done_process(struct pfe *pfe, int count)
 506 {
 507         struct hif_desc *desc;
 508         struct hif_desc_sw *desc_sw;
 509         unsigned int ttc, tx_avl;
 510         int pkts_done[HIF_CLIENTS_MAX] = {0, 0};
 511         struct pfe_hif *hif = &pfe->hif;
 512
 513         ttc = hif->txtoclean;
 514         tx_avl = hif->txavail;
 515
 516         while ((tx_avl < hif->tx_ring_size) && count--) {
 517                 desc = hif->tx_base + ttc;
 518
 519                 if (readl(&desc->ctrl) & BD_CTRL_DESC_EN)
 520                         break;
 521
 522                 desc_sw = &hif->tx_sw_queue[ttc];
 523
 524                 if (desc_sw->client_id > HIF_CLIENTS_MAX)
 525                         PFE_PMD_ERR("Invalid cl id %d", desc_sw->client_id);
 526
 527                 pkts_done[desc_sw->client_id]++;
 528
 529                 client_ack_txpacket(hif, desc_sw->client_id, desc_sw->q_no);
 530
 531                 ttc = (ttc + 1) & (hif->tx_ring_size - 1);
 532                 tx_avl++;
 533         }
 534
 535         if (pkts_done[0])
 536                 hif_lib_indicate_client(pfe->hif_client[0], EVENT_TXDONE_IND,
 537                                 0);
 538         if (pkts_done[1])
 539                 hif_lib_indicate_client(pfe->hif_client[1], EVENT_TXDONE_IND,
 540                                 0);
 541         hif->txtoclean = ttc;
 542         hif->txavail = tx_avl;
 543 }
 544
 545 static inline void
 546 hif_tx_done_process(struct pfe *pfe, int count)
 547 {
 548         struct pfe_hif *hif = &pfe->hif;
 549         rte_spinlock_lock(&hif->tx_lock);
 550         __hif_tx_done_process(pfe, count);
 551         rte_spinlock_unlock(&hif->tx_lock);
 552 }
 553
 554 void
 555 pfe_tx_do_cleanup(struct pfe *pfe)
 556 {
 557         hif_tx_done_process(pfe, HIF_TX_DESC_NT);
 558 }
 559
 560 /*
 561  * __hif_xmit_pkt -
 562  * This function puts one packet in the HIF Tx queue
 563  */
 564 void
 565 hif_xmit_pkt(struct pfe_hif *hif, unsigned int client_id, unsigned int
 566              q_no, void *data, u32 len, unsigned int flags)
 567 {
 568         struct hif_desc *desc;
 569         struct hif_desc_sw *desc_sw;
 570
 571         desc = hif->tx_base + hif->txtosend;
 572         desc_sw = &hif->tx_sw_queue[hif->txtosend];
 573
 574         desc_sw->len = len;
 575         desc_sw->client_id = client_id;
 576         desc_sw->q_no = q_no;
 577         desc_sw->flags = flags;
 578
 579         writel((u32)DDR_PHYS_TO_PFE(data), &desc->data);
 580
 581         hif->txtosend = (hif->txtosend + 1) & (hif->tx_ring_size - 1);
 582         hif->txavail--;
 583
 584         if ((!((flags & HIF_DATA_VALID) && (flags &
 585                                 HIF_LAST_BUFFER))))
 586                 goto skip_tx;
 587
 588         /*
 589          * Ensure everything else is written to DDR before
 590          * writing bd->ctrl
 591          */
 592         rte_wmb();
 593
 594         do {
 595                 desc_sw = &hif->tx_sw_queue[hif->txtoflush];
 596                 desc = hif->tx_base + hif->txtoflush;
 597
 598                 if (desc_sw->flags & HIF_LAST_BUFFER) {
 599                         writel((BD_CTRL_LIFM |
 600                                BD_CTRL_BRFETCH_DISABLE | BD_CTRL_RTFETCH_DISABLE
 601                                | BD_CTRL_PARSE_DISABLE | BD_CTRL_DESC_EN |
 602                                  BD_BUF_LEN(desc_sw->len)),
 603                                 &desc->ctrl);
 604                 } else {
 605                         writel((BD_CTRL_DESC_EN |
 606                                 BD_BUF_LEN(desc_sw->len)), &desc->ctrl);
 607                 }
 608                 hif->txtoflush = (hif->txtoflush + 1) & (hif->tx_ring_size - 1);
 609         }
 610         while (hif->txtoflush != hif->txtosend)
 611                 ;
 612
 613 skip_tx:
 614         return;
 615 }
 616
 617 void
 618 hif_process_client_req(struct pfe_hif *hif, int req,
 619                             int data1, __rte_unused int data2)
 620 {
 621         unsigned int client_id = data1;
 622
 623         if (client_id >= HIF_CLIENTS_MAX) {
 624                 PFE_PMD_ERR("client id %d out of bounds", client_id);
 625                 return;
 626         }
 627
 628         switch (req) {
 629         case REQUEST_CL_REGISTER:
 630                         /* Request for register a client */
 631                         PFE_PMD_INFO("register client_id %d", client_id);
 632                         pfe_hif_client_register(hif, client_id, (struct
 633                                 hif_client_shm *)&hif->shm->client[client_id]);
 634                         break;
 635
 636         case REQUEST_CL_UNREGISTER:
 637                         PFE_PMD_INFO("unregister client_id %d", client_id);
 638
 639                         /* Request for unregister a client */
 640                         pfe_hif_client_unregister(hif, client_id);
 641
 642                         break;
 643
 644         default:
 645                         PFE_PMD_ERR("unsupported request %d", req);
 646                         break;
 647         }
 648
 649         /*
 650          * Process client Tx queues
 651          * Currently we don't have checking for tx pending
 652          */
 653 }
 654
 655 #if defined(LS1012A_PFE_RESET_WA)
 656 static void
 657 pfe_hif_disable_rx_desc(struct pfe_hif *hif)
 658 {
 659         u32 ii;
 660         struct hif_desc *desc = hif->rx_base;
 661
 662         /*Mark all descriptors as LAST_BD */
 663         for (ii = 0; ii < hif->rx_ring_size; ii++) {
 664                 desc->ctrl |= BD_CTRL_LAST_BD;
 665                 desc++;
 666         }
 667 }
 668
 669 struct class_rx_hdr_t {
 670         u32     next_ptr;       /* ptr to the start of the first DDR buffer */
 671         u16     length;         /* total packet length */
 672         u16     phyno;          /* input physical port number */
 673         u32     status;         /* gemac status bits */
 674         u32     status2;            /* reserved for software usage */
 675 };
 676
 677 /* STATUS_BAD_FRAME_ERR is set for all errors (including checksums if enabled)
 678  * except overflow
 679  */
 680 #define STATUS_BAD_FRAME_ERR            BIT(16)
 681 #define STATUS_LENGTH_ERR               BIT(17)
 682 #define STATUS_CRC_ERR                  BIT(18)
 683 #define STATUS_TOO_SHORT_ERR            BIT(19)
 684 #define STATUS_TOO_LONG_ERR             BIT(20)
 685 #define STATUS_CODE_ERR                 BIT(21)
 686 #define STATUS_MC_HASH_MATCH            BIT(22)
 687 #define STATUS_CUMULATIVE_ARC_HIT       BIT(23)
 688 #define STATUS_UNICAST_HASH_MATCH       BIT(24)
 689 #define STATUS_IP_CHECKSUM_CORRECT      BIT(25)
 690 #define STATUS_TCP_CHECKSUM_CORRECT     BIT(26)
 691 #define STATUS_UDP_CHECKSUM_CORRECT     BIT(27)
 692 #define STATUS_OVERFLOW_ERR             BIT(28) /* GPI error */
 693 #define MIN_PKT_SIZE                    64
 694 #define DUMMY_PKT_COUNT                 128
 695
 696 static inline void
 697 copy_to_lmem(u32 *dst, u32 *src, int len)
 698 {
 699         int i;
 700
 701         for (i = 0; i < len; i += sizeof(u32))  {
 702                 *dst = htonl(*src);
 703                 dst++; src++;
 704         }
 705 }
 706 #if defined(RTE_TOOLCHAIN_GCC)
 707 __attribute__ ((optimize(1)))
 708 #endif
 709 static void
 710 send_dummy_pkt_to_hif(void)
 711 {
 712         void *lmem_ptr, *ddr_ptr, *lmem_virt_addr;
 713         u64 physaddr;
 714         struct class_rx_hdr_t local_hdr;
 715         static u32 dummy_pkt[] =  {
 716                 0x33221100, 0x2b785544, 0xd73093cb, 0x01000608,
 717                 0x04060008, 0x2b780200, 0xd73093cb, 0x0a01a8c0,
 718                 0x33221100, 0xa8c05544, 0x00000301, 0x00000000,
 719                 0x00000000, 0x00000000, 0x00000000, 0xbe86c51f };
 720
 721         ddr_ptr = (void *)(size_t)readl(BMU2_BASE_ADDR + BMU_ALLOC_CTRL);
 722         if (!ddr_ptr)
 723                 return;
 724
 725         lmem_ptr = (void *)(size_t)readl(BMU1_BASE_ADDR + BMU_ALLOC_CTRL);
 726         if (!lmem_ptr)
 727                 return;
 728
 729         PFE_PMD_INFO("Sending a dummy pkt to HIF %p %p", ddr_ptr, lmem_ptr);
 730         physaddr = DDR_VIRT_TO_PFE(ddr_ptr);
 731
 732         lmem_virt_addr = (void *)CBUS_PFE_TO_VIRT((unsigned long)lmem_ptr);
 733
 734         local_hdr.phyno = htons(0); /* RX_PHY_0 */
 735         local_hdr.length = htons(MIN_PKT_SIZE);
 736
 737         local_hdr.next_ptr = htonl((u32)physaddr);
 738         /*Mark checksum is correct */
 739         local_hdr.status = htonl((STATUS_IP_CHECKSUM_CORRECT |
 740                                 STATUS_UDP_CHECKSUM_CORRECT |
 741                                 STATUS_TCP_CHECKSUM_CORRECT |
 742                                 STATUS_UNICAST_HASH_MATCH |
 743                                 STATUS_CUMULATIVE_ARC_HIT));
 744         copy_to_lmem((u32 *)lmem_virt_addr, (u32 *)&local_hdr,
 745                      sizeof(local_hdr));
 746
 747         copy_to_lmem((u32 *)(lmem_virt_addr + LMEM_HDR_SIZE), (u32 *)dummy_pkt,
 748                      0x40);
 749
 750         writel((unsigned long)lmem_ptr, CLASS_INQ_PKTPTR);
 751 }
 752
 753 void
 754 pfe_hif_rx_idle(struct pfe_hif *hif)
 755 {
 756         int hif_stop_loop = DUMMY_PKT_COUNT;
 757         u32 rx_status;
 758
 759         pfe_hif_disable_rx_desc(hif);
 760         PFE_PMD_INFO("Bringing hif to idle state...");
 761         writel(0, HIF_INT_ENABLE);
 762         /*If HIF Rx BDP is busy send a dummy packet */
 763         do {
 764                 rx_status = readl(HIF_RX_STATUS);
 765                 if (rx_status & BDP_CSR_RX_DMA_ACTV)
 766                         send_dummy_pkt_to_hif();
 767
 768                 sleep(1);
 769         } while (--hif_stop_loop);
 770
 771         if (readl(HIF_RX_STATUS) & BDP_CSR_RX_DMA_ACTV)
 772                 PFE_PMD_ERR("Failed\n");
 773         else
 774                 PFE_PMD_INFO("Done\n");
 775 }
 776 #endif
 777
 778 /*
 779  * pfe_hif_init
 780  * This function initializes the baseaddresses and irq, etc.
 781  */
 782 int
 783 pfe_hif_init(struct pfe *pfe)
 784 {
 785         struct pfe_hif *hif = &pfe->hif;
 786         int err;
 787
 788         PMD_INIT_FUNC_TRACE();
 789
 790 #if defined(LS1012A_PFE_RESET_WA)
 791         pfe_hif_rx_idle(hif);
 792 #endif
 793
 794         err = pfe_hif_alloc_descr(hif);
 795         if (err)
 796                 goto err0;
 797
 798         rte_spinlock_init(&hif->tx_lock);
 799         rte_spinlock_init(&hif->lock);
 800
 801         gpi_enable(HGPI_BASE_ADDR);
 802         if (getenv("PFE_INTR_SUPPORT")) {
 803                 struct epoll_event epoll_ev;
 804                 int event_fd = -1, epoll_fd, pfe_cdev_fd;
 805
 806                 pfe_cdev_fd = open(PFE_CDEV_PATH, O_RDWR);
 807                 if (pfe_cdev_fd < 0) {
 808                         PFE_PMD_WARN("Unable to open PFE device file (%s).\n",
 809                                      PFE_CDEV_PATH);
 810                         pfe->cdev_fd = PFE_CDEV_INVALID_FD;
 811                         return -1;
 812                 }
 813                 pfe->cdev_fd = pfe_cdev_fd;
 814
 815                 event_fd = eventfd(0, EFD_NONBLOCK);
 816                 /* hif interrupt enable */
 817                 err = ioctl(pfe->cdev_fd, PFE_CDEV_HIF_INTR_EN, &event_fd);
 818                 if (err) {
 819                         PFE_PMD_ERR("\nioctl failed for intr enable err: %d\n",
 820                                         errno);
 821                         goto err0;
 822                 }
 823                 epoll_fd = epoll_create(1);
 824                 epoll_ev.events = EPOLLIN | EPOLLPRI | EPOLLET;
 825                 epoll_ev.data.fd = event_fd;
 826                 err = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, event_fd, &epoll_ev);
 827                 if (err < 0) {
 828                         PFE_PMD_ERR("epoll_ctl failed with err = %d\n", errno);
 829                         goto err0;
 830                 }
 831                 pfe->hif.epoll_fd = epoll_fd;
 832         }
 833         return 0;
 834 err0:
 835         return err;
 836 }
 837
 838 /* pfe_hif_exit- */
 839 void
 840 pfe_hif_exit(struct pfe *pfe)
 841 {
 842         struct pfe_hif *hif = &pfe->hif;
 843
 844         PMD_INIT_FUNC_TRACE();
 845
 846         rte_spinlock_lock(&hif->lock);
 847         hif->shm->g_client_status[0] = 0;
 848         /* Make sure all clients are disabled*/
 849         hif->shm->g_client_status[1] = 0;
 850
 851         rte_spinlock_unlock(&hif->lock);
 852
 853         if (hif->setuped) {
 854 #if defined(LS1012A_PFE_RESET_WA)
 855                 pfe_hif_rx_idle(hif);
 856 #endif
 857                 /*Disable Rx/Tx */
 858                 hif_rx_disable();
 859                 hif_tx_disable();
 860
 861                 pfe_hif_release_buffers(hif);
 862                 pfe_hif_shm_clean(hif->shm);
 863
 864                 pfe_hif_free_descr(hif);
 865                 pfe->hif.setuped = 0;
 866         }
 867         gpi_disable(HGPI_BASE_ADDR);
 868 }