drivers/crypto/bcmfs/hw/bcmfs5_rm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <unistd.h>
   7
   8 #include <rte_bitmap.h>
   9
  10 #include "bcmfs_qp.h"
  11 #include "bcmfs_logs.h"
  12 #include "bcmfs_dev_msg.h"
  13 #include "bcmfs_device.h"
  14 #include "bcmfs_hw_defs.h"
  15 #include "bcmfs_rm_common.h"
  16
  17 /* Ring version */
  18 #define RING_VER_MAGIC                                  0x76303032
  19
  20 /* Per-Ring register offsets */
  21 #define RING_VER                                        0x000
  22 #define RING_BD_START_ADDRESS_LSB                       0x004
  23 #define RING_BD_READ_PTR                                0x008
  24 #define RING_BD_WRITE_PTR                               0x00c
  25 #define RING_BD_READ_PTR_DDR_LS                         0x010
  26 #define RING_BD_READ_PTR_DDR_MS                         0x014
  27 #define RING_CMPL_START_ADDR_LSB                        0x018
  28 #define RING_CMPL_WRITE_PTR                             0x01c
  29 #define RING_NUM_REQ_RECV_LS                            0x020
  30 #define RING_NUM_REQ_RECV_MS                            0x024
  31 #define RING_NUM_REQ_TRANS_LS                           0x028
  32 #define RING_NUM_REQ_TRANS_MS                           0x02c
  33 #define RING_NUM_REQ_OUTSTAND                           0x030
  34 #define RING_CONTROL                                    0x034
  35 #define RING_FLUSH_DONE                                 0x038
  36 #define RING_MSI_ADDR_LS                                0x03c
  37 #define RING_MSI_ADDR_MS                                0x040
  38 #define RING_MSI_CONTROL                                0x048
  39 #define RING_BD_READ_PTR_DDR_CONTROL                    0x04c
  40 #define RING_MSI_DATA_VALUE                             0x064
  41 #define RING_BD_START_ADDRESS_MSB                       0x078
  42 #define RING_CMPL_START_ADDR_MSB                        0x07c
  43 #define RING_DOORBELL_BD_WRITE_COUNT                    0x074
  44
  45 /* Register RING_BD_START_ADDR fields */
  46 #define BD_LAST_UPDATE_HW_SHIFT                         28
  47 #define BD_LAST_UPDATE_HW_MASK                          0x1
  48 #define BD_START_ADDR_VALUE(pa)                         \
  49         ((uint32_t)((((uint64_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
  50 #define BD_START_ADDR_DECODE(val)                       \
  51         ((uint64_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
  52
  53 /* Register RING_CMPL_START_ADDR fields */
  54 #define CMPL_START_ADDR_VALUE(pa)                       \
  55         ((uint32_t)((((uint64_t)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
  56
  57 /* Register RING_CONTROL fields */
  58 #define CONTROL_MASK_DISABLE_CONTROL                    12
  59 #define CONTROL_FLUSH_SHIFT                             5
  60 #define CONTROL_ACTIVE_SHIFT                            4
  61 #define CONTROL_RATE_ADAPT_MASK                         0xf
  62 #define CONTROL_RATE_DYNAMIC                            0x0
  63 #define CONTROL_RATE_FAST                               0x8
  64 #define CONTROL_RATE_MEDIUM                             0x9
  65 #define CONTROL_RATE_SLOW                               0xa
  66 #define CONTROL_RATE_IDLE                               0xb
  67
  68 /* Register RING_FLUSH_DONE fields */
  69 #define FLUSH_DONE_MASK                                 0x1
  70
  71 /* Register RING_MSI_CONTROL fields */
  72 #define MSI_TIMER_VAL_SHIFT                             16
  73 #define MSI_TIMER_VAL_MASK                              0xffff
  74 #define MSI_ENABLE_SHIFT                                15
  75 #define MSI_ENABLE_MASK                                 0x1
  76 #define MSI_COUNT_SHIFT                                 0
  77 #define MSI_COUNT_MASK                                  0x3ff
  78
  79 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
  80 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT                 16
  81 #define BD_READ_PTR_DDR_TIMER_VAL_MASK                  0xffff
  82 #define BD_READ_PTR_DDR_ENABLE_SHIFT                    15
  83 #define BD_READ_PTR_DDR_ENABLE_MASK                     0x1
  84
  85 /* General descriptor format */
  86 #define DESC_TYPE_SHIFT                                 60
  87 #define DESC_TYPE_MASK                                  0xf
  88 #define DESC_PAYLOAD_SHIFT                              0
  89 #define DESC_PAYLOAD_MASK                               0x0fffffffffffffff
  90
  91 /* Null descriptor format  */
  92 #define NULL_TYPE                                       0
  93 #define NULL_TOGGLE_SHIFT                               59
  94 #define NULL_TOGGLE_MASK                                0x1
  95
  96 /* Header descriptor format */
  97 #define HEADER_TYPE                                     1
  98 #define HEADER_TOGGLE_SHIFT                             59
  99 #define HEADER_TOGGLE_MASK                              0x1
 100 #define HEADER_ENDPKT_SHIFT                             57
 101 #define HEADER_ENDPKT_MASK                              0x1
 102 #define HEADER_STARTPKT_SHIFT                           56
 103 #define HEADER_STARTPKT_MASK                            0x1
 104 #define HEADER_BDCOUNT_SHIFT                            36
 105 #define HEADER_BDCOUNT_MASK                             0x1f
 106 #define HEADER_BDCOUNT_MAX                              HEADER_BDCOUNT_MASK
 107 #define HEADER_FLAGS_SHIFT                              16
 108 #define HEADER_FLAGS_MASK                               0xffff
 109 #define HEADER_OPAQUE_SHIFT                             0
 110 #define HEADER_OPAQUE_MASK                              0xffff
 111
 112 /* Source (SRC) descriptor format */
 113
 114 #define SRC_TYPE                                        2
 115 #define SRC_LENGTH_SHIFT                                44
 116 #define SRC_LENGTH_MASK                                 0xffff
 117 #define SRC_ADDR_SHIFT                                  0
 118 #define SRC_ADDR_MASK                                   0x00000fffffffffff
 119
 120 /* Destination (DST) descriptor format */
 121 #define DST_TYPE                                        3
 122 #define DST_LENGTH_SHIFT                                44
 123 #define DST_LENGTH_MASK                                 0xffff
 124 #define DST_ADDR_SHIFT                                  0
 125 #define DST_ADDR_MASK                                   0x00000fffffffffff
 126
 127 /* Next pointer (NPTR) descriptor format */
 128 #define NPTR_TYPE                                       5
 129 #define NPTR_TOGGLE_SHIFT                               59
 130 #define NPTR_TOGGLE_MASK                                0x1
 131 #define NPTR_ADDR_SHIFT                                 0
 132 #define NPTR_ADDR_MASK                                  0x00000fffffffffff
 133
 134 /* Mega source (MSRC) descriptor format */
 135 #define MSRC_TYPE                                       6
 136 #define MSRC_LENGTH_SHIFT                               44
 137 #define MSRC_LENGTH_MASK                                0xffff
 138 #define MSRC_ADDR_SHIFT                                 0
 139 #define MSRC_ADDR_MASK                                  0x00000fffffffffff
 140
 141 /* Mega destination (MDST) descriptor format */
 142 #define MDST_TYPE                                       7
 143 #define MDST_LENGTH_SHIFT                               44
 144 #define MDST_LENGTH_MASK                                0xffff
 145 #define MDST_ADDR_SHIFT                                 0
 146 #define MDST_ADDR_MASK                                  0x00000fffffffffff
 147
 148 static uint8_t
 149 bcmfs5_is_next_table_desc(void *desc_ptr)
 150 {
 151         uint64_t desc = rm_read_desc(desc_ptr);
 152         uint32_t type = FS_DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 153
 154         return (type == NPTR_TYPE) ? true : false;
 155 }
 156
 157 static uint64_t
 158 bcmfs5_next_table_desc(uint64_t next_addr)
 159 {
 160         return (rm_build_desc(NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 161                 rm_build_desc(next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK));
 162 }
 163
 164 static uint64_t
 165 bcmfs5_null_desc(void)
 166 {
 167         return rm_build_desc(NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 168 }
 169
 170 static uint64_t
 171 bcmfs5_header_desc(uint32_t startpkt, uint32_t endpkt,
 172                                        uint32_t bdcount, uint32_t flags,
 173                                        uint32_t opaque)
 174 {
 175         return (rm_build_desc(HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 176                 rm_build_desc(startpkt, HEADER_STARTPKT_SHIFT,
 177                               HEADER_STARTPKT_MASK) |
 178                 rm_build_desc(endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK) |
 179                 rm_build_desc(bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK) |
 180                 rm_build_desc(flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK) |
 181                 rm_build_desc(opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK));
 182 }
 183
 184 static int
 185 bcmfs5_enqueue_desc(uint32_t nhpos, uint32_t nhcnt,
 186                     uint32_t reqid, uint64_t desc,
 187                     void **desc_ptr, void *start_desc,
 188                     void *end_desc)
 189 {
 190         uint64_t d;
 191         uint32_t nhavail, _startpkt, _endpkt, _bdcount;
 192         int is_nxt_page = 0;
 193
 194         /*
 195          * Each request or packet start with a HEADER descriptor followed
 196          * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
 197          * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
 198          * following a HEADER descriptor is represented by BDCOUNT field
 199          * of HEADER descriptor. The max value of BDCOUNT field is 31 which
 200          * means we can only have 31 non-HEADER descriptors following one
 201          * HEADER descriptor.
 202          *
 203          * In general use, number of non-HEADER descriptors can easily go
 204          * beyond 31. To tackle this situation, we have packet (or request)
 205          * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.
 206          *
 207          * To use packet extension, the first HEADER descriptor of request
 208          * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
 209          * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
 210          * HEADER descriptor will have STARTPKT=0 and ENDPKT=1.
 211          */
 212
 213         if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
 214                 /* Prepare the header descriptor */
 215                 nhavail = (nhcnt - nhpos);
 216                 _startpkt = (nhpos == 0) ? 0x1 : 0x0;
 217                 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
 218                 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
 219                                 nhavail : HEADER_BDCOUNT_MAX;
 220                 if (nhavail <= HEADER_BDCOUNT_MAX)
 221                         _bdcount = nhavail;
 222                 else
 223                         _bdcount = HEADER_BDCOUNT_MAX;
 224                 d = bcmfs5_header_desc(_startpkt, _endpkt,
 225                                        _bdcount, 0x0, reqid);
 226
 227                 /* Write header descriptor */
 228                 rm_write_desc(*desc_ptr, d);
 229
 230                 /* Point to next descriptor */
 231                 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
 232                 if (*desc_ptr == end_desc)
 233                         *desc_ptr = start_desc;
 234
 235                 /* Skip next pointer descriptors */
 236                 while (bcmfs5_is_next_table_desc(*desc_ptr)) {
 237                         is_nxt_page = 1;
 238                         *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
 239                         if (*desc_ptr == end_desc)
 240                                 *desc_ptr = start_desc;
 241                 }
 242         }
 243
 244         /* Write desired descriptor */
 245         rm_write_desc(*desc_ptr, desc);
 246
 247         /* Point to next descriptor */
 248         *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
 249         if (*desc_ptr == end_desc)
 250                 *desc_ptr = start_desc;
 251
 252         /* Skip next pointer descriptors */
 253         while (bcmfs5_is_next_table_desc(*desc_ptr)) {
 254                 is_nxt_page = 1;
 255                 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
 256                 if (*desc_ptr == end_desc)
 257                         *desc_ptr = start_desc;
 258         }
 259
 260         return is_nxt_page;
 261 }
 262
 263 static uint64_t
 264 bcmfs5_src_desc(uint64_t addr, unsigned int len)
 265 {
 266         return (rm_build_desc(SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 267                 rm_build_desc(len, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK) |
 268                 rm_build_desc(addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK));
 269 }
 270
 271 static uint64_t
 272 bcmfs5_msrc_desc(uint64_t addr, unsigned int len_div_16)
 273 {
 274         return (rm_build_desc(MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 275                 rm_build_desc(len_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK) |
 276                 rm_build_desc(addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK));
 277 }
 278
 279 static uint64_t
 280 bcmfs5_dst_desc(uint64_t addr, unsigned int len)
 281 {
 282         return (rm_build_desc(DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 283                 rm_build_desc(len, DST_LENGTH_SHIFT, DST_LENGTH_MASK) |
 284                 rm_build_desc(addr, DST_ADDR_SHIFT, DST_ADDR_MASK));
 285 }
 286
 287 static uint64_t
 288 bcmfs5_mdst_desc(uint64_t addr, unsigned int len_div_16)
 289 {
 290         return (rm_build_desc(MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 291                 rm_build_desc(len_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK) |
 292                 rm_build_desc(addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK));
 293 }
 294
 295 static bool
 296 bcmfs5_sanity_check(struct bcmfs_qp_message *msg)
 297 {
 298         unsigned int i = 0;
 299
 300         if (msg == NULL)
 301                 return false;
 302
 303         for (i = 0; i <  msg->srcs_count; i++) {
 304                 if (msg->srcs_len[i] & 0xf) {
 305                         if (msg->srcs_len[i] > SRC_LENGTH_MASK)
 306                                 return false;
 307                 } else {
 308                         if (msg->srcs_len[i] > (MSRC_LENGTH_MASK * 16))
 309                                 return false;
 310                 }
 311         }
 312         for (i = 0; i <  msg->dsts_count; i++) {
 313                 if (msg->dsts_len[i] & 0xf) {
 314                         if (msg->dsts_len[i] > DST_LENGTH_MASK)
 315                                 return false;
 316                 } else {
 317                         if (msg->dsts_len[i] > (MDST_LENGTH_MASK * 16))
 318                                 return false;
 319                 }
 320         }
 321
 322         return true;
 323 }
 324
 325 static void *
 326 bcmfs5_enqueue_msg(struct bcmfs_queue *txq,
 327                    struct bcmfs_qp_message *msg,
 328                    uint32_t reqid, void *desc_ptr,
 329                    void *start_desc, void *end_desc)
 330 {
 331         uint64_t d;
 332         unsigned int src, dst;
 333         uint32_t nhpos = 0;
 334         int nxt_page = 0;
 335         uint32_t nhcnt = msg->srcs_count + msg->dsts_count;
 336
 337         if (desc_ptr == NULL || start_desc == NULL || end_desc == NULL)
 338                 return NULL;
 339
 340         if (desc_ptr < start_desc || end_desc <= desc_ptr)
 341                 return NULL;
 342
 343         for (src = 0; src < msg->srcs_count; src++) {
 344                 if (msg->srcs_len[src] & 0xf)
 345                         d = bcmfs5_src_desc(msg->srcs_addr[src],
 346                                             msg->srcs_len[src]);
 347                 else
 348                         d = bcmfs5_msrc_desc(msg->srcs_addr[src],
 349                                              msg->srcs_len[src] / 16);
 350
 351                 nxt_page = bcmfs5_enqueue_desc(nhpos, nhcnt, reqid,
 352                                                d, &desc_ptr, start_desc,
 353                                                end_desc);
 354                 if (nxt_page)
 355                         txq->descs_inflight++;
 356                 nhpos++;
 357         }
 358
 359         for (dst = 0; dst < msg->dsts_count; dst++) {
 360                 if (msg->dsts_len[dst] & 0xf)
 361                         d = bcmfs5_dst_desc(msg->dsts_addr[dst],
 362                                             msg->dsts_len[dst]);
 363                 else
 364                         d = bcmfs5_mdst_desc(msg->dsts_addr[dst],
 365                                              msg->dsts_len[dst] / 16);
 366
 367                 nxt_page = bcmfs5_enqueue_desc(nhpos, nhcnt, reqid,
 368                                                d, &desc_ptr, start_desc,
 369                                                end_desc);
 370                 if (nxt_page)
 371                         txq->descs_inflight++;
 372                 nhpos++;
 373         }
 374
 375         txq->descs_inflight += nhcnt + 1;
 376
 377         return desc_ptr;
 378 }
 379
 380 static int
 381 bcmfs5_enqueue_single_request_qp(struct bcmfs_qp *qp, void *op)
 382 {
 383         void *next;
 384         int reqid;
 385         int ret = 0;
 386         uint64_t slab = 0;
 387         uint32_t pos = 0;
 388         uint8_t exit_cleanup = false;
 389         struct bcmfs_queue *txq = &qp->tx_q;
 390         struct bcmfs_qp_message *msg = (struct bcmfs_qp_message *)op;
 391
 392         /* Do sanity check on message */
 393         if (!bcmfs5_sanity_check(msg)) {
 394                 BCMFS_DP_LOG(ERR, "Invalid msg on queue %d", qp->qpair_id);
 395                 return -EIO;
 396         }
 397
 398         /* Scan from the beginning */
 399         __rte_bitmap_scan_init(qp->ctx_bmp);
 400         /* Scan bitmap to get the free pool */
 401         ret = rte_bitmap_scan(qp->ctx_bmp, &pos, &slab);
 402         if (ret == 0) {
 403                 BCMFS_DP_LOG(ERR, "BD memory exhausted");
 404                 return -ERANGE;
 405         }
 406
 407         reqid = pos + __builtin_ctzll(slab);
 408         rte_bitmap_clear(qp->ctx_bmp, reqid);
 409         qp->ctx_pool[reqid] = (unsigned long)msg;
 410
 411         /* Write descriptors to ring */
 412         next = bcmfs5_enqueue_msg(txq, msg, reqid,
 413                                   (uint8_t *)txq->base_addr + txq->tx_write_ptr,
 414                                   txq->base_addr,
 415                                   (uint8_t *)txq->base_addr + txq->queue_size);
 416         if (next == NULL) {
 417                 BCMFS_DP_LOG(ERR, "Enqueue for desc failed on queue %d",
 418                              qp->qpair_id);
 419                 ret = -EINVAL;
 420                 exit_cleanup = true;
 421                 goto exit;
 422         }
 423
 424         /* Save ring BD write offset */
 425         txq->tx_write_ptr = (uint32_t)((uint8_t *)next -
 426                                        (uint8_t *)txq->base_addr);
 427
 428         qp->nb_pending_requests++;
 429
 430         return 0;
 431
 432 exit:
 433         /* Cleanup if we failed */
 434         if (exit_cleanup)
 435                 rte_bitmap_set(qp->ctx_bmp, reqid);
 436
 437         return ret;
 438 }
 439
 440 static void bcmfs5_write_doorbell(struct bcmfs_qp *qp)
 441 {
 442         struct bcmfs_queue *txq = &qp->tx_q;
 443
 444         /* sync in bfeore ringing the door-bell */
 445         rte_wmb();
 446
 447         FS_MMIO_WRITE32(txq->descs_inflight,
 448                         (uint8_t *)qp->ioreg + RING_DOORBELL_BD_WRITE_COUNT);
 449
 450         /* reset the count */
 451         txq->descs_inflight = 0;
 452 }
 453
 454 static uint16_t
 455 bcmfs5_dequeue_qp(struct bcmfs_qp *qp, void **ops, uint16_t budget)
 456 {
 457         int err;
 458         uint16_t reqid;
 459         uint64_t desc;
 460         uint16_t count = 0;
 461         unsigned long context = 0;
 462         struct bcmfs_queue *hwq = &qp->cmpl_q;
 463         uint32_t cmpl_read_offset, cmpl_write_offset;
 464
 465         /*
 466          * Check whether budget is valid, else set the budget to maximum
 467          * so that all the available completions will be processed.
 468          */
 469         if (budget > qp->nb_pending_requests)
 470                 budget =  qp->nb_pending_requests;
 471
 472         /*
 473          * Get current completion read and write offset
 474          *
 475          * Note: We should read completion write pointer at least once
 476          * after we get a MSI interrupt because HW maintains internal
 477          * MSI status which will allow next MSI interrupt only after
 478          * completion write pointer is read.
 479          */
 480         cmpl_write_offset = FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_CMPL_WRITE_PTR);
 481         cmpl_write_offset *= FS_RING_DESC_SIZE;
 482         cmpl_read_offset = hwq->cmpl_read_ptr;
 483
 484         /* read the ring cmpl write ptr before cmpl read offset */
 485         rte_io_rmb();
 486
 487         /* For each completed request notify mailbox clients */
 488         reqid = 0;
 489         while ((cmpl_read_offset != cmpl_write_offset) && (budget > 0)) {
 490                 /* Dequeue next completion descriptor */
 491                 desc = *((uint64_t *)((uint8_t *)hwq->base_addr +
 492                                       cmpl_read_offset));
 493
 494                 /* Next read offset */
 495                 cmpl_read_offset += FS_RING_DESC_SIZE;
 496                 if (cmpl_read_offset == FS_RING_CMPL_SIZE)
 497                         cmpl_read_offset = 0;
 498
 499                 /* Decode error from completion descriptor */
 500                 err = rm_cmpl_desc_to_error(desc);
 501                 if (err < 0)
 502                         BCMFS_DP_LOG(ERR, "error desc rcvd");
 503
 504                 /* Determine request id from completion descriptor */
 505                 reqid = rm_cmpl_desc_to_reqid(desc);
 506
 507                 /* Retrieve context */
 508                 context = qp->ctx_pool[reqid];
 509                 if (context == 0)
 510                         BCMFS_DP_LOG(ERR, "HW error detected");
 511
 512                 /* Release reqid for recycling */
 513                 qp->ctx_pool[reqid] = 0;
 514                 rte_bitmap_set(qp->ctx_bmp, reqid);
 515
 516                 *ops = (void *)context;
 517
 518                 /* Increment number of completions processed */
 519                 count++;
 520                 budget--;
 521                 ops++;
 522         }
 523
 524         hwq->cmpl_read_ptr = cmpl_read_offset;
 525
 526         qp->nb_pending_requests -= count;
 527
 528         return count;
 529 }
 530
 531 static int
 532 bcmfs5_start_qp(struct bcmfs_qp *qp)
 533 {
 534         uint32_t val, off;
 535         uint64_t d, next_addr, msi;
 536         int timeout;
 537         uint32_t bd_high, bd_low, cmpl_high, cmpl_low;
 538         struct bcmfs_queue *tx_queue = &qp->tx_q;
 539         struct bcmfs_queue *cmpl_queue = &qp->cmpl_q;
 540
 541         /* Disable/deactivate ring */
 542         FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
 543
 544         /* Configure next table pointer entries in BD memory */
 545         for (off = 0; off < tx_queue->queue_size; off += FS_RING_DESC_SIZE) {
 546                 next_addr = off + FS_RING_DESC_SIZE;
 547                 if (next_addr == tx_queue->queue_size)
 548                         next_addr = 0;
 549                 next_addr += (uint64_t)tx_queue->base_phys_addr;
 550                 if (FS_RING_BD_ALIGN_CHECK(next_addr))
 551                         d = bcmfs5_next_table_desc(next_addr);
 552                 else
 553                         d = bcmfs5_null_desc();
 554                 rm_write_desc((uint8_t *)tx_queue->base_addr + off, d);
 555         }
 556
 557         /*
 558          * If user interrupt the test in between the run(Ctrl+C), then all
 559          * subsequent test run will fail because sw cmpl_read_offset and hw
 560          * cmpl_write_offset will be pointing at different completion BD. To
 561          * handle this we should flush all the rings in the startup instead
 562          * of shutdown function.
 563          * Ring flush will reset hw cmpl_write_offset.
 564          */
 565
 566         /* Set ring flush state */
 567         timeout = 1000;
 568         FS_MMIO_WRITE32(BIT(CONTROL_FLUSH_SHIFT),
 569                         (uint8_t *)qp->ioreg + RING_CONTROL);
 570         do {
 571                 /*
 572                  * If previous test is stopped in between the run, then
 573                  * sw has to read cmpl_write_offset else DME/AE will be not
 574                  * come out of flush state.
 575                  */
 576                 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_CMPL_WRITE_PTR);
 577
 578                 if (FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
 579                                    FLUSH_DONE_MASK)
 580                         break;
 581                 usleep(1000);
 582         } while (--timeout);
 583         if (!timeout) {
 584                 BCMFS_DP_LOG(ERR, "Ring flush timeout hw-queue %d",
 585                              qp->qpair_id);
 586         }
 587
 588         /* Clear ring flush state */
 589         timeout = 1000;
 590         FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
 591         do {
 592                 if (!(FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
 593                                      FLUSH_DONE_MASK))
 594                         break;
 595                 usleep(1000);
 596         } while (--timeout);
 597         if (!timeout) {
 598                 BCMFS_DP_LOG(ERR, "Ring clear flush timeout hw-queue %d",
 599                              qp->qpair_id);
 600         }
 601
 602         /* Program BD start address */
 603         bd_low = lower_32_bits(tx_queue->base_phys_addr);
 604         bd_high = upper_32_bits(tx_queue->base_phys_addr);
 605         FS_MMIO_WRITE32(bd_low, (uint8_t *)qp->ioreg +
 606                                 RING_BD_START_ADDRESS_LSB);
 607         FS_MMIO_WRITE32(bd_high, (uint8_t *)qp->ioreg +
 608                                  RING_BD_START_ADDRESS_MSB);
 609
 610         tx_queue->tx_write_ptr = 0;
 611
 612         for (off = 0; off < FS_RING_CMPL_SIZE; off += FS_RING_DESC_SIZE)
 613                 rm_write_desc((uint8_t *)cmpl_queue->base_addr + off, 0x0);
 614
 615         /* Completion read pointer will be same as HW write pointer */
 616         cmpl_queue->cmpl_read_ptr = FS_MMIO_READ32((uint8_t *)qp->ioreg +
 617                                                    RING_CMPL_WRITE_PTR);
 618         /* Program completion start address */
 619         cmpl_low = lower_32_bits(cmpl_queue->base_phys_addr);
 620         cmpl_high = upper_32_bits(cmpl_queue->base_phys_addr);
 621         FS_MMIO_WRITE32(cmpl_low, (uint8_t *)qp->ioreg +
 622                                 RING_CMPL_START_ADDR_LSB);
 623         FS_MMIO_WRITE32(cmpl_high, (uint8_t *)qp->ioreg +
 624                                 RING_CMPL_START_ADDR_MSB);
 625
 626         cmpl_queue->cmpl_read_ptr *= FS_RING_DESC_SIZE;
 627
 628         /* Read ring Tx, Rx, and Outstanding counts to clear */
 629         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_LS);
 630         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_MS);
 631         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_LS);
 632         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_MS);
 633         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_OUTSTAND);
 634
 635         /* Configure per-Ring MSI registers with dummy location */
 636         msi = cmpl_queue->base_phys_addr + (1024 * FS_RING_DESC_SIZE);
 637         FS_MMIO_WRITE32((msi & 0xFFFFFFFF),
 638                         (uint8_t *)qp->ioreg + RING_MSI_ADDR_LS);
 639         FS_MMIO_WRITE32(((msi >> 32) & 0xFFFFFFFF),
 640                         (uint8_t *)qp->ioreg + RING_MSI_ADDR_MS);
 641         FS_MMIO_WRITE32(qp->qpair_id, (uint8_t *)qp->ioreg +
 642                                       RING_MSI_DATA_VALUE);
 643
 644         /* Configure RING_MSI_CONTROL */
 645         val = 0;
 646         val |= (MSI_TIMER_VAL_MASK << MSI_TIMER_VAL_SHIFT);
 647         val |= BIT(MSI_ENABLE_SHIFT);
 648         val |= (0x1 & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
 649         FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_MSI_CONTROL);
 650
 651         /* Enable/activate ring */
 652         val = BIT(CONTROL_ACTIVE_SHIFT);
 653         FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_CONTROL);
 654
 655         return 0;
 656 }
 657
 658 static void
 659 bcmfs5_shutdown_qp(struct bcmfs_qp *qp)
 660 {
 661         /* Disable/deactivate ring */
 662         FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
 663 }
 664
 665 struct bcmfs_hw_queue_pair_ops bcmfs5_qp_ops = {
 666         .name = "fs5",
 667         .enq_one_req = bcmfs5_enqueue_single_request_qp,
 668         .ring_db = bcmfs5_write_doorbell,
 669         .dequeue = bcmfs5_dequeue_qp,
 670         .startq = bcmfs5_start_qp,
 671         .stopq = bcmfs5_shutdown_qp,
 672 };
 673
 674 RTE_INIT(bcmfs5_register_qp_ops)
 675 {
 676         bcmfs_hw_queue_pair_register_ops(&bcmfs5_qp_ops);
 677 }