drivers/crypto/bcmfs/hw/bcmfs4_rm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <unistd.h>
   7
   8 #include <rte_bitmap.h>
   9
  10 #include "bcmfs_device.h"
  11 #include "bcmfs_dev_msg.h"
  12 #include "bcmfs_hw_defs.h"
  13 #include "bcmfs_logs.h"
  14 #include "bcmfs_qp.h"
  15 #include "bcmfs_rm_common.h"
  16
  17 /* FS4 configuration */
  18 #define RING_BD_TOGGLE_INVALID(offset)                  \
  19                         (((offset) >> FS_RING_BD_ALIGN_ORDER) & 0x1)
  20 #define RING_BD_TOGGLE_VALID(offset)                    \
  21                         (!RING_BD_TOGGLE_INVALID(offset))
  22
  23 #define RING_VER_MAGIC                                  0x76303031
  24
  25 /* Per-Ring register offsets */
  26 #define RING_VER                                        0x000
  27 #define RING_BD_START_ADDR                              0x004
  28 #define RING_BD_READ_PTR                                0x008
  29 #define RING_BD_WRITE_PTR                               0x00c
  30 #define RING_BD_READ_PTR_DDR_LS                         0x010
  31 #define RING_BD_READ_PTR_DDR_MS                         0x014
  32 #define RING_CMPL_START_ADDR                            0x018
  33 #define RING_CMPL_WRITE_PTR                             0x01c
  34 #define RING_NUM_REQ_RECV_LS                            0x020
  35 #define RING_NUM_REQ_RECV_MS                            0x024
  36 #define RING_NUM_REQ_TRANS_LS                           0x028
  37 #define RING_NUM_REQ_TRANS_MS                           0x02c
  38 #define RING_NUM_REQ_OUTSTAND                           0x030
  39 #define RING_CONTROL                                    0x034
  40 #define RING_FLUSH_DONE                                 0x038
  41 #define RING_MSI_ADDR_LS                                0x03c
  42 #define RING_MSI_ADDR_MS                                0x040
  43 #define RING_MSI_CONTROL                                0x048
  44 #define RING_BD_READ_PTR_DDR_CONTROL                    0x04c
  45 #define RING_MSI_DATA_VALUE                             0x064
  46
  47 /* Register RING_BD_START_ADDR fields */
  48 #define BD_LAST_UPDATE_HW_SHIFT                         28
  49 #define BD_LAST_UPDATE_HW_MASK                          0x1
  50 #define BD_START_ADDR_VALUE(pa)                         \
  51         ((uint32_t)((((uint64_t)(pa)) >> FS_RING_BD_ALIGN_ORDER) & 0x0fffffff))
  52 #define BD_START_ADDR_DECODE(val)                       \
  53         ((uint64_t)((val) & 0x0fffffff) << FS_RING_BD_ALIGN_ORDER)
  54
  55 /* Register RING_CMPL_START_ADDR fields */
  56 #define CMPL_START_ADDR_VALUE(pa)                       \
  57         ((uint32_t)((((uint64_t)(pa)) >> FS_RING_CMPL_ALIGN_ORDER) & 0x7ffffff))
  58
  59 /* Register RING_CONTROL fields */
  60 #define CONTROL_MASK_DISABLE_CONTROL                    12
  61 #define CONTROL_FLUSH_SHIFT                             5
  62 #define CONTROL_ACTIVE_SHIFT                            4
  63 #define CONTROL_RATE_ADAPT_MASK                         0xf
  64 #define CONTROL_RATE_DYNAMIC                            0x0
  65 #define CONTROL_RATE_FAST                               0x8
  66 #define CONTROL_RATE_MEDIUM                             0x9
  67 #define CONTROL_RATE_SLOW                               0xa
  68 #define CONTROL_RATE_IDLE                               0xb
  69
  70 /* Register RING_FLUSH_DONE fields */
  71 #define FLUSH_DONE_MASK                                 0x1
  72
  73 /* Register RING_MSI_CONTROL fields */
  74 #define MSI_TIMER_VAL_SHIFT                             16
  75 #define MSI_TIMER_VAL_MASK                              0xffff
  76 #define MSI_ENABLE_SHIFT                                15
  77 #define MSI_ENABLE_MASK                                 0x1
  78 #define MSI_COUNT_SHIFT                                 0
  79 #define MSI_COUNT_MASK                                  0x3ff
  80
  81 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
  82 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT                 16
  83 #define BD_READ_PTR_DDR_TIMER_VAL_MASK                  0xffff
  84 #define BD_READ_PTR_DDR_ENABLE_SHIFT                    15
  85 #define BD_READ_PTR_DDR_ENABLE_MASK                     0x1
  86
  87 /* ====== Broadcom FS4-RM ring descriptor defines ===== */
  88
  89
  90 /* General descriptor format */
  91 #define DESC_TYPE_SHIFT                         60
  92 #define DESC_TYPE_MASK                          0xf
  93 #define DESC_PAYLOAD_SHIFT                      0
  94 #define DESC_PAYLOAD_MASK                       0x0fffffffffffffff
  95
  96 /* Null descriptor format  */
  97 #define NULL_TYPE                               0
  98 #define NULL_TOGGLE_SHIFT                       58
  99 #define NULL_TOGGLE_MASK                        0x1
 100
 101 /* Header descriptor format */
 102 #define HEADER_TYPE                             1
 103 #define HEADER_TOGGLE_SHIFT                     58
 104 #define HEADER_TOGGLE_MASK                      0x1
 105 #define HEADER_ENDPKT_SHIFT                     57
 106 #define HEADER_ENDPKT_MASK                      0x1
 107 #define HEADER_STARTPKT_SHIFT                   56
 108 #define HEADER_STARTPKT_MASK                    0x1
 109 #define HEADER_BDCOUNT_SHIFT                    36
 110 #define HEADER_BDCOUNT_MASK                     0x1f
 111 #define HEADER_BDCOUNT_MAX                      HEADER_BDCOUNT_MASK
 112 #define HEADER_FLAGS_SHIFT                      16
 113 #define HEADER_FLAGS_MASK                       0xffff
 114 #define HEADER_OPAQUE_SHIFT                     0
 115 #define HEADER_OPAQUE_MASK                      0xffff
 116
 117 /* Source (SRC) descriptor format */
 118 #define SRC_TYPE                                2
 119 #define SRC_LENGTH_SHIFT                        44
 120 #define SRC_LENGTH_MASK                         0xffff
 121 #define SRC_ADDR_SHIFT                          0
 122 #define SRC_ADDR_MASK                           0x00000fffffffffff
 123
 124 /* Destination (DST) descriptor format */
 125 #define DST_TYPE                                3
 126 #define DST_LENGTH_SHIFT                        44
 127 #define DST_LENGTH_MASK                         0xffff
 128 #define DST_ADDR_SHIFT                          0
 129 #define DST_ADDR_MASK                           0x00000fffffffffff
 130
 131 /* Next pointer (NPTR) descriptor format */
 132 #define NPTR_TYPE                               5
 133 #define NPTR_TOGGLE_SHIFT                       58
 134 #define NPTR_TOGGLE_MASK                        0x1
 135 #define NPTR_ADDR_SHIFT                         0
 136 #define NPTR_ADDR_MASK                          0x00000fffffffffff
 137
 138 /* Mega source (MSRC) descriptor format */
 139 #define MSRC_TYPE                               6
 140 #define MSRC_LENGTH_SHIFT                       44
 141 #define MSRC_LENGTH_MASK                        0xffff
 142 #define MSRC_ADDR_SHIFT                         0
 143 #define MSRC_ADDR_MASK                          0x00000fffffffffff
 144
 145 /* Mega destination (MDST) descriptor format */
 146 #define MDST_TYPE                               7
 147 #define MDST_LENGTH_SHIFT                       44
 148 #define MDST_LENGTH_MASK                        0xffff
 149 #define MDST_ADDR_SHIFT                         0
 150 #define MDST_ADDR_MASK                          0x00000fffffffffff
 151
 152 static uint8_t
 153 bcmfs4_is_next_table_desc(void *desc_ptr)
 154 {
 155         uint64_t desc = rm_read_desc(desc_ptr);
 156         uint32_t type = FS_DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
 157
 158         return (type == NPTR_TYPE) ? true : false;
 159 }
 160
 161 static uint64_t
 162 bcmfs4_next_table_desc(uint32_t toggle, uint64_t next_addr)
 163 {
 164         return (rm_build_desc(NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 165                 rm_build_desc(toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK) |
 166                 rm_build_desc(next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK));
 167 }
 168
 169 static uint64_t
 170 bcmfs4_null_desc(uint32_t toggle)
 171 {
 172         return (rm_build_desc(NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 173                 rm_build_desc(toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK));
 174 }
 175
 176 static void
 177 bcmfs4_flip_header_toggle(void *desc_ptr)
 178 {
 179         uint64_t desc = rm_read_desc(desc_ptr);
 180
 181         if (desc & ((uint64_t)0x1 << HEADER_TOGGLE_SHIFT))
 182                 desc &= ~((uint64_t)0x1 << HEADER_TOGGLE_SHIFT);
 183         else
 184                 desc |= ((uint64_t)0x1 << HEADER_TOGGLE_SHIFT);
 185
 186         rm_write_desc(desc_ptr, desc);
 187 }
 188
 189 static uint64_t
 190 bcmfs4_header_desc(uint32_t toggle, uint32_t startpkt,
 191                    uint32_t endpkt, uint32_t bdcount,
 192                    uint32_t flags, uint32_t opaque)
 193 {
 194         return (rm_build_desc(HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 195                 rm_build_desc(toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK) |
 196                 rm_build_desc(startpkt, HEADER_STARTPKT_SHIFT,
 197                               HEADER_STARTPKT_MASK) |
 198                 rm_build_desc(endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK) |
 199                 rm_build_desc(bdcount, HEADER_BDCOUNT_SHIFT,
 200                               HEADER_BDCOUNT_MASK) |
 201                 rm_build_desc(flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK) |
 202                 rm_build_desc(opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK));
 203 }
 204
 205 static void
 206 bcmfs4_enqueue_desc(uint32_t nhpos, uint32_t nhcnt,
 207                     uint32_t reqid, uint64_t desc,
 208                     void **desc_ptr, uint32_t *toggle,
 209                     void *start_desc, void *end_desc)
 210 {
 211         uint64_t d;
 212         uint32_t nhavail, _toggle, _startpkt, _endpkt, _bdcount;
 213
 214         /*
 215          * Each request or packet start with a HEADER descriptor followed
 216          * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
 217          * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
 218          * following a HEADER descriptor is represented by BDCOUNT field
 219          * of HEADER descriptor. The max value of BDCOUNT field is 31 which
 220          * means we can only have 31 non-HEADER descriptors following one
 221          * HEADER descriptor.
 222          *
 223          * In general use, number of non-HEADER descriptors can easily go
 224          * beyond 31. To tackle this situation, we have packet (or request)
 225          * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.
 226          *
 227          * To use packet extension, the first HEADER descriptor of request
 228          * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
 229          * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
 230          * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
 231          * TOGGLE bit of the first HEADER will be set to invalid state to
 232          * ensure that FlexDMA engine does not start fetching descriptors
 233          * till all descriptors are enqueued. The user of this function
 234          * will flip the TOGGLE bit of first HEADER after all descriptors
 235          * are enqueued.
 236          */
 237
 238         if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
 239                 /* Prepare the header descriptor */
 240                 nhavail = (nhcnt - nhpos);
 241                 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
 242                 _startpkt = (nhpos == 0) ? 0x1 : 0x0;
 243                 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
 244                 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
 245                                 nhavail : HEADER_BDCOUNT_MAX;
 246                 if (nhavail <= HEADER_BDCOUNT_MAX)
 247                         _bdcount = nhavail;
 248                 else
 249                         _bdcount = HEADER_BDCOUNT_MAX;
 250                 d = bcmfs4_header_desc(_toggle, _startpkt, _endpkt,
 251                                         _bdcount, 0x0, reqid);
 252
 253                 /* Write header descriptor */
 254                 rm_write_desc(*desc_ptr, d);
 255
 256                 /* Point to next descriptor */
 257                 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
 258                 if (*desc_ptr == end_desc)
 259                         *desc_ptr = start_desc;
 260
 261                 /* Skip next pointer descriptors */
 262                 while (bcmfs4_is_next_table_desc(*desc_ptr)) {
 263                         *toggle = (*toggle) ? 0 : 1;
 264                         *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
 265                         if (*desc_ptr == end_desc)
 266                                 *desc_ptr = start_desc;
 267                 }
 268         }
 269
 270         /* Write desired descriptor */
 271         rm_write_desc(*desc_ptr, desc);
 272
 273         /* Point to next descriptor */
 274         *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
 275         if (*desc_ptr == end_desc)
 276                 *desc_ptr = start_desc;
 277
 278         /* Skip next pointer descriptors */
 279         while (bcmfs4_is_next_table_desc(*desc_ptr)) {
 280                 *toggle = (*toggle) ? 0 : 1;
 281                 *desc_ptr = (uint8_t *)*desc_ptr + sizeof(desc);
 282                 if (*desc_ptr == end_desc)
 283                         *desc_ptr = start_desc;
 284         }
 285 }
 286
 287 static uint64_t
 288 bcmfs4_src_desc(uint64_t addr, unsigned int length)
 289 {
 290         return (rm_build_desc(SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 291                 rm_build_desc(length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK) |
 292                 rm_build_desc(addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK));
 293 }
 294
 295 static uint64_t
 296 bcmfs4_msrc_desc(uint64_t addr, unsigned int length_div_16)
 297 {
 298         return (rm_build_desc(MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 299                 rm_build_desc(length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK) |
 300                 rm_build_desc(addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK));
 301 }
 302
 303 static uint64_t
 304 bcmfs4_dst_desc(uint64_t addr, unsigned int length)
 305 {
 306         return (rm_build_desc(DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 307                 rm_build_desc(length, DST_LENGTH_SHIFT, DST_LENGTH_MASK) |
 308                 rm_build_desc(addr, DST_ADDR_SHIFT, DST_ADDR_MASK));
 309 }
 310
 311 static uint64_t
 312 bcmfs4_mdst_desc(uint64_t addr, unsigned int length_div_16)
 313 {
 314         return (rm_build_desc(MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK) |
 315                 rm_build_desc(length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK) |
 316                 rm_build_desc(addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK));
 317 }
 318
 319 static bool
 320 bcmfs4_sanity_check(struct bcmfs_qp_message *msg)
 321 {
 322         unsigned int i = 0;
 323
 324         if (msg == NULL)
 325                 return false;
 326
 327         for (i = 0; i <  msg->srcs_count; i++) {
 328                 if (msg->srcs_len[i] & 0xf) {
 329                         if (msg->srcs_len[i] > SRC_LENGTH_MASK)
 330                                 return false;
 331                 } else {
 332                         if (msg->srcs_len[i] > (MSRC_LENGTH_MASK * 16))
 333                                 return false;
 334                 }
 335         }
 336         for (i = 0; i <  msg->dsts_count; i++) {
 337                 if (msg->dsts_len[i] & 0xf) {
 338                         if (msg->dsts_len[i] > DST_LENGTH_MASK)
 339                                 return false;
 340                 } else {
 341                         if (msg->dsts_len[i] > (MDST_LENGTH_MASK * 16))
 342                                 return false;
 343                 }
 344         }
 345
 346         return true;
 347 }
 348
 349 static uint32_t
 350 estimate_nonheader_desc_count(struct bcmfs_qp_message *msg)
 351 {
 352         uint32_t cnt = 0;
 353         unsigned int src = 0;
 354         unsigned int dst = 0;
 355         unsigned int dst_target = 0;
 356
 357         while (src < msg->srcs_count ||
 358                dst < msg->dsts_count) {
 359                 if (src < msg->srcs_count) {
 360                         cnt++;
 361                         dst_target = msg->srcs_len[src];
 362                         src++;
 363                 } else {
 364                         dst_target = UINT_MAX;
 365                 }
 366                 while (dst_target && dst < msg->dsts_count) {
 367                         cnt++;
 368                         if (msg->dsts_len[dst] < dst_target)
 369                                 dst_target -= msg->dsts_len[dst];
 370                         else
 371                                 dst_target = 0;
 372                         dst++;
 373                 }
 374         }
 375
 376         return cnt;
 377 }
 378
 379 static void *
 380 bcmfs4_enqueue_msg(struct bcmfs_qp_message *msg,
 381                    uint32_t nhcnt, uint32_t reqid,
 382                    void *desc_ptr, uint32_t toggle,
 383                    void *start_desc, void *end_desc)
 384 {
 385         uint64_t d;
 386         uint32_t nhpos = 0;
 387         unsigned int src = 0;
 388         unsigned int dst = 0;
 389         unsigned int dst_target = 0;
 390         void *orig_desc_ptr = desc_ptr;
 391
 392         if (!desc_ptr || !start_desc || !end_desc)
 393                 return NULL;
 394
 395         if (desc_ptr < start_desc || end_desc <= desc_ptr)
 396                 return NULL;
 397
 398         while (src < msg->srcs_count || dst < msg->dsts_count) {
 399                 if (src < msg->srcs_count) {
 400                         if (msg->srcs_len[src] & 0xf) {
 401                                 d = bcmfs4_src_desc(msg->srcs_addr[src],
 402                                                     msg->srcs_len[src]);
 403                         } else {
 404                                 d = bcmfs4_msrc_desc(msg->srcs_addr[src],
 405                                                      msg->srcs_len[src] / 16);
 406                         }
 407                         bcmfs4_enqueue_desc(nhpos, nhcnt, reqid,
 408                                             d, &desc_ptr, &toggle,
 409                                             start_desc, end_desc);
 410                         nhpos++;
 411                         dst_target = msg->srcs_len[src];
 412                         src++;
 413                 } else {
 414                         dst_target = UINT_MAX;
 415                 }
 416
 417                 while (dst_target && (dst < msg->dsts_count)) {
 418                         if (msg->dsts_len[dst] & 0xf) {
 419                                 d = bcmfs4_dst_desc(msg->dsts_addr[dst],
 420                                                     msg->dsts_len[dst]);
 421                         } else {
 422                                 d = bcmfs4_mdst_desc(msg->dsts_addr[dst],
 423                                                      msg->dsts_len[dst] / 16);
 424                         }
 425                         bcmfs4_enqueue_desc(nhpos, nhcnt, reqid,
 426                                             d, &desc_ptr, &toggle,
 427                                             start_desc, end_desc);
 428                         nhpos++;
 429                         if (msg->dsts_len[dst] < dst_target)
 430                                 dst_target -= msg->dsts_len[dst];
 431                         else
 432                                 dst_target = 0;
 433                         dst++; /* for next buffer */
 434                 }
 435         }
 436
 437         /* Null descriptor with invalid toggle bit */
 438         rm_write_desc(desc_ptr, bcmfs4_null_desc(!toggle));
 439
 440         /* Ensure that descriptors have been written to memory */
 441         rte_io_wmb();
 442
 443         bcmfs4_flip_header_toggle(orig_desc_ptr);
 444
 445         return desc_ptr;
 446 }
 447
 448 static int
 449 bcmfs4_enqueue_single_request_qp(struct bcmfs_qp *qp, void *op)
 450 {
 451         int reqid;
 452         void *next;
 453         uint32_t nhcnt;
 454         int ret = 0;
 455         uint32_t pos = 0;
 456         uint64_t slab = 0;
 457         uint8_t exit_cleanup = false;
 458         struct bcmfs_queue *txq = &qp->tx_q;
 459         struct bcmfs_qp_message *msg = (struct bcmfs_qp_message *)op;
 460
 461         /* Do sanity check on message */
 462         if (!bcmfs4_sanity_check(msg)) {
 463                 BCMFS_DP_LOG(ERR, "Invalid msg on queue %d", qp->qpair_id);
 464                 return -EIO;
 465         }
 466
 467         /* Scan from the beginning */
 468         __rte_bitmap_scan_init(qp->ctx_bmp);
 469         /* Scan bitmap to get the free pool */
 470         ret = rte_bitmap_scan(qp->ctx_bmp, &pos, &slab);
 471         if (ret == 0) {
 472                 BCMFS_DP_LOG(ERR, "BD memory exhausted");
 473                 return -ERANGE;
 474         }
 475
 476         reqid = pos + __builtin_ctzll(slab);
 477         rte_bitmap_clear(qp->ctx_bmp, reqid);
 478         qp->ctx_pool[reqid] = (unsigned long)msg;
 479
 480         /*
 481          * Number required descriptors = number of non-header descriptors +
 482          *                               number of header descriptors +
 483          *                               1x null descriptor
 484          */
 485         nhcnt = estimate_nonheader_desc_count(msg);
 486
 487         /* Write descriptors to ring */
 488         next = bcmfs4_enqueue_msg(msg, nhcnt, reqid,
 489                                   (uint8_t *)txq->base_addr + txq->tx_write_ptr,
 490                                   RING_BD_TOGGLE_VALID(txq->tx_write_ptr),
 491                                   txq->base_addr,
 492                                   (uint8_t *)txq->base_addr + txq->queue_size);
 493         if (next == NULL) {
 494                 BCMFS_DP_LOG(ERR, "Enqueue for desc failed on queue %d",
 495                              qp->qpair_id);
 496                 ret = -EINVAL;
 497                 exit_cleanup = true;
 498                 goto exit;
 499         }
 500
 501         /* Save ring BD write offset */
 502         txq->tx_write_ptr = (uint32_t)((uint8_t *)next -
 503                                        (uint8_t *)txq->base_addr);
 504
 505         qp->nb_pending_requests++;
 506
 507         return 0;
 508
 509 exit:
 510         /* Cleanup if we failed */
 511         if (exit_cleanup)
 512                 rte_bitmap_set(qp->ctx_bmp, reqid);
 513
 514         return ret;
 515 }
 516
 517 static void
 518 bcmfs4_ring_doorbell_qp(struct bcmfs_qp *qp __rte_unused)
 519 {
 520         /* no door bell method supported */
 521 }
 522
 523 static uint16_t
 524 bcmfs4_dequeue_qp(struct bcmfs_qp *qp, void **ops, uint16_t budget)
 525 {
 526         int err;
 527         uint16_t reqid;
 528         uint64_t desc;
 529         uint16_t count = 0;
 530         unsigned long context = 0;
 531         struct bcmfs_queue *hwq = &qp->cmpl_q;
 532         uint32_t cmpl_read_offset, cmpl_write_offset;
 533
 534         /*
 535          * Check whether budget is valid, else set the budget to maximum
 536          * so that all the available completions will be processed.
 537          */
 538         if (budget > qp->nb_pending_requests)
 539                 budget =  qp->nb_pending_requests;
 540
 541         /*
 542          * Get current completion read and write offset
 543          * Note: We should read completion write pointer at least once
 544          * after we get a MSI interrupt because HW maintains internal
 545          * MSI status which will allow next MSI interrupt only after
 546          * completion write pointer is read.
 547          */
 548         cmpl_write_offset = FS_MMIO_READ32((uint8_t *)qp->ioreg +
 549                                            RING_CMPL_WRITE_PTR);
 550         cmpl_write_offset *= FS_RING_DESC_SIZE;
 551         cmpl_read_offset = hwq->cmpl_read_ptr;
 552
 553         /* Ensure completion pointer is read before proceeding */
 554         rte_io_rmb();
 555
 556         /* For each completed request notify mailbox clients */
 557         reqid = 0;
 558         while ((cmpl_read_offset != cmpl_write_offset) && (budget > 0)) {
 559                 /* Dequeue next completion descriptor */
 560                 desc = *((uint64_t *)((uint8_t *)hwq->base_addr +
 561                                        cmpl_read_offset));
 562
 563                 /* Next read offset */
 564                 cmpl_read_offset += FS_RING_DESC_SIZE;
 565                 if (cmpl_read_offset == FS_RING_CMPL_SIZE)
 566                         cmpl_read_offset = 0;
 567
 568                 /* Decode error from completion descriptor */
 569                 err = rm_cmpl_desc_to_error(desc);
 570                 if (err < 0)
 571                         BCMFS_DP_LOG(ERR, "error desc rcvd");
 572
 573                 /* Determine request id from completion descriptor */
 574                 reqid = rm_cmpl_desc_to_reqid(desc);
 575
 576                 /* Determine message pointer based on reqid */
 577                 context = qp->ctx_pool[reqid];
 578                 if (context == 0)
 579                         BCMFS_DP_LOG(ERR, "HW error detected");
 580
 581                 /* Release reqid for recycling */
 582                 qp->ctx_pool[reqid] = 0;
 583                 rte_bitmap_set(qp->ctx_bmp, reqid);
 584
 585                 *ops = (void *)context;
 586
 587                 /* Increment number of completions processed */
 588                 count++;
 589                 budget--;
 590                 ops++;
 591         }
 592
 593         hwq->cmpl_read_ptr = cmpl_read_offset;
 594
 595         qp->nb_pending_requests -= count;
 596
 597         return count;
 598 }
 599
 600 static int
 601 bcmfs4_start_qp(struct bcmfs_qp *qp)
 602 {
 603         int timeout;
 604         uint32_t val, off;
 605         uint64_t d, next_addr, msi;
 606         struct bcmfs_queue *tx_queue = &qp->tx_q;
 607         struct bcmfs_queue *cmpl_queue = &qp->cmpl_q;
 608
 609         /* Disable/deactivate ring */
 610         FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
 611
 612         /* Configure next table pointer entries in BD memory */
 613         for (off = 0; off < tx_queue->queue_size; off += FS_RING_DESC_SIZE) {
 614                 next_addr = off + FS_RING_DESC_SIZE;
 615                 if (next_addr == tx_queue->queue_size)
 616                         next_addr = 0;
 617                 next_addr += (uint64_t)tx_queue->base_phys_addr;
 618                 if (FS_RING_BD_ALIGN_CHECK(next_addr))
 619                         d = bcmfs4_next_table_desc(RING_BD_TOGGLE_VALID(off),
 620                                                     next_addr);
 621                 else
 622                         d = bcmfs4_null_desc(RING_BD_TOGGLE_INVALID(off));
 623                 rm_write_desc((uint8_t *)tx_queue->base_addr + off, d);
 624         }
 625
 626         /*
 627          * If user interrupt the test in between the run(Ctrl+C), then all
 628          * subsequent test run will fail because sw cmpl_read_offset and hw
 629          * cmpl_write_offset will be pointing at different completion BD. To
 630          * handle this we should flush all the rings in the startup instead
 631          * of shutdown function.
 632          * Ring flush will reset hw cmpl_write_offset.
 633          */
 634
 635         /* Set ring flush state */
 636         timeout = 1000;
 637         FS_MMIO_WRITE32(BIT(CONTROL_FLUSH_SHIFT),
 638                         (uint8_t *)qp->ioreg + RING_CONTROL);
 639         do {
 640                 /*
 641                  * If previous test is stopped in between the run, then
 642                  * sw has to read cmpl_write_offset else DME/AE will be not
 643                  * come out of flush state.
 644                  */
 645                 FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_CMPL_WRITE_PTR);
 646
 647                 if (FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
 648                                 FLUSH_DONE_MASK)
 649                         break;
 650                 usleep(1000);
 651         } while (--timeout);
 652         if (!timeout) {
 653                 BCMFS_DP_LOG(ERR, "Ring flush timeout hw-queue %d",
 654                              qp->qpair_id);
 655         }
 656
 657         /* Clear ring flush state */
 658         timeout = 1000;
 659         FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
 660         do {
 661                 if (!(FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_FLUSH_DONE) &
 662                                   FLUSH_DONE_MASK))
 663                         break;
 664                 usleep(1000);
 665         } while (--timeout);
 666         if (!timeout) {
 667                 BCMFS_DP_LOG(ERR, "Ring clear flush timeout hw-queue %d",
 668                              qp->qpair_id);
 669         }
 670
 671         /* Program BD start address */
 672         val = BD_START_ADDR_VALUE(tx_queue->base_phys_addr);
 673         FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_BD_START_ADDR);
 674
 675         /* BD write pointer will be same as HW write pointer */
 676         tx_queue->tx_write_ptr = FS_MMIO_READ32((uint8_t *)qp->ioreg +
 677                                                 RING_BD_WRITE_PTR);
 678         tx_queue->tx_write_ptr *= FS_RING_DESC_SIZE;
 679
 680
 681         for (off = 0; off < FS_RING_CMPL_SIZE; off += FS_RING_DESC_SIZE)
 682                 rm_write_desc((uint8_t *)cmpl_queue->base_addr + off, 0x0);
 683
 684         /* Program completion start address */
 685         val = CMPL_START_ADDR_VALUE(cmpl_queue->base_phys_addr);
 686         FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_CMPL_START_ADDR);
 687
 688         /* Completion read pointer will be same as HW write pointer */
 689         cmpl_queue->cmpl_read_ptr = FS_MMIO_READ32((uint8_t *)qp->ioreg +
 690                                                    RING_CMPL_WRITE_PTR);
 691         cmpl_queue->cmpl_read_ptr *= FS_RING_DESC_SIZE;
 692
 693         /* Read ring Tx, Rx, and Outstanding counts to clear */
 694         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_LS);
 695         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_RECV_MS);
 696         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_LS);
 697         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_TRANS_MS);
 698         FS_MMIO_READ32((uint8_t *)qp->ioreg + RING_NUM_REQ_OUTSTAND);
 699
 700         /* Configure per-Ring MSI registers with dummy location */
 701         /* We leave 1k * FS_RING_DESC_SIZE size from base phys for MSI */
 702         msi = cmpl_queue->base_phys_addr + (1024 * FS_RING_DESC_SIZE);
 703         FS_MMIO_WRITE32((msi & 0xFFFFFFFF),
 704                         (uint8_t *)qp->ioreg + RING_MSI_ADDR_LS);
 705         FS_MMIO_WRITE32(((msi >> 32) & 0xFFFFFFFF),
 706                         (uint8_t *)qp->ioreg + RING_MSI_ADDR_MS);
 707         FS_MMIO_WRITE32(qp->qpair_id,
 708                         (uint8_t *)qp->ioreg + RING_MSI_DATA_VALUE);
 709
 710         /* Configure RING_MSI_CONTROL */
 711         val = 0;
 712         val |= (MSI_TIMER_VAL_MASK << MSI_TIMER_VAL_SHIFT);
 713         val |= BIT(MSI_ENABLE_SHIFT);
 714         val |= (0x1 & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
 715         FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_MSI_CONTROL);
 716
 717         /* Enable/activate ring */
 718         val = BIT(CONTROL_ACTIVE_SHIFT);
 719         FS_MMIO_WRITE32(val, (uint8_t *)qp->ioreg + RING_CONTROL);
 720
 721         return 0;
 722 }
 723
 724 static void
 725 bcmfs4_shutdown_qp(struct bcmfs_qp *qp)
 726 {
 727         /* Disable/deactivate ring */
 728         FS_MMIO_WRITE32(0x0, (uint8_t *)qp->ioreg + RING_CONTROL);
 729 }
 730
 731 struct bcmfs_hw_queue_pair_ops bcmfs4_qp_ops = {
 732         .name = "fs4",
 733         .enq_one_req = bcmfs4_enqueue_single_request_qp,
 734         .ring_db = bcmfs4_ring_doorbell_qp,
 735         .dequeue = bcmfs4_dequeue_qp,
 736         .startq = bcmfs4_start_qp,
 737         .stopq = bcmfs4_shutdown_qp,
 738 };
 739
 740 RTE_INIT(bcmfs4_register_qp_ops)
 741 {
 742          bcmfs_hw_queue_pair_register_ops(&bcmfs4_qp_ops);
 743 }