drivers/net/hinic/base/hinic_pmd_cmdq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include "hinic_compat.h"
   6 #include "hinic_pmd_hwdev.h"
   7 #include "hinic_pmd_hwif.h"
   8 #include "hinic_pmd_wq.h"
   9 #include "hinic_pmd_mgmt.h"
  10 #include "hinic_pmd_cmdq.h"
  11
  12 #define CMDQ_CMD_TIMEOUT                                5000 /* millisecond */
  13
  14 #define UPPER_8_BITS(data)                              (((data) >> 8) & 0xFF)
  15 #define LOWER_8_BITS(data)                              ((data) & 0xFF)
  16
  17 #define CMDQ_DB_INFO_HI_PROD_IDX_SHIFT                  0
  18 #define CMDQ_DB_INFO_QUEUE_TYPE_SHIFT                   23
  19 #define CMDQ_DB_INFO_CMDQ_TYPE_SHIFT                    24
  20 #define CMDQ_DB_INFO_SRC_TYPE_SHIFT                     27
  21
  22 #define CMDQ_DB_INFO_HI_PROD_IDX_MASK                   0xFFU
  23 #define CMDQ_DB_INFO_QUEUE_TYPE_MASK                    0x1U
  24 #define CMDQ_DB_INFO_CMDQ_TYPE_MASK                     0x7U
  25 #define CMDQ_DB_INFO_SRC_TYPE_MASK                      0x1FU
  26
  27 #define CMDQ_DB_INFO_SET(val, member)           \
  28         (((val) & CMDQ_DB_INFO_##member##_MASK) <<      \
  29                 CMDQ_DB_INFO_##member##_SHIFT)
  30
  31 #define CMDQ_CTRL_PI_SHIFT                              0
  32 #define CMDQ_CTRL_CMD_SHIFT                             16
  33 #define CMDQ_CTRL_MOD_SHIFT                             24
  34 #define CMDQ_CTRL_ACK_TYPE_SHIFT                        29
  35 #define CMDQ_CTRL_HW_BUSY_BIT_SHIFT                     31
  36
  37 #define CMDQ_CTRL_PI_MASK                               0xFFFFU
  38 #define CMDQ_CTRL_CMD_MASK                              0xFFU
  39 #define CMDQ_CTRL_MOD_MASK                              0x1FU
  40 #define CMDQ_CTRL_ACK_TYPE_MASK                         0x3U
  41 #define CMDQ_CTRL_HW_BUSY_BIT_MASK                      0x1U
  42
  43 #define CMDQ_CTRL_SET(val, member)              \
  44         (((val) & CMDQ_CTRL_##member##_MASK) << CMDQ_CTRL_##member##_SHIFT)
  45
  46 #define CMDQ_CTRL_GET(val, member)              \
  47         (((val) >> CMDQ_CTRL_##member##_SHIFT) & CMDQ_CTRL_##member##_MASK)
  48
  49 #define CMDQ_WQE_HEADER_BUFDESC_LEN_SHIFT               0
  50 #define CMDQ_WQE_HEADER_COMPLETE_FMT_SHIFT              15
  51 #define CMDQ_WQE_HEADER_DATA_FMT_SHIFT                  22
  52 #define CMDQ_WQE_HEADER_COMPLETE_REQ_SHIFT              23
  53 #define CMDQ_WQE_HEADER_COMPLETE_SECT_LEN_SHIFT         27
  54 #define CMDQ_WQE_HEADER_CTRL_LEN_SHIFT                  29
  55 #define CMDQ_WQE_HEADER_HW_BUSY_BIT_SHIFT               31
  56
  57 #define CMDQ_WQE_HEADER_BUFDESC_LEN_MASK                0xFFU
  58 #define CMDQ_WQE_HEADER_COMPLETE_FMT_MASK               0x1U
  59 #define CMDQ_WQE_HEADER_DATA_FMT_MASK                   0x1U
  60 #define CMDQ_WQE_HEADER_COMPLETE_REQ_MASK               0x1U
  61 #define CMDQ_WQE_HEADER_COMPLETE_SECT_LEN_MASK          0x3U
  62 #define CMDQ_WQE_HEADER_CTRL_LEN_MASK                   0x3U
  63 #define CMDQ_WQE_HEADER_HW_BUSY_BIT_MASK                0x1U
  64
  65 #define CMDQ_WQE_HEADER_SET(val, member)        \
  66         (((val) & CMDQ_WQE_HEADER_##member##_MASK) <<   \
  67                 CMDQ_WQE_HEADER_##member##_SHIFT)
  68
  69 #define CMDQ_WQE_HEADER_GET(val, member)        \
  70         (((val) >> CMDQ_WQE_HEADER_##member##_SHIFT) &  \
  71                 CMDQ_WQE_HEADER_##member##_MASK)
  72
  73 #define CMDQ_CTXT_CURR_WQE_PAGE_PFN_SHIFT               0
  74 #define CMDQ_CTXT_EQ_ID_SHIFT                           56
  75 #define CMDQ_CTXT_CEQ_ARM_SHIFT                         61
  76 #define CMDQ_CTXT_CEQ_EN_SHIFT                          62
  77 #define CMDQ_CTXT_HW_BUSY_BIT_SHIFT                     63
  78
  79 #define CMDQ_CTXT_CURR_WQE_PAGE_PFN_MASK                0xFFFFFFFFFFFFF
  80 #define CMDQ_CTXT_EQ_ID_MASK                            0x1F
  81 #define CMDQ_CTXT_CEQ_ARM_MASK                          0x1
  82 #define CMDQ_CTXT_CEQ_EN_MASK                           0x1
  83 #define CMDQ_CTXT_HW_BUSY_BIT_MASK                      0x1
  84
  85 #define CMDQ_CTXT_PAGE_INFO_SET(val, member)            \
  86         (((u64)(val) & CMDQ_CTXT_##member##_MASK) << CMDQ_CTXT_##member##_SHIFT)
  87
  88 #define CMDQ_CTXT_PAGE_INFO_CLEAR(val, member)          \
  89         ((val) & (~((u64)CMDQ_CTXT_##member##_MASK <<   \
  90                 CMDQ_CTXT_##member##_SHIFT)))
  91
  92 #define CMDQ_CTXT_WQ_BLOCK_PFN_SHIFT                    0
  93 #define CMDQ_CTXT_CI_SHIFT                              52
  94
  95 #define CMDQ_CTXT_WQ_BLOCK_PFN_MASK                     0xFFFFFFFFFFFFF
  96 #define CMDQ_CTXT_CI_MASK                               0xFFF
  97
  98 #define CMDQ_CTXT_BLOCK_INFO_SET(val, member)           \
  99         (((u64)(val) & CMDQ_CTXT_##member##_MASK) << CMDQ_CTXT_##member##_SHIFT)
 100
 101 #define SAVED_DATA_ARM_SHIFT                    31
 102
 103 #define SAVED_DATA_ARM_MASK                     0x1U
 104
 105 #define SAVED_DATA_SET(val, member)             \
 106         (((val) & SAVED_DATA_##member##_MASK) << SAVED_DATA_##member##_SHIFT)
 107
 108 #define SAVED_DATA_CLEAR(val, member)           \
 109         ((val) & (~(SAVED_DATA_##member##_MASK << SAVED_DATA_##member##_SHIFT)))
 110
 111 #define WQE_ERRCODE_VAL_SHIFT                   20
 112
 113 #define WQE_ERRCODE_VAL_MASK                    0xF
 114
 115 #define WQE_ERRCODE_GET(val, member)            \
 116         (((val) >> WQE_ERRCODE_##member##_SHIFT) & WQE_ERRCODE_##member##_MASK)
 117
 118 #define WQE_COMPLETED(ctrl_info)        CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT)
 119
 120 #define WQE_HEADER(wqe)         ((struct hinic_cmdq_header *)(wqe))
 121
 122 #define CMDQ_DB_PI_OFF(pi)              (((u16)LOWER_8_BITS(pi)) << 3)
 123
 124 #define CMDQ_DB_ADDR(db_base, pi)       \
 125         (((u8 *)(db_base) + HINIC_DB_OFF) + CMDQ_DB_PI_OFF(pi))
 126
 127 #define CMDQ_PFN(addr, page_size)       ((addr) >> (ilog2(page_size)))
 128
 129 #define FIRST_DATA_TO_WRITE_LAST        sizeof(u64)
 130
 131 #define WQE_LCMD_SIZE           64
 132 #define WQE_SCMD_SIZE           64
 133
 134 #define COMPLETE_LEN            3
 135
 136 #define CMDQ_WQEBB_SIZE         64
 137 #define CMDQ_WQEBB_SHIFT        6
 138
 139 #define CMDQ_WQE_SIZE           64
 140
 141 #define HINIC_CMDQ_WQ_BUF_SIZE  4096
 142
 143 #define WQE_NUM_WQEBBS(wqe_size, wq)    \
 144         ((u16)(ALIGN((u32)(wqe_size), (wq)->wqebb_size) / (wq)->wqebb_size))
 145
 146 #define cmdq_to_cmdqs(cmdq)     container_of((cmdq) - (cmdq)->cmdq_type, \
 147                                 struct hinic_cmdqs, cmdq[0])
 148
 149 #define WAIT_CMDQ_ENABLE_TIMEOUT        300
 150
 151
 152 static void cmdq_init_queue_ctxt(struct hinic_cmdq *cmdq,
 153                                  struct hinic_cmdq_ctxt *cmdq_ctxt);
 154 static void hinic_cmdqs_free(struct hinic_hwdev *hwdev);
 155
 156 bool hinic_cmdq_idle(struct hinic_cmdq *cmdq)
 157 {
 158         struct hinic_wq *wq = cmdq->wq;
 159
 160         return ((wq->delta) == wq->q_depth ? true : false);
 161 }
 162
 163 struct hinic_cmd_buf *hinic_alloc_cmd_buf(void *hwdev)
 164 {
 165         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 166         struct hinic_cmd_buf *cmd_buf;
 167
 168         cmd_buf = kzalloc(sizeof(*cmd_buf), GFP_KERNEL);
 169         if (!cmd_buf) {
 170                 PMD_DRV_LOG(ERR, "Allocate cmd buffer failed");
 171                 return NULL;
 172         }
 173
 174         cmd_buf->buf = pci_pool_alloc(cmdqs->cmd_buf_pool, GFP_KERNEL,
 175                                       &cmd_buf->dma_addr);
 176         if (!cmd_buf->buf) {
 177                 PMD_DRV_LOG(ERR, "Allocate cmd from the pool failed");
 178                 goto alloc_pci_buf_err;
 179         }
 180
 181         return cmd_buf;
 182
 183 alloc_pci_buf_err:
 184         kfree(cmd_buf);
 185         return NULL;
 186 }
 187
 188 void hinic_free_cmd_buf(void *hwdev, struct hinic_cmd_buf *cmd_buf)
 189 {
 190         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 191
 192         pci_pool_free(cmdqs->cmd_buf_pool, cmd_buf->buf, cmd_buf->dma_addr);
 193         kfree(cmd_buf);
 194 }
 195
 196 static u32 cmdq_wqe_size(enum cmdq_wqe_type wqe_type)
 197 {
 198         u32 wqe_size = 0;
 199
 200         switch (wqe_type) {
 201         case WQE_LCMD_TYPE:
 202                 wqe_size = WQE_LCMD_SIZE;
 203                 break;
 204         case WQE_SCMD_TYPE:
 205                 wqe_size = WQE_SCMD_SIZE;
 206                 break;
 207         }
 208
 209         return wqe_size;
 210 }
 211
 212 static int cmdq_get_wqe_size(enum bufdesc_len len)
 213 {
 214         int wqe_size = 0;
 215
 216         switch (len) {
 217         case BUFDESC_LCMD_LEN:
 218                 wqe_size = WQE_LCMD_SIZE;
 219                 break;
 220         case BUFDESC_SCMD_LEN:
 221                 wqe_size = WQE_SCMD_SIZE;
 222                 break;
 223         }
 224
 225         return wqe_size;
 226 }
 227
 228 static void cmdq_set_completion(struct hinic_cmdq_completion *complete,
 229                                         struct hinic_cmd_buf *buf_out)
 230 {
 231         struct hinic_sge_resp *sge_resp = &complete->sge_resp;
 232
 233         hinic_set_sge(&sge_resp->sge, buf_out->dma_addr,
 234                       HINIC_CMDQ_BUF_SIZE);
 235 }
 236
 237 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe,
 238                                         struct hinic_cmd_buf *buf_in)
 239 {
 240         hinic_set_sge(&wqe->buf_desc.sge, buf_in->dma_addr, buf_in->size);
 241 }
 242
 243 static void cmdq_fill_db(struct hinic_cmdq_db *db,
 244                         enum hinic_cmdq_type cmdq_type, u16 prod_idx)
 245 {
 246         db->db_info = CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) |
 247                         CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, QUEUE_TYPE) |
 248                         CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE)          |
 249                         CMDQ_DB_INFO_SET(HINIC_DB_SRC_CMDQ_TYPE, SRC_TYPE);
 250 }
 251
 252 static void cmdq_set_db(struct hinic_cmdq *cmdq,
 253                         enum hinic_cmdq_type cmdq_type, u16 prod_idx)
 254 {
 255         struct hinic_cmdq_db db;
 256
 257         cmdq_fill_db(&db, cmdq_type, prod_idx);
 258
 259         /* The data that is written to HW should be in Big Endian Format */
 260         db.db_info = cpu_to_be32(db.db_info);
 261
 262         rte_wmb();      /* write all before the doorbell */
 263
 264         writel(db.db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx));
 265 }
 266
 267 static void cmdq_wqe_fill(void *dst, void *src)
 268 {
 269         memcpy((u8 *)dst + FIRST_DATA_TO_WRITE_LAST,
 270                (u8 *)src + FIRST_DATA_TO_WRITE_LAST,
 271                CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST);
 272
 273         rte_wmb();/* The first 8 bytes should be written last */
 274
 275         *(u64 *)dst = *(u64 *)src;
 276 }
 277
 278 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped,
 279                                   enum hinic_ack_type ack_type,
 280                                   enum hinic_mod_type mod, u8 cmd, u16 prod_idx,
 281                                   enum completion_format complete_format,
 282                                   enum data_format local_data_format,
 283                                   enum bufdesc_len buf_len)
 284 {
 285         struct hinic_ctrl *ctrl;
 286         enum ctrl_sect_len ctrl_len;
 287         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 288         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 289         u32 saved_data = WQE_HEADER(wqe)->saved_data;
 290
 291         if (local_data_format == DATA_SGE) {
 292                 wqe_lcmd = &wqe->wqe_lcmd;
 293
 294                 wqe_lcmd->status.status_info = 0;
 295                 ctrl = &wqe_lcmd->ctrl;
 296                 ctrl_len = CTRL_SECT_LEN;
 297         } else {
 298                 wqe_scmd = &wqe->inline_wqe.wqe_scmd;
 299
 300                 wqe_scmd->status.status_info = 0;
 301                 ctrl = &wqe_scmd->ctrl;
 302                 ctrl_len = CTRL_DIRECT_SECT_LEN;
 303         }
 304
 305         ctrl->ctrl_info = CMDQ_CTRL_SET(prod_idx, PI)           |
 306                         CMDQ_CTRL_SET(cmd, CMD)                 |
 307                         CMDQ_CTRL_SET(mod, MOD)                 |
 308                         CMDQ_CTRL_SET(ack_type, ACK_TYPE);
 309
 310         WQE_HEADER(wqe)->header_info =
 311                 CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN) |
 312                 CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT) |
 313                 CMDQ_WQE_HEADER_SET(local_data_format, DATA_FMT)        |
 314                 CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ)      |
 315                 CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) |
 316                 CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN)         |
 317                 CMDQ_WQE_HEADER_SET((u32)wrapped, HW_BUSY_BIT);
 318
 319         if (cmd == CMDQ_SET_ARM_CMD && mod == HINIC_MOD_COMM) {
 320                 saved_data &= SAVED_DATA_CLEAR(saved_data, ARM);
 321                 WQE_HEADER(wqe)->saved_data = saved_data |
 322                                                 SAVED_DATA_SET(1, ARM);
 323         } else {
 324                 saved_data &= SAVED_DATA_CLEAR(saved_data, ARM);
 325                 WQE_HEADER(wqe)->saved_data = saved_data;
 326         }
 327 }
 328
 329 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe,
 330                               enum cmdq_cmd_type cmd_type,
 331                               struct hinic_cmd_buf *buf_in,
 332                               struct hinic_cmd_buf *buf_out, int wrapped,
 333                               enum hinic_ack_type ack_type,
 334                               enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
 335 {
 336         struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
 337         enum completion_format complete_format = COMPLETE_DIRECT;
 338
 339         switch (cmd_type) {
 340         case SYNC_CMD_SGE_RESP:
 341                 if (buf_out) {
 342                         complete_format = COMPLETE_SGE;
 343                         cmdq_set_completion(&wqe_lcmd->completion, buf_out);
 344                 }
 345                 break;
 346         case SYNC_CMD_DIRECT_RESP:
 347                 complete_format = COMPLETE_DIRECT;
 348                 wqe_lcmd->completion.direct_resp = 0;
 349                 break;
 350         case ASYNC_CMD:
 351                 complete_format = COMPLETE_DIRECT;
 352                 wqe_lcmd->completion.direct_resp = 0;
 353
 354                 wqe_lcmd->buf_desc.saved_async_buf = (u64)(buf_in);
 355                 break;
 356         }
 357
 358         cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd,
 359                               prod_idx, complete_format, DATA_SGE,
 360                               BUFDESC_LCMD_LEN);
 361
 362         cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in);
 363 }
 364
 365 static int cmdq_params_valid(struct hinic_cmd_buf *buf_in)
 366 {
 367         if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE) {
 368                 PMD_DRV_LOG(ERR, "Invalid CMDQ buffer size");
 369                 return -EINVAL;
 370         }
 371
 372         return 0;
 373 }
 374
 375 static int wait_cmdqs_enable(struct hinic_cmdqs *cmdqs)
 376 {
 377         unsigned long end;
 378
 379         end = jiffies + msecs_to_jiffies(WAIT_CMDQ_ENABLE_TIMEOUT);
 380         do {
 381                 if (cmdqs->status & HINIC_CMDQ_ENABLE)
 382                         return 0;
 383
 384         } while (time_before(jiffies, end));
 385
 386         return -EBUSY;
 387 }
 388
 389 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx,
 390                                 int errcode)
 391 {
 392         cmdq->errcode[prod_idx] = errcode;
 393 }
 394
 395 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq,
 396                                    struct hinic_cmdq_wqe *wqe)
 397 {
 398         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 399         struct hinic_cmdq_inline_wqe *inline_wqe;
 400         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 401         struct hinic_ctrl *ctrl;
 402         u32 header_info = be32_to_cpu(WQE_HEADER(wqe)->header_info);
 403         int buf_len = CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN);
 404         int wqe_size = cmdq_get_wqe_size(buf_len);
 405         u16 num_wqebbs;
 406
 407         if (wqe_size == WQE_LCMD_SIZE) {
 408                 wqe_lcmd = &wqe->wqe_lcmd;
 409                 ctrl = &wqe_lcmd->ctrl;
 410         } else {
 411                 inline_wqe = &wqe->inline_wqe;
 412                 wqe_scmd = &inline_wqe->wqe_scmd;
 413                 ctrl = &wqe_scmd->ctrl;
 414         }
 415
 416         /* clear HW busy bit */
 417         ctrl->ctrl_info = 0;
 418
 419         rte_wmb();      /* verify wqe is clear */
 420
 421         num_wqebbs = WQE_NUM_WQEBBS(wqe_size, cmdq->wq);
 422         hinic_put_wqe(cmdq->wq, num_wqebbs);
 423 }
 424
 425 static int hinic_set_cmdq_ctxts(struct hinic_hwdev *hwdev)
 426 {
 427         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 428         struct hinic_cmdq_ctxt *cmdq_ctxt;
 429         enum hinic_cmdq_type cmdq_type;
 430         u16 in_size;
 431         int err;
 432
 433         cmdq_type = HINIC_CMDQ_SYNC;
 434         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 435                 cmdq_ctxt = &cmdqs->cmdq[cmdq_type].cmdq_ctxt;
 436                 cmdq_ctxt->resp_aeq_num = HINIC_AEQ1;
 437                 in_size = sizeof(*cmdq_ctxt);
 438                 err = hinic_msg_to_mgmt_sync(hwdev, HINIC_MOD_COMM,
 439                                              HINIC_MGMT_CMD_CMDQ_CTXT_SET,
 440                                              cmdq_ctxt, in_size, NULL,
 441                                              NULL, 0);
 442                 if (err) {
 443                         PMD_DRV_LOG(ERR, "Set cmdq ctxt failed");
 444                         return -EFAULT;
 445                 }
 446         }
 447
 448         cmdqs->status |= HINIC_CMDQ_ENABLE;
 449
 450         return 0;
 451 }
 452
 453 void hinic_comm_cmdqs_free(struct hinic_hwdev *hwdev)
 454 {
 455         hinic_cmdqs_free(hwdev);
 456 }
 457
 458 int hinic_reinit_cmdq_ctxts(struct hinic_hwdev *hwdev)
 459 {
 460         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 461         enum hinic_cmdq_type cmdq_type;
 462
 463         cmdq_type = HINIC_CMDQ_SYNC;
 464         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 465                 cmdqs->cmdq[cmdq_type].wrapped = 1;
 466                 hinic_wq_wqe_pg_clear(cmdqs->cmdq[cmdq_type].wq);
 467         }
 468
 469         return hinic_set_cmdq_ctxts(hwdev);
 470 }
 471
 472 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_hwdev *hwdev,
 473                      struct hinic_wq *wq, enum hinic_cmdq_type q_type)
 474 {
 475         void __iomem *db_base;
 476         int err = 0;
 477         size_t errcode_size;
 478         size_t cmd_infos_size;
 479
 480         cmdq->wq = wq;
 481         cmdq->cmdq_type = q_type;
 482         cmdq->wrapped = 1;
 483
 484         spin_lock_init(&cmdq->cmdq_lock);
 485
 486         errcode_size = wq->q_depth * sizeof(*cmdq->errcode);
 487         cmdq->errcode = kzalloc(errcode_size, GFP_KERNEL);
 488         if (!cmdq->errcode) {
 489                 PMD_DRV_LOG(ERR, "Allocate errcode for cmdq failed");
 490                 spin_lock_deinit(&cmdq->cmdq_lock);
 491                 return -ENOMEM;
 492         }
 493
 494         cmd_infos_size = wq->q_depth * sizeof(*cmdq->cmd_infos);
 495         cmdq->cmd_infos = kzalloc(cmd_infos_size, GFP_KERNEL);
 496         if (!cmdq->cmd_infos) {
 497                 PMD_DRV_LOG(ERR, "Allocate errcode for cmdq failed");
 498                 err = -ENOMEM;
 499                 goto cmd_infos_err;
 500         }
 501
 502         err = hinic_alloc_db_addr(hwdev, &db_base);
 503         if (err)
 504                 goto alloc_db_err;
 505
 506         cmdq->db_base = (u8 *)db_base;
 507         return 0;
 508
 509 alloc_db_err:
 510         kfree(cmdq->cmd_infos);
 511
 512 cmd_infos_err:
 513         kfree(cmdq->errcode);
 514         spin_lock_deinit(&cmdq->cmdq_lock);
 515
 516         return err;
 517 }
 518
 519 static void free_cmdq(struct hinic_hwdev *hwdev, struct hinic_cmdq *cmdq)
 520 {
 521         hinic_free_db_addr(hwdev, cmdq->db_base);
 522         kfree(cmdq->cmd_infos);
 523         kfree(cmdq->errcode);
 524         spin_lock_deinit(&cmdq->cmdq_lock);
 525 }
 526
 527 static int hinic_cmdqs_init(struct hinic_hwdev *hwdev)
 528 {
 529         struct hinic_cmdqs *cmdqs;
 530         struct hinic_cmdq_ctxt *cmdq_ctxt;
 531         enum hinic_cmdq_type type, cmdq_type;
 532         size_t saved_wqs_size;
 533         int err;
 534
 535         cmdqs = kzalloc(sizeof(*cmdqs), GFP_KERNEL);
 536         if (!cmdqs)
 537                 return -ENOMEM;
 538
 539         hwdev->cmdqs = cmdqs;
 540         cmdqs->hwdev = hwdev;
 541
 542         saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq);
 543         cmdqs->saved_wqs = kzalloc(saved_wqs_size, GFP_KERNEL);
 544         if (!cmdqs->saved_wqs) {
 545                 PMD_DRV_LOG(ERR, "Allocate saved wqs failed");
 546                 err = -ENOMEM;
 547                 goto alloc_wqs_err;
 548         }
 549
 550         cmdqs->cmd_buf_pool = dma_pool_create("hinic_cmdq", hwdev,
 551                                               HINIC_CMDQ_BUF_SIZE,
 552                                               HINIC_CMDQ_BUF_SIZE, 0ULL);
 553         if (!cmdqs->cmd_buf_pool) {
 554                 PMD_DRV_LOG(ERR, "Create cmdq buffer pool failed");
 555                 err = -ENOMEM;
 556                 goto pool_create_err;
 557         }
 558
 559         err = hinic_cmdq_alloc(cmdqs->saved_wqs, hwdev,
 560                                HINIC_MAX_CMDQ_TYPES, HINIC_CMDQ_WQ_BUF_SIZE,
 561                                CMDQ_WQEBB_SHIFT, HINIC_CMDQ_DEPTH);
 562         if (err) {
 563                 PMD_DRV_LOG(ERR, "Allocate cmdq failed");
 564                 goto cmdq_alloc_err;
 565         }
 566
 567         cmdq_type = HINIC_CMDQ_SYNC;
 568         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 569                 err = init_cmdq(&cmdqs->cmdq[cmdq_type], hwdev,
 570                                 &cmdqs->saved_wqs[cmdq_type], cmdq_type);
 571                 if (err) {
 572                         PMD_DRV_LOG(ERR, "Initialize cmdq failed");
 573                         goto init_cmdq_err;
 574                 }
 575
 576                 cmdq_ctxt = &cmdqs->cmdq[cmdq_type].cmdq_ctxt;
 577                 cmdq_init_queue_ctxt(&cmdqs->cmdq[cmdq_type], cmdq_ctxt);
 578         }
 579
 580         err = hinic_set_cmdq_ctxts(hwdev);
 581         if (err)
 582                 goto init_cmdq_err;
 583
 584         return 0;
 585
 586 init_cmdq_err:
 587         type = HINIC_CMDQ_SYNC;
 588         for ( ; type < cmdq_type; type++)
 589                 free_cmdq(hwdev, &cmdqs->cmdq[type]);
 590
 591         hinic_cmdq_free(hwdev, cmdqs->saved_wqs, HINIC_MAX_CMDQ_TYPES);
 592
 593 cmdq_alloc_err:
 594         dma_pool_destroy(cmdqs->cmd_buf_pool);
 595
 596 pool_create_err:
 597         kfree(cmdqs->saved_wqs);
 598
 599 alloc_wqs_err:
 600         kfree(cmdqs);
 601
 602         return err;
 603 }
 604
 605 static void hinic_cmdqs_free(struct hinic_hwdev *hwdev)
 606 {
 607         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 608         enum hinic_cmdq_type cmdq_type = HINIC_CMDQ_SYNC;
 609
 610         cmdqs->status &= ~HINIC_CMDQ_ENABLE;
 611
 612         for ( ; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++)
 613                 free_cmdq(cmdqs->hwdev, &cmdqs->cmdq[cmdq_type]);
 614
 615         hinic_cmdq_free(hwdev, cmdqs->saved_wqs,
 616                         HINIC_MAX_CMDQ_TYPES);
 617
 618         dma_pool_destroy(cmdqs->cmd_buf_pool);
 619
 620         kfree(cmdqs->saved_wqs);
 621
 622         kfree(cmdqs);
 623 }
 624
 625 static int hinic_set_cmdq_depth(struct hinic_hwdev *hwdev, u16 cmdq_depth)
 626 {
 627         struct hinic_root_ctxt root_ctxt;
 628
 629         memset(&root_ctxt, 0, sizeof(root_ctxt));
 630         root_ctxt.mgmt_msg_head.resp_aeq_num = HINIC_AEQ1;
 631         root_ctxt.func_idx = hinic_global_func_id(hwdev);
 632         root_ctxt.ppf_idx = hinic_ppf_idx(hwdev);
 633         root_ctxt.set_cmdq_depth = 1;
 634         root_ctxt.cmdq_depth = (u8)ilog2(cmdq_depth);
 635         return hinic_msg_to_mgmt_sync(hwdev, HINIC_MOD_COMM,
 636                                       HINIC_MGMT_CMD_VAT_SET,
 637                                       &root_ctxt, sizeof(root_ctxt),
 638                                       NULL, NULL, 0);
 639 }
 640
 641 int hinic_comm_cmdqs_init(struct hinic_hwdev *hwdev)
 642 {
 643         int err;
 644
 645         err = hinic_cmdqs_init(hwdev);
 646         if (err) {
 647                 PMD_DRV_LOG(ERR, "Init cmd queues failed");
 648                 return err;
 649         }
 650
 651         err = hinic_set_cmdq_depth(hwdev, HINIC_CMDQ_DEPTH);
 652         if (err) {
 653                 PMD_DRV_LOG(ERR, "Set cmdq depth failed");
 654                 goto set_cmdq_depth_err;
 655         }
 656
 657         return 0;
 658
 659 set_cmdq_depth_err:
 660         hinic_cmdqs_free(hwdev);
 661
 662         return err;
 663 }
 664
 665 static void cmdq_init_queue_ctxt(struct hinic_cmdq *cmdq,
 666                                  struct hinic_cmdq_ctxt *cmdq_ctxt)
 667 {
 668         struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)cmdq_to_cmdqs(cmdq);
 669         struct hinic_hwdev *hwdev = cmdqs->hwdev;
 670         struct hinic_wq *wq = cmdq->wq;
 671         struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info;
 672         u64 wq_first_page_paddr, pfn;
 673
 674         u16 start_ci = (u16)(wq->cons_idx);
 675
 676         /* The data in the HW is in Big Endian Format */
 677         wq_first_page_paddr = wq->queue_buf_paddr;
 678
 679         pfn = CMDQ_PFN(wq_first_page_paddr, HINIC_PAGE_SIZE);
 680         ctxt_info->curr_wqe_page_pfn =
 681                 CMDQ_CTXT_PAGE_INFO_SET(1, HW_BUSY_BIT) |
 682                 CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN)      |
 683                 CMDQ_CTXT_PAGE_INFO_SET(0, CEQ_ARM)     |
 684                 CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) |
 685                 CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN);
 686
 687         ctxt_info->wq_block_pfn = CMDQ_CTXT_BLOCK_INFO_SET(start_ci, CI) |
 688                                 CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN);
 689         cmdq_ctxt->func_idx = HINIC_HWIF_GLOBAL_IDX(hwdev->hwif);
 690         cmdq_ctxt->ppf_idx  = HINIC_HWIF_PPF_IDX(hwdev->hwif);
 691         cmdq_ctxt->cmdq_id  = cmdq->cmdq_type;
 692 }
 693
 694 static int hinic_cmdq_poll_msg(struct hinic_cmdq *cmdq, u32 timeout)
 695 {
 696         struct hinic_cmdq_wqe *wqe;
 697         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 698         struct hinic_ctrl *ctrl;
 699         struct hinic_cmdq_cmd_info *cmd_info;
 700         u32 status_info, ctrl_info;
 701         u16 ci;
 702         int errcode;
 703         unsigned long end;
 704         int done = 0;
 705         int rc = 0;
 706
 707         wqe = hinic_read_wqe(cmdq->wq, 1, &ci);
 708         if (wqe == NULL) {
 709                 PMD_DRV_LOG(ERR, "No outstanding cmdq msg");
 710                 return -EINVAL;
 711         }
 712
 713         cmd_info = &cmdq->cmd_infos[ci];
 714         /* this cmd has not been filled and send to hw, or get TMO msg ack*/
 715         if (cmd_info->cmd_type == HINIC_CMD_TYPE_NONE) {
 716                 PMD_DRV_LOG(ERR, "Cmdq msg has not been filled and send to hw, or get TMO msg ack. cmdq ci: %u",
 717                             ci);
 718                 return -EINVAL;
 719         }
 720
 721         /* only arm bit is using scmd wqe, the wqe is lcmd */
 722         wqe_lcmd = &wqe->wqe_lcmd;
 723         ctrl = &wqe_lcmd->ctrl;
 724         end = jiffies + msecs_to_jiffies(timeout);
 725         do {
 726                 ctrl_info = be32_to_cpu((ctrl)->ctrl_info);
 727                 if (WQE_COMPLETED(ctrl_info)) {
 728                         done = 1;
 729                         break;
 730                 }
 731
 732                 rte_delay_ms(1);
 733         } while (time_before(jiffies, end));
 734
 735         if (done) {
 736                 status_info = be32_to_cpu(wqe_lcmd->status.status_info);
 737                 errcode = WQE_ERRCODE_GET(status_info, VAL);
 738                 cmdq_update_errcode(cmdq, ci, errcode);
 739                 clear_wqe_complete_bit(cmdq, wqe);
 740                 rc = 0;
 741         } else {
 742                 PMD_DRV_LOG(ERR, "Poll cmdq msg time out, ci: %u", ci);
 743                 rc = -ETIMEDOUT;
 744         }
 745
 746         /* set this cmd invalid */
 747         cmd_info->cmd_type = HINIC_CMD_TYPE_NONE;
 748
 749         return rc;
 750 }
 751
 752 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,
 753                                      enum hinic_ack_type ack_type,
 754                                      enum hinic_mod_type mod, u8 cmd,
 755                                      struct hinic_cmd_buf *buf_in,
 756                                      u64 *out_param, u32 timeout)
 757 {
 758         struct hinic_wq *wq = cmdq->wq;
 759         struct hinic_cmdq_wqe *curr_wqe, wqe;
 760         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 761         u16 curr_prod_idx, next_prod_idx, num_wqebbs;
 762         int wrapped;
 763         u32 timeo, wqe_size;
 764         int err;
 765
 766         wqe_size = cmdq_wqe_size(WQE_LCMD_TYPE);
 767         num_wqebbs = WQE_NUM_WQEBBS(wqe_size, wq);
 768
 769         /* Keep wrapped and doorbell index correct. */
 770         spin_lock(&cmdq->cmdq_lock);
 771
 772         curr_wqe = hinic_get_wqe(cmdq->wq, num_wqebbs, &curr_prod_idx);
 773         if (!curr_wqe) {
 774                 err = -EBUSY;
 775                 goto cmdq_unlock;
 776         }
 777
 778         memset(&wqe, 0, sizeof(wqe));
 779         wrapped = cmdq->wrapped;
 780
 781         next_prod_idx = curr_prod_idx + num_wqebbs;
 782         if (next_prod_idx >= wq->q_depth) {
 783                 cmdq->wrapped = !cmdq->wrapped;
 784                 next_prod_idx -= wq->q_depth;
 785         }
 786
 787         cmdq_set_lcmd_wqe(&wqe, SYNC_CMD_DIRECT_RESP, buf_in, NULL,
 788                           wrapped, ack_type, mod, cmd, curr_prod_idx);
 789
 790         /* The data that is written to HW should be in Big Endian Format */
 791         hinic_cpu_to_be32(&wqe, wqe_size);
 792
 793         /* CMDQ WQE is not shadow, therefore wqe will be written to wq */
 794         cmdq_wqe_fill(curr_wqe, &wqe);
 795
 796         cmdq->cmd_infos[curr_prod_idx].cmd_type = HINIC_CMD_TYPE_NORMAL;
 797
 798         cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
 799
 800         timeo = msecs_to_jiffies(timeout ? timeout : CMDQ_CMD_TIMEOUT);
 801         err = hinic_cmdq_poll_msg(cmdq, timeo);
 802         if (err) {
 803                 PMD_DRV_LOG(ERR, "Cmdq poll msg ack failed, prod idx: 0x%x",
 804                         curr_prod_idx);
 805                 err = -ETIMEDOUT;
 806                 goto cmdq_unlock;
 807         }
 808
 809         rte_smp_rmb();  /* read error code after completion */
 810
 811         if (out_param) {
 812                 wqe_lcmd = &curr_wqe->wqe_lcmd;
 813                 *out_param = cpu_to_be64(wqe_lcmd->completion.direct_resp);
 814         }
 815
 816         if (cmdq->errcode[curr_prod_idx] > 1) {
 817                 err = cmdq->errcode[curr_prod_idx];
 818                 goto cmdq_unlock;
 819         }
 820
 821 cmdq_unlock:
 822         spin_unlock(&cmdq->cmdq_lock);
 823
 824         return err;
 825 }
 826
 827 int hinic_cmdq_direct_resp(void *hwdev, enum hinic_ack_type ack_type,
 828                            enum hinic_mod_type mod, u8 cmd,
 829                            struct hinic_cmd_buf *buf_in,
 830                            u64 *out_param, u32 timeout)
 831 {
 832         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 833         int err = cmdq_params_valid(buf_in);
 834
 835         if (err) {
 836                 PMD_DRV_LOG(ERR, "Invalid CMDQ parameters");
 837                 return err;
 838         }
 839
 840         err = wait_cmdqs_enable(cmdqs);
 841         if (err) {
 842                 PMD_DRV_LOG(ERR, "Cmdq is disable");
 843                 return err;
 844         }
 845
 846         return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC],
 847                                          ack_type, mod, cmd, buf_in,
 848                                          out_param, timeout);
 849 }