drivers/net/hinic/base/hinic_pmd_cmdq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include "hinic_compat.h"
   6 #include "hinic_pmd_hwdev.h"
   7 #include "hinic_pmd_hwif.h"
   8 #include "hinic_pmd_wq.h"
   9 #include "hinic_pmd_mgmt.h"
  10 #include "hinic_pmd_mbox.h"
  11 #include "hinic_pmd_cmdq.h"
  12
  13 #define CMDQ_CMD_TIMEOUT                                5000 /* millisecond */
  14
  15 #define UPPER_8_BITS(data)                              (((data) >> 8) & 0xFF)
  16 #define LOWER_8_BITS(data)                              ((data) & 0xFF)
  17
  18 #define CMDQ_DB_INFO_HI_PROD_IDX_SHIFT                  0
  19 #define CMDQ_DB_INFO_QUEUE_TYPE_SHIFT                   23
  20 #define CMDQ_DB_INFO_CMDQ_TYPE_SHIFT                    24
  21 #define CMDQ_DB_INFO_SRC_TYPE_SHIFT                     27
  22
  23 #define CMDQ_DB_INFO_HI_PROD_IDX_MASK                   0xFFU
  24 #define CMDQ_DB_INFO_QUEUE_TYPE_MASK                    0x1U
  25 #define CMDQ_DB_INFO_CMDQ_TYPE_MASK                     0x7U
  26 #define CMDQ_DB_INFO_SRC_TYPE_MASK                      0x1FU
  27
  28 #define CMDQ_DB_INFO_SET(val, member)           \
  29         (((val) & CMDQ_DB_INFO_##member##_MASK) <<      \
  30                 CMDQ_DB_INFO_##member##_SHIFT)
  31
  32 #define CMDQ_CTRL_PI_SHIFT                              0
  33 #define CMDQ_CTRL_CMD_SHIFT                             16
  34 #define CMDQ_CTRL_MOD_SHIFT                             24
  35 #define CMDQ_CTRL_ACK_TYPE_SHIFT                        29
  36 #define CMDQ_CTRL_HW_BUSY_BIT_SHIFT                     31
  37
  38 #define CMDQ_CTRL_PI_MASK                               0xFFFFU
  39 #define CMDQ_CTRL_CMD_MASK                              0xFFU
  40 #define CMDQ_CTRL_MOD_MASK                              0x1FU
  41 #define CMDQ_CTRL_ACK_TYPE_MASK                         0x3U
  42 #define CMDQ_CTRL_HW_BUSY_BIT_MASK                      0x1U
  43
  44 #define CMDQ_CTRL_SET(val, member)              \
  45         (((val) & CMDQ_CTRL_##member##_MASK) << CMDQ_CTRL_##member##_SHIFT)
  46
  47 #define CMDQ_CTRL_GET(val, member)              \
  48         (((val) >> CMDQ_CTRL_##member##_SHIFT) & CMDQ_CTRL_##member##_MASK)
  49
  50 #define CMDQ_WQE_HEADER_BUFDESC_LEN_SHIFT               0
  51 #define CMDQ_WQE_HEADER_COMPLETE_FMT_SHIFT              15
  52 #define CMDQ_WQE_HEADER_DATA_FMT_SHIFT                  22
  53 #define CMDQ_WQE_HEADER_COMPLETE_REQ_SHIFT              23
  54 #define CMDQ_WQE_HEADER_COMPLETE_SECT_LEN_SHIFT         27
  55 #define CMDQ_WQE_HEADER_CTRL_LEN_SHIFT                  29
  56 #define CMDQ_WQE_HEADER_HW_BUSY_BIT_SHIFT               31
  57
  58 #define CMDQ_WQE_HEADER_BUFDESC_LEN_MASK                0xFFU
  59 #define CMDQ_WQE_HEADER_COMPLETE_FMT_MASK               0x1U
  60 #define CMDQ_WQE_HEADER_DATA_FMT_MASK                   0x1U
  61 #define CMDQ_WQE_HEADER_COMPLETE_REQ_MASK               0x1U
  62 #define CMDQ_WQE_HEADER_COMPLETE_SECT_LEN_MASK          0x3U
  63 #define CMDQ_WQE_HEADER_CTRL_LEN_MASK                   0x3U
  64 #define CMDQ_WQE_HEADER_HW_BUSY_BIT_MASK                0x1U
  65
  66 #define CMDQ_WQE_HEADER_SET(val, member)        \
  67         (((val) & CMDQ_WQE_HEADER_##member##_MASK) <<   \
  68                 CMDQ_WQE_HEADER_##member##_SHIFT)
  69
  70 #define CMDQ_WQE_HEADER_GET(val, member)        \
  71         (((val) >> CMDQ_WQE_HEADER_##member##_SHIFT) &  \
  72                 CMDQ_WQE_HEADER_##member##_MASK)
  73
  74 #define CMDQ_CTXT_CURR_WQE_PAGE_PFN_SHIFT               0
  75 #define CMDQ_CTXT_EQ_ID_SHIFT                           56
  76 #define CMDQ_CTXT_CEQ_ARM_SHIFT                         61
  77 #define CMDQ_CTXT_CEQ_EN_SHIFT                          62
  78 #define CMDQ_CTXT_HW_BUSY_BIT_SHIFT                     63
  79
  80 #define CMDQ_CTXT_CURR_WQE_PAGE_PFN_MASK                0xFFFFFFFFFFFFF
  81 #define CMDQ_CTXT_EQ_ID_MASK                            0x1F
  82 #define CMDQ_CTXT_CEQ_ARM_MASK                          0x1
  83 #define CMDQ_CTXT_CEQ_EN_MASK                           0x1
  84 #define CMDQ_CTXT_HW_BUSY_BIT_MASK                      0x1
  85
  86 #define CMDQ_CTXT_PAGE_INFO_SET(val, member)            \
  87         (((u64)(val) & CMDQ_CTXT_##member##_MASK) << CMDQ_CTXT_##member##_SHIFT)
  88
  89 #define CMDQ_CTXT_PAGE_INFO_CLEAR(val, member)          \
  90         ((val) & (~((u64)CMDQ_CTXT_##member##_MASK <<   \
  91                 CMDQ_CTXT_##member##_SHIFT)))
  92
  93 #define CMDQ_CTXT_WQ_BLOCK_PFN_SHIFT                    0
  94 #define CMDQ_CTXT_CI_SHIFT                              52
  95
  96 #define CMDQ_CTXT_WQ_BLOCK_PFN_MASK                     0xFFFFFFFFFFFFF
  97 #define CMDQ_CTXT_CI_MASK                               0xFFF
  98
  99 #define CMDQ_CTXT_BLOCK_INFO_SET(val, member)           \
 100         (((u64)(val) & CMDQ_CTXT_##member##_MASK) << CMDQ_CTXT_##member##_SHIFT)
 101
 102 #define SAVED_DATA_ARM_SHIFT                    31
 103
 104 #define SAVED_DATA_ARM_MASK                     0x1U
 105
 106 #define SAVED_DATA_SET(val, member)             \
 107         (((val) & SAVED_DATA_##member##_MASK) << SAVED_DATA_##member##_SHIFT)
 108
 109 #define SAVED_DATA_CLEAR(val, member)           \
 110         ((val) & (~(SAVED_DATA_##member##_MASK << SAVED_DATA_##member##_SHIFT)))
 111
 112 #define WQE_ERRCODE_VAL_SHIFT                   20
 113
 114 #define WQE_ERRCODE_VAL_MASK                    0xF
 115
 116 #define WQE_ERRCODE_GET(val, member)            \
 117         (((val) >> WQE_ERRCODE_##member##_SHIFT) & WQE_ERRCODE_##member##_MASK)
 118
 119 #define WQE_COMPLETED(ctrl_info)        CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT)
 120
 121 #define WQE_HEADER(wqe)         ((struct hinic_cmdq_header *)(wqe))
 122
 123 #define CMDQ_DB_PI_OFF(pi)              (((u16)LOWER_8_BITS(pi)) << 3)
 124
 125 #define CMDQ_DB_ADDR(db_base, pi)       \
 126         (((u8 *)(db_base) + HINIC_DB_OFF) + CMDQ_DB_PI_OFF(pi))
 127
 128 #define CMDQ_PFN(addr, page_size)       ((addr) >> (ilog2(page_size)))
 129
 130 #define FIRST_DATA_TO_WRITE_LAST        sizeof(u64)
 131
 132 #define WQE_LCMD_SIZE           64
 133 #define WQE_SCMD_SIZE           64
 134
 135 #define COMPLETE_LEN            3
 136
 137 #define CMDQ_WQEBB_SIZE         64
 138 #define CMDQ_WQEBB_SHIFT        6
 139
 140 #define CMDQ_WQE_SIZE           64
 141
 142 #define HINIC_CMDQ_WQ_BUF_SIZE  4096
 143
 144 #define WQE_NUM_WQEBBS(wqe_size, wq)    \
 145         ((u16)(ALIGN((u32)(wqe_size), (wq)->wqebb_size) / (wq)->wqebb_size))
 146
 147 #define cmdq_to_cmdqs(cmdq)     container_of((cmdq) - (cmdq)->cmdq_type, \
 148                                 struct hinic_cmdqs, cmdq[0])
 149
 150 #define WAIT_CMDQ_ENABLE_TIMEOUT        300
 151
 152
 153 static void cmdq_init_queue_ctxt(struct hinic_cmdq *cmdq,
 154                                  struct hinic_cmdq_ctxt *cmdq_ctxt);
 155 static void hinic_cmdqs_free(struct hinic_hwdev *hwdev);
 156
 157 bool hinic_cmdq_idle(struct hinic_cmdq *cmdq)
 158 {
 159         struct hinic_wq *wq = cmdq->wq;
 160
 161         return ((wq->delta) == wq->q_depth ? true : false);
 162 }
 163
 164 struct hinic_cmd_buf *hinic_alloc_cmd_buf(void *hwdev)
 165 {
 166         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 167         struct hinic_cmd_buf *cmd_buf;
 168
 169         cmd_buf = kzalloc(sizeof(*cmd_buf), GFP_KERNEL);
 170         if (!cmd_buf) {
 171                 PMD_DRV_LOG(ERR, "Allocate cmd buffer failed");
 172                 return NULL;
 173         }
 174
 175         cmd_buf->buf = pci_pool_alloc(cmdqs->cmd_buf_pool, &cmd_buf->dma_addr);
 176         if (!cmd_buf->buf) {
 177                 PMD_DRV_LOG(ERR, "Allocate cmd from the pool failed");
 178                 goto alloc_pci_buf_err;
 179         }
 180
 181         return cmd_buf;
 182
 183 alloc_pci_buf_err:
 184         kfree(cmd_buf);
 185         return NULL;
 186 }
 187
 188 void hinic_free_cmd_buf(void *hwdev, struct hinic_cmd_buf *cmd_buf)
 189 {
 190         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 191
 192         pci_pool_free(cmdqs->cmd_buf_pool, cmd_buf->buf, cmd_buf->dma_addr);
 193         kfree(cmd_buf);
 194 }
 195
 196 static u32 cmdq_wqe_size(enum cmdq_wqe_type wqe_type)
 197 {
 198         u32 wqe_size = 0;
 199
 200         switch (wqe_type) {
 201         case WQE_LCMD_TYPE:
 202                 wqe_size = WQE_LCMD_SIZE;
 203                 break;
 204         case WQE_SCMD_TYPE:
 205                 wqe_size = WQE_SCMD_SIZE;
 206                 break;
 207         }
 208
 209         return wqe_size;
 210 }
 211
 212 static int cmdq_get_wqe_size(enum bufdesc_len len)
 213 {
 214         int wqe_size = 0;
 215
 216         switch (len) {
 217         case BUFDESC_LCMD_LEN:
 218                 wqe_size = WQE_LCMD_SIZE;
 219                 break;
 220         case BUFDESC_SCMD_LEN:
 221                 wqe_size = WQE_SCMD_SIZE;
 222                 break;
 223         }
 224
 225         return wqe_size;
 226 }
 227
 228 static void cmdq_set_completion(struct hinic_cmdq_completion *complete,
 229                                         struct hinic_cmd_buf *buf_out)
 230 {
 231         struct hinic_sge_resp *sge_resp = &complete->sge_resp;
 232
 233         hinic_set_sge(&sge_resp->sge, buf_out->dma_addr,
 234                       HINIC_CMDQ_BUF_SIZE);
 235 }
 236
 237 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe,
 238                                         struct hinic_cmd_buf *buf_in)
 239 {
 240         hinic_set_sge(&wqe->buf_desc.sge, buf_in->dma_addr, buf_in->size);
 241 }
 242
 243 static void cmdq_fill_db(struct hinic_cmdq_db *db,
 244                         enum hinic_cmdq_type cmdq_type, u16 prod_idx)
 245 {
 246         db->db_info = CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) |
 247                         CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, QUEUE_TYPE) |
 248                         CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE)          |
 249                         CMDQ_DB_INFO_SET(HINIC_DB_SRC_CMDQ_TYPE, SRC_TYPE);
 250 }
 251
 252 static void cmdq_set_db(struct hinic_cmdq *cmdq,
 253                         enum hinic_cmdq_type cmdq_type, u16 prod_idx)
 254 {
 255         struct hinic_cmdq_db db;
 256
 257         cmdq_fill_db(&db, cmdq_type, prod_idx);
 258
 259         /* The data that is written to HW should be in Big Endian Format */
 260         db.db_info = cpu_to_be32(db.db_info);
 261
 262         rte_wmb();      /* write all before the doorbell */
 263
 264         writel(db.db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx));
 265 }
 266
 267 static void cmdq_wqe_fill(void *dst, void *src)
 268 {
 269         memcpy((u8 *)dst + FIRST_DATA_TO_WRITE_LAST,
 270                (u8 *)src + FIRST_DATA_TO_WRITE_LAST,
 271                CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST);
 272
 273         rte_wmb();/* The first 8 bytes should be written last */
 274
 275         *(u64 *)dst = *(u64 *)src;
 276 }
 277
 278 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped,
 279                                   enum hinic_ack_type ack_type,
 280                                   enum hinic_mod_type mod, u8 cmd, u16 prod_idx,
 281                                   enum completion_format complete_format,
 282                                   enum data_format local_data_format,
 283                                   enum bufdesc_len buf_len)
 284 {
 285         struct hinic_ctrl *ctrl;
 286         enum ctrl_sect_len ctrl_len;
 287         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 288         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 289         u32 saved_data = WQE_HEADER(wqe)->saved_data;
 290
 291         if (local_data_format == DATA_SGE) {
 292                 wqe_lcmd = &wqe->wqe_lcmd;
 293
 294                 wqe_lcmd->status.status_info = 0;
 295                 ctrl = &wqe_lcmd->ctrl;
 296                 ctrl_len = CTRL_SECT_LEN;
 297         } else {
 298                 wqe_scmd = &wqe->inline_wqe.wqe_scmd;
 299
 300                 wqe_scmd->status.status_info = 0;
 301                 ctrl = &wqe_scmd->ctrl;
 302                 ctrl_len = CTRL_DIRECT_SECT_LEN;
 303         }
 304
 305         ctrl->ctrl_info = CMDQ_CTRL_SET(prod_idx, PI)           |
 306                         CMDQ_CTRL_SET(cmd, CMD)                 |
 307                         CMDQ_CTRL_SET(mod, MOD)                 |
 308                         CMDQ_CTRL_SET(ack_type, ACK_TYPE);
 309
 310         WQE_HEADER(wqe)->header_info =
 311                 CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN) |
 312                 CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT) |
 313                 CMDQ_WQE_HEADER_SET(local_data_format, DATA_FMT)        |
 314                 CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ)      |
 315                 CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) |
 316                 CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN)         |
 317                 CMDQ_WQE_HEADER_SET((u32)wrapped, HW_BUSY_BIT);
 318
 319         if (cmd == CMDQ_SET_ARM_CMD && mod == HINIC_MOD_COMM) {
 320                 saved_data &= SAVED_DATA_CLEAR(saved_data, ARM);
 321                 WQE_HEADER(wqe)->saved_data = saved_data |
 322                                                 SAVED_DATA_SET(1, ARM);
 323         } else {
 324                 saved_data &= SAVED_DATA_CLEAR(saved_data, ARM);
 325                 WQE_HEADER(wqe)->saved_data = saved_data;
 326         }
 327 }
 328
 329 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe,
 330                               enum cmdq_cmd_type cmd_type,
 331                               struct hinic_cmd_buf *buf_in,
 332                               struct hinic_cmd_buf *buf_out, int wrapped,
 333                               enum hinic_ack_type ack_type,
 334                               enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
 335 {
 336         struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
 337         enum completion_format complete_format = COMPLETE_DIRECT;
 338
 339         switch (cmd_type) {
 340         case SYNC_CMD_SGE_RESP:
 341                 if (buf_out) {
 342                         complete_format = COMPLETE_SGE;
 343                         cmdq_set_completion(&wqe_lcmd->completion, buf_out);
 344                 }
 345                 break;
 346         case SYNC_CMD_DIRECT_RESP:
 347                 complete_format = COMPLETE_DIRECT;
 348                 wqe_lcmd->completion.direct_resp = 0;
 349                 break;
 350         case ASYNC_CMD:
 351                 complete_format = COMPLETE_DIRECT;
 352                 wqe_lcmd->completion.direct_resp = 0;
 353
 354                 wqe_lcmd->buf_desc.saved_async_buf = (u64)(buf_in);
 355                 break;
 356         }
 357
 358         cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd,
 359                               prod_idx, complete_format, DATA_SGE,
 360                               BUFDESC_LCMD_LEN);
 361
 362         cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in);
 363 }
 364
 365 static int cmdq_params_valid(struct hinic_cmd_buf *buf_in)
 366 {
 367         if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE) {
 368                 PMD_DRV_LOG(ERR, "Invalid CMDQ buffer size");
 369                 return -EINVAL;
 370         }
 371
 372         return 0;
 373 }
 374
 375 static int wait_cmdqs_enable(struct hinic_cmdqs *cmdqs)
 376 {
 377         unsigned long end;
 378
 379         end = jiffies + msecs_to_jiffies(WAIT_CMDQ_ENABLE_TIMEOUT);
 380         do {
 381                 if (cmdqs->status & HINIC_CMDQ_ENABLE)
 382                         return 0;
 383
 384         } while (time_before(jiffies, end));
 385
 386         return -EBUSY;
 387 }
 388
 389 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx,
 390                                 int errcode)
 391 {
 392         cmdq->errcode[prod_idx] = errcode;
 393 }
 394
 395 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq,
 396                                    struct hinic_cmdq_wqe *wqe)
 397 {
 398         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 399         struct hinic_cmdq_inline_wqe *inline_wqe;
 400         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 401         struct hinic_ctrl *ctrl;
 402         u32 header_info = be32_to_cpu(WQE_HEADER(wqe)->header_info);
 403         int buf_len = CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN);
 404         int wqe_size = cmdq_get_wqe_size(buf_len);
 405         u16 num_wqebbs;
 406
 407         if (wqe_size == WQE_LCMD_SIZE) {
 408                 wqe_lcmd = &wqe->wqe_lcmd;
 409                 ctrl = &wqe_lcmd->ctrl;
 410         } else {
 411                 inline_wqe = &wqe->inline_wqe;
 412                 wqe_scmd = &inline_wqe->wqe_scmd;
 413                 ctrl = &wqe_scmd->ctrl;
 414         }
 415
 416         /* clear HW busy bit */
 417         ctrl->ctrl_info = 0;
 418
 419         rte_wmb();      /* verify wqe is clear */
 420
 421         num_wqebbs = WQE_NUM_WQEBBS(wqe_size, cmdq->wq);
 422         hinic_put_wqe(cmdq->wq, num_wqebbs);
 423 }
 424
 425 static int hinic_set_cmdq_ctxts(struct hinic_hwdev *hwdev)
 426 {
 427         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 428         struct hinic_cmdq_ctxt *cmdq_ctxt;
 429         enum hinic_cmdq_type cmdq_type;
 430         u16 in_size;
 431         int err;
 432
 433         cmdq_type = HINIC_CMDQ_SYNC;
 434         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 435                 cmdq_ctxt = &cmdqs->cmdq[cmdq_type].cmdq_ctxt;
 436                 cmdq_ctxt->resp_aeq_num = HINIC_AEQ1;
 437                 in_size = sizeof(*cmdq_ctxt);
 438                 err = hinic_msg_to_mgmt_sync(hwdev, HINIC_MOD_COMM,
 439                                              HINIC_MGMT_CMD_CMDQ_CTXT_SET,
 440                                              cmdq_ctxt, in_size, NULL,
 441                                              NULL, 0);
 442                 if (err) {
 443                         if (err == HINIC_MBOX_PF_BUSY_ACTIVE_FW ||
 444                                 err == HINIC_DEV_BUSY_ACTIVE_FW) {
 445                                 cmdqs->status |= HINIC_CMDQ_SET_FAIL;
 446                                 PMD_DRV_LOG(ERR, "PF or VF fw is hot active");
 447                         }
 448                         PMD_DRV_LOG(ERR, "Set cmdq ctxt failed, err: %d", err);
 449                         return -EFAULT;
 450                 }
 451         }
 452
 453         cmdqs->status &= ~HINIC_CMDQ_SET_FAIL;
 454         cmdqs->status |= HINIC_CMDQ_ENABLE;
 455
 456         return 0;
 457 }
 458
 459 void hinic_comm_cmdqs_free(struct hinic_hwdev *hwdev)
 460 {
 461         hinic_cmdqs_free(hwdev);
 462 }
 463
 464 int hinic_reinit_cmdq_ctxts(struct hinic_hwdev *hwdev)
 465 {
 466         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 467         enum hinic_cmdq_type cmdq_type;
 468
 469         cmdq_type = HINIC_CMDQ_SYNC;
 470         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 471                 cmdqs->cmdq[cmdq_type].wrapped = 1;
 472                 hinic_wq_wqe_pg_clear(cmdqs->cmdq[cmdq_type].wq);
 473         }
 474
 475         return hinic_set_cmdq_ctxts(hwdev);
 476 }
 477
 478 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_hwdev *hwdev,
 479                      struct hinic_wq *wq, enum hinic_cmdq_type q_type)
 480 {
 481         void __iomem *db_base;
 482         int err = 0;
 483         size_t errcode_size;
 484         size_t cmd_infos_size;
 485
 486         cmdq->wq = wq;
 487         cmdq->cmdq_type = q_type;
 488         cmdq->wrapped = 1;
 489
 490         spin_lock_init(&cmdq->cmdq_lock);
 491
 492         errcode_size = wq->q_depth * sizeof(*cmdq->errcode);
 493         cmdq->errcode = kzalloc(errcode_size, GFP_KERNEL);
 494         if (!cmdq->errcode) {
 495                 PMD_DRV_LOG(ERR, "Allocate errcode for cmdq failed");
 496                 spin_lock_deinit(&cmdq->cmdq_lock);
 497                 return -ENOMEM;
 498         }
 499
 500         cmd_infos_size = wq->q_depth * sizeof(*cmdq->cmd_infos);
 501         cmdq->cmd_infos = kzalloc(cmd_infos_size, GFP_KERNEL);
 502         if (!cmdq->cmd_infos) {
 503                 PMD_DRV_LOG(ERR, "Allocate errcode for cmdq failed");
 504                 err = -ENOMEM;
 505                 goto cmd_infos_err;
 506         }
 507
 508         err = hinic_alloc_db_addr(hwdev, &db_base);
 509         if (err)
 510                 goto alloc_db_err;
 511
 512         cmdq->db_base = (u8 *)db_base;
 513         return 0;
 514
 515 alloc_db_err:
 516         kfree(cmdq->cmd_infos);
 517
 518 cmd_infos_err:
 519         kfree(cmdq->errcode);
 520         spin_lock_deinit(&cmdq->cmdq_lock);
 521
 522         return err;
 523 }
 524
 525 static void free_cmdq(struct hinic_hwdev *hwdev, struct hinic_cmdq *cmdq)
 526 {
 527         hinic_free_db_addr(hwdev, cmdq->db_base);
 528         kfree(cmdq->cmd_infos);
 529         kfree(cmdq->errcode);
 530         spin_lock_deinit(&cmdq->cmdq_lock);
 531 }
 532
 533 static int hinic_cmdqs_init(struct hinic_hwdev *hwdev)
 534 {
 535         struct hinic_cmdqs *cmdqs;
 536         struct hinic_cmdq_ctxt *cmdq_ctxt;
 537         enum hinic_cmdq_type type, cmdq_type;
 538         size_t saved_wqs_size;
 539         int err;
 540
 541         cmdqs = kzalloc(sizeof(*cmdqs), GFP_KERNEL);
 542         if (!cmdqs)
 543                 return -ENOMEM;
 544
 545         hwdev->cmdqs = cmdqs;
 546         cmdqs->hwdev = hwdev;
 547
 548         saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq);
 549         cmdqs->saved_wqs = kzalloc(saved_wqs_size, GFP_KERNEL);
 550         if (!cmdqs->saved_wqs) {
 551                 PMD_DRV_LOG(ERR, "Allocate saved wqs failed");
 552                 err = -ENOMEM;
 553                 goto alloc_wqs_err;
 554         }
 555
 556         cmdqs->cmd_buf_pool = dma_pool_create("hinic_cmdq", hwdev,
 557                                               HINIC_CMDQ_BUF_SIZE,
 558                                               HINIC_CMDQ_BUF_SIZE, 0ULL);
 559         if (!cmdqs->cmd_buf_pool) {
 560                 PMD_DRV_LOG(ERR, "Create cmdq buffer pool failed");
 561                 err = -ENOMEM;
 562                 goto pool_create_err;
 563         }
 564
 565         err = hinic_cmdq_alloc(cmdqs->saved_wqs, hwdev,
 566                                HINIC_MAX_CMDQ_TYPES, HINIC_CMDQ_WQ_BUF_SIZE,
 567                                CMDQ_WQEBB_SHIFT, HINIC_CMDQ_DEPTH);
 568         if (err) {
 569                 PMD_DRV_LOG(ERR, "Allocate cmdq failed");
 570                 goto cmdq_alloc_err;
 571         }
 572
 573         cmdq_type = HINIC_CMDQ_SYNC;
 574         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 575                 err = init_cmdq(&cmdqs->cmdq[cmdq_type], hwdev,
 576                                 &cmdqs->saved_wqs[cmdq_type], cmdq_type);
 577                 if (err) {
 578                         PMD_DRV_LOG(ERR, "Initialize cmdq failed");
 579                         goto init_cmdq_err;
 580                 }
 581
 582                 cmdq_ctxt = &cmdqs->cmdq[cmdq_type].cmdq_ctxt;
 583                 cmdq_init_queue_ctxt(&cmdqs->cmdq[cmdq_type], cmdq_ctxt);
 584         }
 585
 586         err = hinic_set_cmdq_ctxts(hwdev);
 587         if (err)
 588                 goto init_cmdq_err;
 589
 590         return 0;
 591
 592 init_cmdq_err:
 593         type = HINIC_CMDQ_SYNC;
 594         for ( ; type < cmdq_type; type++)
 595                 free_cmdq(hwdev, &cmdqs->cmdq[type]);
 596
 597         hinic_cmdq_free(hwdev, cmdqs->saved_wqs, HINIC_MAX_CMDQ_TYPES);
 598
 599 cmdq_alloc_err:
 600         dma_pool_destroy(cmdqs->cmd_buf_pool);
 601
 602 pool_create_err:
 603         kfree(cmdqs->saved_wqs);
 604
 605 alloc_wqs_err:
 606         kfree(cmdqs);
 607
 608         return err;
 609 }
 610
 611 static void hinic_cmdqs_free(struct hinic_hwdev *hwdev)
 612 {
 613         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 614         enum hinic_cmdq_type cmdq_type = HINIC_CMDQ_SYNC;
 615
 616         cmdqs->status &= ~HINIC_CMDQ_ENABLE;
 617
 618         for ( ; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++)
 619                 free_cmdq(cmdqs->hwdev, &cmdqs->cmdq[cmdq_type]);
 620
 621         hinic_cmdq_free(hwdev, cmdqs->saved_wqs,
 622                         HINIC_MAX_CMDQ_TYPES);
 623
 624         dma_pool_destroy(cmdqs->cmd_buf_pool);
 625
 626         kfree(cmdqs->saved_wqs);
 627
 628         kfree(cmdqs);
 629 }
 630
 631 static int hinic_set_cmdq_depth(struct hinic_hwdev *hwdev, u16 cmdq_depth)
 632 {
 633         struct hinic_root_ctxt root_ctxt;
 634
 635         memset(&root_ctxt, 0, sizeof(root_ctxt));
 636         root_ctxt.mgmt_msg_head.resp_aeq_num = HINIC_AEQ1;
 637         root_ctxt.func_idx = hinic_global_func_id(hwdev);
 638         root_ctxt.ppf_idx = hinic_ppf_idx(hwdev);
 639         root_ctxt.set_cmdq_depth = 1;
 640         root_ctxt.cmdq_depth = (u8)ilog2(cmdq_depth);
 641         return hinic_msg_to_mgmt_sync(hwdev, HINIC_MOD_COMM,
 642                                       HINIC_MGMT_CMD_VAT_SET,
 643                                       &root_ctxt, sizeof(root_ctxt),
 644                                       NULL, NULL, 0);
 645 }
 646
 647 int hinic_comm_cmdqs_init(struct hinic_hwdev *hwdev)
 648 {
 649         int err;
 650
 651         err = hinic_cmdqs_init(hwdev);
 652         if (err) {
 653                 PMD_DRV_LOG(ERR, "Init cmd queues failed");
 654                 return err;
 655         }
 656
 657         err = hinic_set_cmdq_depth(hwdev, HINIC_CMDQ_DEPTH);
 658         if (err) {
 659                 PMD_DRV_LOG(ERR, "Set cmdq depth failed");
 660                 goto set_cmdq_depth_err;
 661         }
 662
 663         return 0;
 664
 665 set_cmdq_depth_err:
 666         hinic_cmdqs_free(hwdev);
 667
 668         return err;
 669 }
 670
 671 static void cmdq_init_queue_ctxt(struct hinic_cmdq *cmdq,
 672                                  struct hinic_cmdq_ctxt *cmdq_ctxt)
 673 {
 674         struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)cmdq_to_cmdqs(cmdq);
 675         struct hinic_hwdev *hwdev = cmdqs->hwdev;
 676         struct hinic_wq *wq = cmdq->wq;
 677         struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info;
 678         u64 wq_first_page_paddr, pfn;
 679
 680         u16 start_ci = (u16)(wq->cons_idx);
 681
 682         /* The data in the HW is in Big Endian Format */
 683         wq_first_page_paddr = wq->queue_buf_paddr;
 684
 685         pfn = CMDQ_PFN(wq_first_page_paddr, HINIC_PAGE_SIZE);
 686         ctxt_info->curr_wqe_page_pfn =
 687                 CMDQ_CTXT_PAGE_INFO_SET(1, HW_BUSY_BIT) |
 688                 CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN)      |
 689                 CMDQ_CTXT_PAGE_INFO_SET(0, CEQ_ARM)     |
 690                 CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) |
 691                 CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN);
 692
 693         ctxt_info->wq_block_pfn = CMDQ_CTXT_BLOCK_INFO_SET(start_ci, CI) |
 694                                 CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN);
 695         cmdq_ctxt->func_idx = HINIC_HWIF_GLOBAL_IDX(hwdev->hwif);
 696         cmdq_ctxt->ppf_idx  = HINIC_HWIF_PPF_IDX(hwdev->hwif);
 697         cmdq_ctxt->cmdq_id  = cmdq->cmdq_type;
 698 }
 699
 700 static int hinic_cmdq_poll_msg(struct hinic_cmdq *cmdq, u32 timeout)
 701 {
 702         struct hinic_cmdq_wqe *wqe;
 703         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 704         struct hinic_ctrl *ctrl;
 705         struct hinic_cmdq_cmd_info *cmd_info;
 706         u32 status_info, ctrl_info;
 707         u16 ci;
 708         int errcode;
 709         unsigned long end;
 710         int done = 0;
 711         int rc = 0;
 712
 713         wqe = hinic_read_wqe(cmdq->wq, 1, &ci);
 714         if (wqe == NULL) {
 715                 PMD_DRV_LOG(ERR, "No outstanding cmdq msg");
 716                 return -EINVAL;
 717         }
 718
 719         cmd_info = &cmdq->cmd_infos[ci];
 720         /* this cmd has not been filled and send to hw, or get TMO msg ack*/
 721         if (cmd_info->cmd_type == HINIC_CMD_TYPE_NONE) {
 722                 PMD_DRV_LOG(ERR, "Cmdq msg has not been filled and send to hw, or get TMO msg ack. cmdq ci: %u",
 723                             ci);
 724                 return -EINVAL;
 725         }
 726
 727         /* only arm bit is using scmd wqe, the wqe is lcmd */
 728         wqe_lcmd = &wqe->wqe_lcmd;
 729         ctrl = &wqe_lcmd->ctrl;
 730         end = jiffies + msecs_to_jiffies(timeout);
 731         do {
 732                 ctrl_info = be32_to_cpu((ctrl)->ctrl_info);
 733                 if (WQE_COMPLETED(ctrl_info)) {
 734                         done = 1;
 735                         break;
 736                 }
 737
 738                 rte_delay_ms(1);
 739         } while (time_before(jiffies, end));
 740
 741         if (done) {
 742                 status_info = be32_to_cpu(wqe_lcmd->status.status_info);
 743                 errcode = WQE_ERRCODE_GET(status_info, VAL);
 744                 cmdq_update_errcode(cmdq, ci, errcode);
 745                 clear_wqe_complete_bit(cmdq, wqe);
 746                 rc = 0;
 747         } else {
 748                 PMD_DRV_LOG(ERR, "Poll cmdq msg time out, ci: %u", ci);
 749                 rc = -ETIMEDOUT;
 750         }
 751
 752         /* set this cmd invalid */
 753         cmd_info->cmd_type = HINIC_CMD_TYPE_NONE;
 754
 755         return rc;
 756 }
 757
 758 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,
 759                                      enum hinic_ack_type ack_type,
 760                                      enum hinic_mod_type mod, u8 cmd,
 761                                      struct hinic_cmd_buf *buf_in,
 762                                      u64 *out_param, u32 timeout)
 763 {
 764         struct hinic_wq *wq = cmdq->wq;
 765         struct hinic_cmdq_wqe *curr_wqe, wqe;
 766         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 767         u16 curr_prod_idx, next_prod_idx, num_wqebbs;
 768         int wrapped;
 769         u32 timeo, wqe_size;
 770         int err;
 771
 772         wqe_size = cmdq_wqe_size(WQE_LCMD_TYPE);
 773         num_wqebbs = WQE_NUM_WQEBBS(wqe_size, wq);
 774
 775         /* Keep wrapped and doorbell index correct. */
 776         spin_lock(&cmdq->cmdq_lock);
 777
 778         curr_wqe = hinic_get_wqe(cmdq->wq, num_wqebbs, &curr_prod_idx);
 779         if (!curr_wqe) {
 780                 err = -EBUSY;
 781                 goto cmdq_unlock;
 782         }
 783
 784         memset(&wqe, 0, sizeof(wqe));
 785         wrapped = cmdq->wrapped;
 786
 787         next_prod_idx = curr_prod_idx + num_wqebbs;
 788         if (next_prod_idx >= wq->q_depth) {
 789                 cmdq->wrapped = !cmdq->wrapped;
 790                 next_prod_idx -= wq->q_depth;
 791         }
 792
 793         cmdq_set_lcmd_wqe(&wqe, SYNC_CMD_DIRECT_RESP, buf_in, NULL,
 794                           wrapped, ack_type, mod, cmd, curr_prod_idx);
 795
 796         /* The data that is written to HW should be in Big Endian Format */
 797         hinic_cpu_to_be32(&wqe, wqe_size);
 798
 799         /* CMDQ WQE is not shadow, therefore wqe will be written to wq */
 800         cmdq_wqe_fill(curr_wqe, &wqe);
 801
 802         cmdq->cmd_infos[curr_prod_idx].cmd_type = HINIC_CMD_TYPE_NORMAL;
 803
 804         cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
 805
 806         timeo = msecs_to_jiffies(timeout ? timeout : CMDQ_CMD_TIMEOUT);
 807         err = hinic_cmdq_poll_msg(cmdq, timeo);
 808         if (err) {
 809                 PMD_DRV_LOG(ERR, "Cmdq poll msg ack failed, prod idx: 0x%x",
 810                         curr_prod_idx);
 811                 err = -ETIMEDOUT;
 812                 goto cmdq_unlock;
 813         }
 814
 815         rte_smp_rmb();  /* read error code after completion */
 816
 817         if (out_param) {
 818                 wqe_lcmd = &curr_wqe->wqe_lcmd;
 819                 *out_param = cpu_to_be64(wqe_lcmd->completion.direct_resp);
 820         }
 821
 822         if (cmdq->errcode[curr_prod_idx] > 1) {
 823                 err = cmdq->errcode[curr_prod_idx];
 824                 goto cmdq_unlock;
 825         }
 826
 827 cmdq_unlock:
 828         spin_unlock(&cmdq->cmdq_lock);
 829
 830         return err;
 831 }
 832
 833 int hinic_cmdq_direct_resp(void *hwdev, enum hinic_ack_type ack_type,
 834                            enum hinic_mod_type mod, u8 cmd,
 835                            struct hinic_cmd_buf *buf_in,
 836                            u64 *out_param, u32 timeout)
 837 {
 838         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 839         int err = cmdq_params_valid(buf_in);
 840
 841         if (err) {
 842                 PMD_DRV_LOG(ERR, "Invalid CMDQ parameters");
 843                 return err;
 844         }
 845
 846         err = wait_cmdqs_enable(cmdqs);
 847         if (err) {
 848                 PMD_DRV_LOG(ERR, "Cmdq is disable");
 849                 return err;
 850         }
 851
 852         return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC],
 853                                          ack_type, mod, cmd, buf_in,
 854                                          out_param, timeout);
 855 }