drivers/net/hinic/base/hinic_pmd_cmdq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include "hinic_compat.h"
   6 #include "hinic_pmd_hwdev.h"
   7 #include "hinic_pmd_hwif.h"
   8 #include "hinic_pmd_wq.h"
   9 #include "hinic_pmd_mgmt.h"
  10 #include "hinic_pmd_mbox.h"
  11 #include "hinic_pmd_cmdq.h"
  12
  13 #define CMDQ_CMD_TIMEOUT                                5000 /* millisecond */
  14
  15 #define UPPER_8_BITS(data)                              (((data) >> 8) & 0xFF)
  16 #define LOWER_8_BITS(data)                              ((data) & 0xFF)
  17
  18 #define CMDQ_DB_INFO_HI_PROD_IDX_SHIFT                  0
  19 #define CMDQ_DB_INFO_QUEUE_TYPE_SHIFT                   23
  20 #define CMDQ_DB_INFO_CMDQ_TYPE_SHIFT                    24
  21 #define CMDQ_DB_INFO_SRC_TYPE_SHIFT                     27
  22
  23 #define CMDQ_DB_INFO_HI_PROD_IDX_MASK                   0xFFU
  24 #define CMDQ_DB_INFO_QUEUE_TYPE_MASK                    0x1U
  25 #define CMDQ_DB_INFO_CMDQ_TYPE_MASK                     0x7U
  26 #define CMDQ_DB_INFO_SRC_TYPE_MASK                      0x1FU
  27
  28 #define CMDQ_DB_INFO_SET(val, member)           \
  29         (((val) & CMDQ_DB_INFO_##member##_MASK) <<      \
  30                 CMDQ_DB_INFO_##member##_SHIFT)
  31
  32 #define CMDQ_CTRL_PI_SHIFT                              0
  33 #define CMDQ_CTRL_CMD_SHIFT                             16
  34 #define CMDQ_CTRL_MOD_SHIFT                             24
  35 #define CMDQ_CTRL_ACK_TYPE_SHIFT                        29
  36 #define CMDQ_CTRL_HW_BUSY_BIT_SHIFT                     31
  37
  38 #define CMDQ_CTRL_PI_MASK                               0xFFFFU
  39 #define CMDQ_CTRL_CMD_MASK                              0xFFU
  40 #define CMDQ_CTRL_MOD_MASK                              0x1FU
  41 #define CMDQ_CTRL_ACK_TYPE_MASK                         0x3U
  42 #define CMDQ_CTRL_HW_BUSY_BIT_MASK                      0x1U
  43
  44 #define CMDQ_CTRL_SET(val, member)              \
  45         (((val) & CMDQ_CTRL_##member##_MASK) << CMDQ_CTRL_##member##_SHIFT)
  46
  47 #define CMDQ_CTRL_GET(val, member)              \
  48         (((val) >> CMDQ_CTRL_##member##_SHIFT) & CMDQ_CTRL_##member##_MASK)
  49
  50 #define CMDQ_WQE_HEADER_BUFDESC_LEN_SHIFT               0
  51 #define CMDQ_WQE_HEADER_COMPLETE_FMT_SHIFT              15
  52 #define CMDQ_WQE_HEADER_DATA_FMT_SHIFT                  22
  53 #define CMDQ_WQE_HEADER_COMPLETE_REQ_SHIFT              23
  54 #define CMDQ_WQE_HEADER_COMPLETE_SECT_LEN_SHIFT         27
  55 #define CMDQ_WQE_HEADER_CTRL_LEN_SHIFT                  29
  56 #define CMDQ_WQE_HEADER_HW_BUSY_BIT_SHIFT               31
  57
  58 #define CMDQ_WQE_HEADER_BUFDESC_LEN_MASK                0xFFU
  59 #define CMDQ_WQE_HEADER_COMPLETE_FMT_MASK               0x1U
  60 #define CMDQ_WQE_HEADER_DATA_FMT_MASK                   0x1U
  61 #define CMDQ_WQE_HEADER_COMPLETE_REQ_MASK               0x1U
  62 #define CMDQ_WQE_HEADER_COMPLETE_SECT_LEN_MASK          0x3U
  63 #define CMDQ_WQE_HEADER_CTRL_LEN_MASK                   0x3U
  64 #define CMDQ_WQE_HEADER_HW_BUSY_BIT_MASK                0x1U
  65
  66 #define CMDQ_WQE_HEADER_SET(val, member)        \
  67         (((val) & CMDQ_WQE_HEADER_##member##_MASK) <<   \
  68                 CMDQ_WQE_HEADER_##member##_SHIFT)
  69
  70 #define CMDQ_WQE_HEADER_GET(val, member)        \
  71         (((val) >> CMDQ_WQE_HEADER_##member##_SHIFT) &  \
  72                 CMDQ_WQE_HEADER_##member##_MASK)
  73
  74 #define CMDQ_CTXT_CURR_WQE_PAGE_PFN_SHIFT               0
  75 #define CMDQ_CTXT_EQ_ID_SHIFT                           56
  76 #define CMDQ_CTXT_CEQ_ARM_SHIFT                         61
  77 #define CMDQ_CTXT_CEQ_EN_SHIFT                          62
  78 #define CMDQ_CTXT_HW_BUSY_BIT_SHIFT                     63
  79
  80 #define CMDQ_CTXT_CURR_WQE_PAGE_PFN_MASK                0xFFFFFFFFFFFFF
  81 #define CMDQ_CTXT_EQ_ID_MASK                            0x1F
  82 #define CMDQ_CTXT_CEQ_ARM_MASK                          0x1
  83 #define CMDQ_CTXT_CEQ_EN_MASK                           0x1
  84 #define CMDQ_CTXT_HW_BUSY_BIT_MASK                      0x1
  85
  86 #define CMDQ_CTXT_PAGE_INFO_SET(val, member)            \
  87         (((u64)(val) & CMDQ_CTXT_##member##_MASK) << CMDQ_CTXT_##member##_SHIFT)
  88
  89 #define CMDQ_CTXT_PAGE_INFO_CLEAR(val, member)          \
  90         ((val) & (~((u64)CMDQ_CTXT_##member##_MASK <<   \
  91                 CMDQ_CTXT_##member##_SHIFT)))
  92
  93 #define CMDQ_CTXT_WQ_BLOCK_PFN_SHIFT                    0
  94 #define CMDQ_CTXT_CI_SHIFT                              52
  95
  96 #define CMDQ_CTXT_WQ_BLOCK_PFN_MASK                     0xFFFFFFFFFFFFF
  97 #define CMDQ_CTXT_CI_MASK                               0xFFF
  98
  99 #define CMDQ_CTXT_BLOCK_INFO_SET(val, member)           \
 100         (((u64)(val) & CMDQ_CTXT_##member##_MASK) << CMDQ_CTXT_##member##_SHIFT)
 101
 102 #define SAVED_DATA_ARM_SHIFT                    31
 103
 104 #define SAVED_DATA_ARM_MASK                     0x1U
 105
 106 #define SAVED_DATA_SET(val, member)             \
 107         (((val) & SAVED_DATA_##member##_MASK) << SAVED_DATA_##member##_SHIFT)
 108
 109 #define SAVED_DATA_CLEAR(val, member)           \
 110         ((val) & (~(SAVED_DATA_##member##_MASK << SAVED_DATA_##member##_SHIFT)))
 111
 112 #define WQE_ERRCODE_VAL_SHIFT                   20
 113
 114 #define WQE_ERRCODE_VAL_MASK                    0xF
 115
 116 #define WQE_ERRCODE_GET(val, member)            \
 117         (((val) >> WQE_ERRCODE_##member##_SHIFT) & WQE_ERRCODE_##member##_MASK)
 118
 119 #define WQE_COMPLETED(ctrl_info)        CMDQ_CTRL_GET(ctrl_info, HW_BUSY_BIT)
 120
 121 #define WQE_HEADER(wqe)         ((struct hinic_cmdq_header *)(wqe))
 122
 123 #define CMDQ_DB_PI_OFF(pi)              (((u16)LOWER_8_BITS(pi)) << 3)
 124
 125 #define CMDQ_DB_ADDR(db_base, pi)       \
 126         (((u8 *)(db_base) + HINIC_DB_OFF) + CMDQ_DB_PI_OFF(pi))
 127
 128 #define CMDQ_PFN(addr, page_size)       ((addr) >> (ilog2(page_size)))
 129
 130 #define FIRST_DATA_TO_WRITE_LAST        sizeof(u64)
 131
 132 #define WQE_LCMD_SIZE           64
 133 #define WQE_SCMD_SIZE           64
 134
 135 #define COMPLETE_LEN            3
 136
 137 #define CMDQ_WQEBB_SIZE         64
 138 #define CMDQ_WQEBB_SHIFT        6
 139
 140 #define CMDQ_WQE_SIZE           64
 141
 142 #define HINIC_CMDQ_WQ_BUF_SIZE  4096
 143
 144 #define WQE_NUM_WQEBBS(wqe_size, wq)    \
 145         ((u16)(ALIGN((u32)(wqe_size), (wq)->wqebb_size) / (wq)->wqebb_size))
 146
 147 #define cmdq_to_cmdqs(cmdq)     container_of((cmdq) - (cmdq)->cmdq_type, \
 148                                 struct hinic_cmdqs, cmdq[0])
 149
 150 #define WAIT_CMDQ_ENABLE_TIMEOUT        300
 151
 152
 153 static void cmdq_init_queue_ctxt(struct hinic_cmdq *cmdq,
 154                                  struct hinic_cmdq_ctxt *cmdq_ctxt);
 155 static void hinic_cmdqs_free(struct hinic_hwdev *hwdev);
 156
 157 bool hinic_cmdq_idle(struct hinic_cmdq *cmdq)
 158 {
 159         struct hinic_wq *wq = cmdq->wq;
 160
 161         return ((wq->delta) == wq->q_depth ? true : false);
 162 }
 163
 164 struct hinic_cmd_buf *hinic_alloc_cmd_buf(void *hwdev)
 165 {
 166         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 167         struct hinic_cmd_buf *cmd_buf;
 168
 169         cmd_buf = kzalloc(sizeof(*cmd_buf), GFP_KERNEL);
 170         if (!cmd_buf) {
 171                 PMD_DRV_LOG(ERR, "Allocate cmd buffer failed");
 172                 return NULL;
 173         }
 174
 175         cmd_buf->buf = pci_pool_alloc(cmdqs->cmd_buf_pool, &cmd_buf->dma_addr);
 176         if (!cmd_buf->buf) {
 177                 PMD_DRV_LOG(ERR, "Allocate cmd from the pool failed");
 178                 goto alloc_pci_buf_err;
 179         }
 180
 181         return cmd_buf;
 182
 183 alloc_pci_buf_err:
 184         kfree(cmd_buf);
 185         return NULL;
 186 }
 187
 188 void hinic_free_cmd_buf(void *hwdev, struct hinic_cmd_buf *cmd_buf)
 189 {
 190         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 191
 192         pci_pool_free(cmdqs->cmd_buf_pool, cmd_buf->buf, cmd_buf->dma_addr);
 193         kfree(cmd_buf);
 194 }
 195
 196 static u32 cmdq_wqe_size(enum cmdq_wqe_type wqe_type)
 197 {
 198         u32 wqe_size = 0;
 199
 200         switch (wqe_type) {
 201         case WQE_LCMD_TYPE:
 202                 wqe_size = WQE_LCMD_SIZE;
 203                 break;
 204         case WQE_SCMD_TYPE:
 205                 wqe_size = WQE_SCMD_SIZE;
 206                 break;
 207         }
 208
 209         return wqe_size;
 210 }
 211
 212 static int cmdq_get_wqe_size(enum bufdesc_len len)
 213 {
 214         int wqe_size = 0;
 215
 216         switch (len) {
 217         case BUFDESC_LCMD_LEN:
 218                 wqe_size = WQE_LCMD_SIZE;
 219                 break;
 220         case BUFDESC_SCMD_LEN:
 221                 wqe_size = WQE_SCMD_SIZE;
 222                 break;
 223         }
 224
 225         return wqe_size;
 226 }
 227
 228 static void cmdq_set_completion(struct hinic_cmdq_completion *complete,
 229                                         struct hinic_cmd_buf *buf_out)
 230 {
 231         struct hinic_sge_resp *sge_resp = &complete->sge_resp;
 232
 233         hinic_set_sge(&sge_resp->sge, buf_out->dma_addr,
 234                       HINIC_CMDQ_BUF_SIZE);
 235 }
 236
 237 static void cmdq_set_lcmd_bufdesc(struct hinic_cmdq_wqe_lcmd *wqe,
 238                                         struct hinic_cmd_buf *buf_in)
 239 {
 240         hinic_set_sge(&wqe->buf_desc.sge, buf_in->dma_addr, buf_in->size);
 241 }
 242
 243 static void cmdq_fill_db(struct hinic_cmdq_db *db,
 244                         enum hinic_cmdq_type cmdq_type, u16 prod_idx)
 245 {
 246         db->db_info = CMDQ_DB_INFO_SET(UPPER_8_BITS(prod_idx), HI_PROD_IDX) |
 247                         CMDQ_DB_INFO_SET(HINIC_DB_CMDQ_TYPE, QUEUE_TYPE) |
 248                         CMDQ_DB_INFO_SET(cmdq_type, CMDQ_TYPE)          |
 249                         CMDQ_DB_INFO_SET(HINIC_DB_SRC_CMDQ_TYPE, SRC_TYPE);
 250 }
 251
 252 static void cmdq_set_db(struct hinic_cmdq *cmdq,
 253                         enum hinic_cmdq_type cmdq_type, u16 prod_idx)
 254 {
 255         struct hinic_cmdq_db db;
 256
 257         cmdq_fill_db(&db, cmdq_type, prod_idx);
 258
 259         /* The data that is written to HW should be in Big Endian Format */
 260         db.db_info = cpu_to_be32(db.db_info);
 261
 262         rte_wmb();      /* write all before the doorbell */
 263
 264         writel(db.db_info, CMDQ_DB_ADDR(cmdq->db_base, prod_idx));
 265 }
 266
 267 static void cmdq_wqe_fill(void *dst, void *src)
 268 {
 269         memcpy((u8 *)dst + FIRST_DATA_TO_WRITE_LAST,
 270                (u8 *)src + FIRST_DATA_TO_WRITE_LAST,
 271                CMDQ_WQE_SIZE - FIRST_DATA_TO_WRITE_LAST);
 272
 273         rte_wmb();/* The first 8 bytes should be written last */
 274
 275         *(u64 *)dst = *(u64 *)src;
 276 }
 277
 278 static void cmdq_prepare_wqe_ctrl(struct hinic_cmdq_wqe *wqe, int wrapped,
 279                                   enum hinic_ack_type ack_type,
 280                                   enum hinic_mod_type mod, u8 cmd, u16 prod_idx,
 281                                   enum completion_format complete_format,
 282                                   enum data_format local_data_format,
 283                                   enum bufdesc_len buf_len)
 284 {
 285         struct hinic_ctrl *ctrl;
 286         enum ctrl_sect_len ctrl_len;
 287         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 288         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 289         u32 saved_data = WQE_HEADER(wqe)->saved_data;
 290
 291         if (local_data_format == DATA_SGE) {
 292                 wqe_lcmd = &wqe->wqe_lcmd;
 293
 294                 wqe_lcmd->status.status_info = 0;
 295                 ctrl = &wqe_lcmd->ctrl;
 296                 ctrl_len = CTRL_SECT_LEN;
 297         } else {
 298                 wqe_scmd = &wqe->inline_wqe.wqe_scmd;
 299
 300                 wqe_scmd->status.status_info = 0;
 301                 ctrl = &wqe_scmd->ctrl;
 302                 ctrl_len = CTRL_DIRECT_SECT_LEN;
 303         }
 304
 305         ctrl->ctrl_info = CMDQ_CTRL_SET(prod_idx, PI)           |
 306                         CMDQ_CTRL_SET(cmd, CMD)                 |
 307                         CMDQ_CTRL_SET(mod, MOD)                 |
 308                         CMDQ_CTRL_SET(ack_type, ACK_TYPE);
 309
 310         WQE_HEADER(wqe)->header_info =
 311                 CMDQ_WQE_HEADER_SET(buf_len, BUFDESC_LEN) |
 312                 CMDQ_WQE_HEADER_SET(complete_format, COMPLETE_FMT) |
 313                 CMDQ_WQE_HEADER_SET(local_data_format, DATA_FMT)        |
 314                 CMDQ_WQE_HEADER_SET(CEQ_SET, COMPLETE_REQ)      |
 315                 CMDQ_WQE_HEADER_SET(COMPLETE_LEN, COMPLETE_SECT_LEN) |
 316                 CMDQ_WQE_HEADER_SET(ctrl_len, CTRL_LEN)         |
 317                 CMDQ_WQE_HEADER_SET((u32)wrapped, HW_BUSY_BIT);
 318
 319         if (cmd == CMDQ_SET_ARM_CMD && mod == HINIC_MOD_COMM) {
 320                 saved_data &= SAVED_DATA_CLEAR(saved_data, ARM);
 321                 WQE_HEADER(wqe)->saved_data = saved_data |
 322                                                 SAVED_DATA_SET(1, ARM);
 323         } else {
 324                 saved_data &= SAVED_DATA_CLEAR(saved_data, ARM);
 325                 WQE_HEADER(wqe)->saved_data = saved_data;
 326         }
 327 }
 328
 329 static void cmdq_set_lcmd_wqe(struct hinic_cmdq_wqe *wqe,
 330                               enum cmdq_cmd_type cmd_type,
 331                               struct hinic_cmd_buf *buf_in,
 332                               struct hinic_cmd_buf *buf_out, int wrapped,
 333                               enum hinic_ack_type ack_type,
 334                               enum hinic_mod_type mod, u8 cmd, u16 prod_idx)
 335 {
 336         struct hinic_cmdq_wqe_lcmd *wqe_lcmd = &wqe->wqe_lcmd;
 337         enum completion_format complete_format = COMPLETE_DIRECT;
 338
 339         switch (cmd_type) {
 340         case SYNC_CMD_SGE_RESP:
 341                 if (buf_out) {
 342                         complete_format = COMPLETE_SGE;
 343                         cmdq_set_completion(&wqe_lcmd->completion, buf_out);
 344                 }
 345                 break;
 346         case SYNC_CMD_DIRECT_RESP:
 347                 complete_format = COMPLETE_DIRECT;
 348                 wqe_lcmd->completion.direct_resp = 0;
 349                 break;
 350         case ASYNC_CMD:
 351                 complete_format = COMPLETE_DIRECT;
 352                 wqe_lcmd->completion.direct_resp = 0;
 353
 354                 wqe_lcmd->buf_desc.saved_async_buf = (u64)(buf_in);
 355                 break;
 356         }
 357
 358         cmdq_prepare_wqe_ctrl(wqe, wrapped, ack_type, mod, cmd,
 359                               prod_idx, complete_format, DATA_SGE,
 360                               BUFDESC_LCMD_LEN);
 361
 362         cmdq_set_lcmd_bufdesc(wqe_lcmd, buf_in);
 363 }
 364
 365 static int cmdq_params_valid(struct hinic_cmd_buf *buf_in)
 366 {
 367         if (buf_in->size > HINIC_CMDQ_MAX_DATA_SIZE) {
 368                 PMD_DRV_LOG(ERR, "Invalid CMDQ buffer size");
 369                 return -EINVAL;
 370         }
 371
 372         return 0;
 373 }
 374
 375 static int wait_cmdqs_enable(struct hinic_cmdqs *cmdqs)
 376 {
 377         unsigned long end;
 378
 379         end = jiffies + msecs_to_jiffies(WAIT_CMDQ_ENABLE_TIMEOUT);
 380         do {
 381                 if (cmdqs->status & HINIC_CMDQ_ENABLE)
 382                         return 0;
 383
 384         } while (time_before(jiffies, end));
 385
 386         return -EBUSY;
 387 }
 388
 389 static void cmdq_update_errcode(struct hinic_cmdq *cmdq, u16 prod_idx,
 390                                 int errcode)
 391 {
 392         cmdq->errcode[prod_idx] = errcode;
 393 }
 394
 395 static void clear_wqe_complete_bit(struct hinic_cmdq *cmdq,
 396                                    struct hinic_cmdq_wqe *wqe)
 397 {
 398         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 399         struct hinic_cmdq_inline_wqe *inline_wqe;
 400         struct hinic_cmdq_wqe_scmd *wqe_scmd;
 401         struct hinic_ctrl *ctrl;
 402         u32 header_info = be32_to_cpu(WQE_HEADER(wqe)->header_info);
 403         int buf_len = CMDQ_WQE_HEADER_GET(header_info, BUFDESC_LEN);
 404         int wqe_size = cmdq_get_wqe_size(buf_len);
 405         u16 num_wqebbs;
 406
 407         if (wqe_size == WQE_LCMD_SIZE) {
 408                 wqe_lcmd = &wqe->wqe_lcmd;
 409                 ctrl = &wqe_lcmd->ctrl;
 410         } else {
 411                 inline_wqe = &wqe->inline_wqe;
 412                 wqe_scmd = &inline_wqe->wqe_scmd;
 413                 ctrl = &wqe_scmd->ctrl;
 414         }
 415
 416         /* clear HW busy bit */
 417         ctrl->ctrl_info = 0;
 418
 419         rte_wmb();      /* verify wqe is clear */
 420
 421         num_wqebbs = WQE_NUM_WQEBBS(wqe_size, cmdq->wq);
 422         hinic_put_wqe(cmdq->wq, num_wqebbs);
 423 }
 424
 425 static int hinic_set_cmdq_ctxts(struct hinic_hwdev *hwdev)
 426 {
 427         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 428         struct hinic_cmdq_ctxt *cmdq_ctxt;
 429         struct hinic_cmdq_ctxt cmdq_ctxt_out;
 430         enum hinic_cmdq_type cmdq_type;
 431         u16 out_size = sizeof(cmdq_ctxt_out);
 432         u16 in_size;
 433         int err;
 434
 435         cmdq_type = HINIC_CMDQ_SYNC;
 436         memset(&cmdq_ctxt_out, 0, out_size);
 437         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 438                 cmdq_ctxt = &cmdqs->cmdq[cmdq_type].cmdq_ctxt;
 439                 cmdq_ctxt->resp_aeq_num = HINIC_AEQ1;
 440                 in_size = sizeof(*cmdq_ctxt);
 441                 err = hinic_msg_to_mgmt_sync(hwdev, HINIC_MOD_COMM,
 442                                              HINIC_MGMT_CMD_CMDQ_CTXT_SET,
 443                                              cmdq_ctxt, in_size, &cmdq_ctxt_out,
 444                                              &out_size, 0);
 445                 if (err || !out_size || cmdq_ctxt_out.status) {
 446                         if (err == HINIC_MBOX_PF_BUSY_ACTIVE_FW ||
 447                                 err == HINIC_DEV_BUSY_ACTIVE_FW) {
 448                                 cmdqs->status |= HINIC_CMDQ_SET_FAIL;
 449                                 PMD_DRV_LOG(ERR, "PF or VF fw is hot active");
 450                         }
 451                         PMD_DRV_LOG(ERR, "Set cmdq ctxt failed, err: %d, status: 0x%x, out_size: 0x%x",
 452                                 err, cmdq_ctxt_out.status, out_size);
 453                         return -EIO;
 454                 }
 455         }
 456
 457         cmdqs->status &= ~HINIC_CMDQ_SET_FAIL;
 458         cmdqs->status |= HINIC_CMDQ_ENABLE;
 459
 460         return 0;
 461 }
 462
 463 void hinic_comm_cmdqs_free(struct hinic_hwdev *hwdev)
 464 {
 465         hinic_cmdqs_free(hwdev);
 466 }
 467
 468 int hinic_reinit_cmdq_ctxts(struct hinic_hwdev *hwdev)
 469 {
 470         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 471         enum hinic_cmdq_type cmdq_type;
 472
 473         cmdq_type = HINIC_CMDQ_SYNC;
 474         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 475                 cmdqs->cmdq[cmdq_type].wrapped = 1;
 476                 hinic_wq_wqe_pg_clear(cmdqs->cmdq[cmdq_type].wq);
 477         }
 478
 479         return hinic_set_cmdq_ctxts(hwdev);
 480 }
 481
 482 static int init_cmdq(struct hinic_cmdq *cmdq, struct hinic_hwdev *hwdev,
 483                      struct hinic_wq *wq, enum hinic_cmdq_type q_type)
 484 {
 485         void __iomem *db_base;
 486         int err = 0;
 487         size_t errcode_size;
 488         size_t cmd_infos_size;
 489
 490         cmdq->wq = wq;
 491         cmdq->cmdq_type = q_type;
 492         cmdq->wrapped = 1;
 493
 494         spin_lock_init(&cmdq->cmdq_lock);
 495
 496         errcode_size = wq->q_depth * sizeof(*cmdq->errcode);
 497         cmdq->errcode = kzalloc(errcode_size, GFP_KERNEL);
 498         if (!cmdq->errcode) {
 499                 PMD_DRV_LOG(ERR, "Allocate errcode for cmdq failed");
 500                 spin_lock_deinit(&cmdq->cmdq_lock);
 501                 return -ENOMEM;
 502         }
 503
 504         cmd_infos_size = wq->q_depth * sizeof(*cmdq->cmd_infos);
 505         cmdq->cmd_infos = kzalloc(cmd_infos_size, GFP_KERNEL);
 506         if (!cmdq->cmd_infos) {
 507                 PMD_DRV_LOG(ERR, "Allocate errcode for cmdq failed");
 508                 err = -ENOMEM;
 509                 goto cmd_infos_err;
 510         }
 511
 512         err = hinic_alloc_db_addr(hwdev, &db_base);
 513         if (err)
 514                 goto alloc_db_err;
 515
 516         cmdq->db_base = (u8 *)db_base;
 517         return 0;
 518
 519 alloc_db_err:
 520         kfree(cmdq->cmd_infos);
 521
 522 cmd_infos_err:
 523         kfree(cmdq->errcode);
 524         spin_lock_deinit(&cmdq->cmdq_lock);
 525
 526         return err;
 527 }
 528
 529 static void free_cmdq(struct hinic_hwdev *hwdev, struct hinic_cmdq *cmdq)
 530 {
 531         hinic_free_db_addr(hwdev, cmdq->db_base);
 532         kfree(cmdq->cmd_infos);
 533         kfree(cmdq->errcode);
 534         spin_lock_deinit(&cmdq->cmdq_lock);
 535 }
 536
 537 static int hinic_cmdqs_init(struct hinic_hwdev *hwdev)
 538 {
 539         struct hinic_cmdqs *cmdqs;
 540         struct hinic_cmdq_ctxt *cmdq_ctxt;
 541         enum hinic_cmdq_type type, cmdq_type;
 542         size_t saved_wqs_size;
 543         int err;
 544
 545         cmdqs = kzalloc(sizeof(*cmdqs), GFP_KERNEL);
 546         if (!cmdqs)
 547                 return -ENOMEM;
 548
 549         hwdev->cmdqs = cmdqs;
 550         cmdqs->hwdev = hwdev;
 551
 552         saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq);
 553         cmdqs->saved_wqs = kzalloc(saved_wqs_size, GFP_KERNEL);
 554         if (!cmdqs->saved_wqs) {
 555                 PMD_DRV_LOG(ERR, "Allocate saved wqs failed");
 556                 err = -ENOMEM;
 557                 goto alloc_wqs_err;
 558         }
 559
 560         cmdqs->cmd_buf_pool = dma_pool_create("hinic_cmdq", hwdev,
 561                                               HINIC_CMDQ_BUF_SIZE,
 562                                               HINIC_CMDQ_BUF_SIZE, 0ULL);
 563         if (!cmdqs->cmd_buf_pool) {
 564                 PMD_DRV_LOG(ERR, "Create cmdq buffer pool failed");
 565                 err = -ENOMEM;
 566                 goto pool_create_err;
 567         }
 568
 569         err = hinic_cmdq_alloc(cmdqs->saved_wqs, hwdev,
 570                                HINIC_MAX_CMDQ_TYPES, HINIC_CMDQ_WQ_BUF_SIZE,
 571                                CMDQ_WQEBB_SHIFT, HINIC_CMDQ_DEPTH);
 572         if (err) {
 573                 PMD_DRV_LOG(ERR, "Allocate cmdq failed");
 574                 goto cmdq_alloc_err;
 575         }
 576
 577         cmdq_type = HINIC_CMDQ_SYNC;
 578         for (; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++) {
 579                 err = init_cmdq(&cmdqs->cmdq[cmdq_type], hwdev,
 580                                 &cmdqs->saved_wqs[cmdq_type], cmdq_type);
 581                 if (err) {
 582                         PMD_DRV_LOG(ERR, "Initialize cmdq failed");
 583                         goto init_cmdq_err;
 584                 }
 585
 586                 cmdq_ctxt = &cmdqs->cmdq[cmdq_type].cmdq_ctxt;
 587                 cmdq_init_queue_ctxt(&cmdqs->cmdq[cmdq_type], cmdq_ctxt);
 588         }
 589
 590         err = hinic_set_cmdq_ctxts(hwdev);
 591         if (err)
 592                 goto init_cmdq_err;
 593
 594         return 0;
 595
 596 init_cmdq_err:
 597         type = HINIC_CMDQ_SYNC;
 598         for ( ; type < cmdq_type; type++)
 599                 free_cmdq(hwdev, &cmdqs->cmdq[type]);
 600
 601         hinic_cmdq_free(hwdev, cmdqs->saved_wqs, HINIC_MAX_CMDQ_TYPES);
 602
 603 cmdq_alloc_err:
 604         dma_pool_destroy(cmdqs->cmd_buf_pool);
 605
 606 pool_create_err:
 607         kfree(cmdqs->saved_wqs);
 608
 609 alloc_wqs_err:
 610         kfree(cmdqs);
 611
 612         return err;
 613 }
 614
 615 static void hinic_cmdqs_free(struct hinic_hwdev *hwdev)
 616 {
 617         struct hinic_cmdqs *cmdqs = hwdev->cmdqs;
 618         enum hinic_cmdq_type cmdq_type = HINIC_CMDQ_SYNC;
 619
 620         cmdqs->status &= ~HINIC_CMDQ_ENABLE;
 621
 622         for ( ; cmdq_type < HINIC_MAX_CMDQ_TYPES; cmdq_type++)
 623                 free_cmdq(cmdqs->hwdev, &cmdqs->cmdq[cmdq_type]);
 624
 625         hinic_cmdq_free(hwdev, cmdqs->saved_wqs,
 626                         HINIC_MAX_CMDQ_TYPES);
 627
 628         dma_pool_destroy(cmdqs->cmd_buf_pool);
 629
 630         kfree(cmdqs->saved_wqs);
 631
 632         kfree(cmdqs);
 633 }
 634
 635 static int hinic_set_cmdq_depth(struct hinic_hwdev *hwdev, u16 cmdq_depth)
 636 {
 637         struct hinic_root_ctxt root_ctxt;
 638         u16 out_size = sizeof(root_ctxt);
 639         int err;
 640
 641         memset(&root_ctxt, 0, sizeof(root_ctxt));
 642         root_ctxt.mgmt_msg_head.resp_aeq_num = HINIC_AEQ1;
 643         root_ctxt.func_idx = hinic_global_func_id(hwdev);
 644         root_ctxt.ppf_idx = hinic_ppf_idx(hwdev);
 645         root_ctxt.set_cmdq_depth = 1;
 646         root_ctxt.cmdq_depth = (u8)ilog2(cmdq_depth);
 647         err = hinic_msg_to_mgmt_sync(hwdev, HINIC_MOD_COMM,
 648                                      HINIC_MGMT_CMD_VAT_SET,
 649                                      &root_ctxt, sizeof(root_ctxt),
 650                                      &root_ctxt, &out_size, 0);
 651         if (err || !out_size || root_ctxt.mgmt_msg_head.status) {
 652                 PMD_DRV_LOG(ERR, "Set cmdq depth failed, err: %d, status: 0x%x, out_size: 0x%x",
 653                         err, root_ctxt.mgmt_msg_head.status, out_size);
 654                 return -EIO;
 655         }
 656
 657         return 0;
 658 }
 659
 660 int hinic_comm_cmdqs_init(struct hinic_hwdev *hwdev)
 661 {
 662         int err;
 663
 664         err = hinic_cmdqs_init(hwdev);
 665         if (err) {
 666                 PMD_DRV_LOG(ERR, "Init cmd queues failed");
 667                 return err;
 668         }
 669
 670         err = hinic_set_cmdq_depth(hwdev, HINIC_CMDQ_DEPTH);
 671         if (err) {
 672                 PMD_DRV_LOG(ERR, "Set cmdq depth failed");
 673                 goto set_cmdq_depth_err;
 674         }
 675
 676         return 0;
 677
 678 set_cmdq_depth_err:
 679         hinic_cmdqs_free(hwdev);
 680
 681         return err;
 682 }
 683
 684 static void cmdq_init_queue_ctxt(struct hinic_cmdq *cmdq,
 685                                  struct hinic_cmdq_ctxt *cmdq_ctxt)
 686 {
 687         struct hinic_cmdqs *cmdqs = (struct hinic_cmdqs *)cmdq_to_cmdqs(cmdq);
 688         struct hinic_hwdev *hwdev = cmdqs->hwdev;
 689         struct hinic_wq *wq = cmdq->wq;
 690         struct hinic_cmdq_ctxt_info *ctxt_info = &cmdq_ctxt->ctxt_info;
 691         u64 wq_first_page_paddr, pfn;
 692
 693         u16 start_ci = (u16)(wq->cons_idx);
 694
 695         /* The data in the HW is in Big Endian Format */
 696         wq_first_page_paddr = wq->queue_buf_paddr;
 697
 698         pfn = CMDQ_PFN(wq_first_page_paddr, HINIC_PAGE_SIZE);
 699         ctxt_info->curr_wqe_page_pfn =
 700                 CMDQ_CTXT_PAGE_INFO_SET(1, HW_BUSY_BIT) |
 701                 CMDQ_CTXT_PAGE_INFO_SET(1, CEQ_EN)      |
 702                 CMDQ_CTXT_PAGE_INFO_SET(0, CEQ_ARM)     |
 703                 CMDQ_CTXT_PAGE_INFO_SET(HINIC_CEQ_ID_CMDQ, EQ_ID) |
 704                 CMDQ_CTXT_PAGE_INFO_SET(pfn, CURR_WQE_PAGE_PFN);
 705
 706         ctxt_info->wq_block_pfn = CMDQ_CTXT_BLOCK_INFO_SET(start_ci, CI) |
 707                                 CMDQ_CTXT_BLOCK_INFO_SET(pfn, WQ_BLOCK_PFN);
 708         cmdq_ctxt->func_idx = HINIC_HWIF_GLOBAL_IDX(hwdev->hwif);
 709         cmdq_ctxt->ppf_idx  = HINIC_HWIF_PPF_IDX(hwdev->hwif);
 710         cmdq_ctxt->cmdq_id  = cmdq->cmdq_type;
 711 }
 712
 713 static int hinic_cmdq_poll_msg(struct hinic_cmdq *cmdq, u32 timeout)
 714 {
 715         struct hinic_cmdq_wqe *wqe;
 716         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 717         struct hinic_ctrl *ctrl;
 718         struct hinic_cmdq_cmd_info *cmd_info;
 719         u32 status_info, ctrl_info;
 720         u16 ci;
 721         int errcode;
 722         unsigned long end;
 723         int done = 0;
 724         int rc = 0;
 725
 726         wqe = hinic_read_wqe(cmdq->wq, 1, &ci);
 727         if (wqe == NULL) {
 728                 PMD_DRV_LOG(ERR, "No outstanding cmdq msg");
 729                 return -EINVAL;
 730         }
 731
 732         cmd_info = &cmdq->cmd_infos[ci];
 733         /* this cmd has not been filled and send to hw, or get TMO msg ack*/
 734         if (cmd_info->cmd_type == HINIC_CMD_TYPE_NONE) {
 735                 PMD_DRV_LOG(ERR, "Cmdq msg has not been filled and send to hw, or get TMO msg ack. cmdq ci: %u",
 736                             ci);
 737                 return -EINVAL;
 738         }
 739
 740         /* only arm bit is using scmd wqe, the wqe is lcmd */
 741         wqe_lcmd = &wqe->wqe_lcmd;
 742         ctrl = &wqe_lcmd->ctrl;
 743         end = jiffies + msecs_to_jiffies(timeout);
 744         do {
 745                 ctrl_info = be32_to_cpu((ctrl)->ctrl_info);
 746                 if (WQE_COMPLETED(ctrl_info)) {
 747                         done = 1;
 748                         break;
 749                 }
 750
 751                 rte_delay_ms(1);
 752         } while (time_before(jiffies, end));
 753
 754         if (done) {
 755                 status_info = be32_to_cpu(wqe_lcmd->status.status_info);
 756                 errcode = WQE_ERRCODE_GET(status_info, VAL);
 757                 cmdq_update_errcode(cmdq, ci, errcode);
 758                 clear_wqe_complete_bit(cmdq, wqe);
 759                 rc = 0;
 760         } else {
 761                 PMD_DRV_LOG(ERR, "Poll cmdq msg time out, ci: %u", ci);
 762                 rc = -ETIMEDOUT;
 763         }
 764
 765         /* set this cmd invalid */
 766         cmd_info->cmd_type = HINIC_CMD_TYPE_NONE;
 767
 768         return rc;
 769 }
 770
 771 static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,
 772                                      enum hinic_ack_type ack_type,
 773                                      enum hinic_mod_type mod, u8 cmd,
 774                                      struct hinic_cmd_buf *buf_in,
 775                                      u64 *out_param, u32 timeout)
 776 {
 777         struct hinic_wq *wq = cmdq->wq;
 778         struct hinic_cmdq_wqe *curr_wqe, wqe;
 779         struct hinic_cmdq_wqe_lcmd *wqe_lcmd;
 780         u16 curr_prod_idx, next_prod_idx, num_wqebbs;
 781         int wrapped;
 782         u32 timeo, wqe_size;
 783         int err;
 784
 785         wqe_size = cmdq_wqe_size(WQE_LCMD_TYPE);
 786         num_wqebbs = WQE_NUM_WQEBBS(wqe_size, wq);
 787
 788         /* Keep wrapped and doorbell index correct. */
 789         spin_lock(&cmdq->cmdq_lock);
 790
 791         curr_wqe = hinic_get_wqe(cmdq->wq, num_wqebbs, &curr_prod_idx);
 792         if (!curr_wqe) {
 793                 err = -EBUSY;
 794                 goto cmdq_unlock;
 795         }
 796
 797         memset(&wqe, 0, sizeof(wqe));
 798         wrapped = cmdq->wrapped;
 799
 800         next_prod_idx = curr_prod_idx + num_wqebbs;
 801         if (next_prod_idx >= wq->q_depth) {
 802                 cmdq->wrapped = !cmdq->wrapped;
 803                 next_prod_idx -= wq->q_depth;
 804         }
 805
 806         cmdq_set_lcmd_wqe(&wqe, SYNC_CMD_DIRECT_RESP, buf_in, NULL,
 807                           wrapped, ack_type, mod, cmd, curr_prod_idx);
 808
 809         /* The data that is written to HW should be in Big Endian Format */
 810         hinic_cpu_to_be32(&wqe, wqe_size);
 811
 812         /* CMDQ WQE is not shadow, therefore wqe will be written to wq */
 813         cmdq_wqe_fill(curr_wqe, &wqe);
 814
 815         cmdq->cmd_infos[curr_prod_idx].cmd_type = HINIC_CMD_TYPE_NORMAL;
 816
 817         cmdq_set_db(cmdq, HINIC_CMDQ_SYNC, next_prod_idx);
 818
 819         timeo = msecs_to_jiffies(timeout ? timeout : CMDQ_CMD_TIMEOUT);
 820         err = hinic_cmdq_poll_msg(cmdq, timeo);
 821         if (err) {
 822                 PMD_DRV_LOG(ERR, "Cmdq poll msg ack failed, prod idx: 0x%x",
 823                         curr_prod_idx);
 824                 err = -ETIMEDOUT;
 825                 goto cmdq_unlock;
 826         }
 827
 828         rte_smp_rmb();  /* read error code after completion */
 829
 830         if (out_param) {
 831                 wqe_lcmd = &curr_wqe->wqe_lcmd;
 832                 *out_param = cpu_to_be64(wqe_lcmd->completion.direct_resp);
 833         }
 834
 835         if (cmdq->errcode[curr_prod_idx] > 1) {
 836                 err = cmdq->errcode[curr_prod_idx];
 837                 goto cmdq_unlock;
 838         }
 839
 840 cmdq_unlock:
 841         spin_unlock(&cmdq->cmdq_lock);
 842
 843         return err;
 844 }
 845
 846 int hinic_cmdq_direct_resp(void *hwdev, enum hinic_ack_type ack_type,
 847                            enum hinic_mod_type mod, u8 cmd,
 848                            struct hinic_cmd_buf *buf_in,
 849                            u64 *out_param, u32 timeout)
 850 {
 851         struct hinic_cmdqs *cmdqs = ((struct hinic_hwdev *)hwdev)->cmdqs;
 852         int err = cmdq_params_valid(buf_in);
 853
 854         if (err) {
 855                 PMD_DRV_LOG(ERR, "Invalid CMDQ parameters");
 856                 return err;
 857         }
 858
 859         err = wait_cmdqs_enable(cmdqs);
 860         if (err) {
 861                 PMD_DRV_LOG(ERR, "Cmdq is disable");
 862                 return err;
 863         }
 864
 865         return cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC_CMDQ_SYNC],
 866                                          ack_type, mod, cmd, buf_in,
 867                                          out_param, timeout);
 868 }