drivers/net/hinic/base/hinic_pmd_api_cmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include "hinic_compat.h"
   6 #include "hinic_csr.h"
   7 #include "hinic_pmd_hwdev.h"
   8 #include "hinic_pmd_cmd.h"
   9 #include "hinic_pmd_hwif.h"
  10 #include "hinic_pmd_api_cmd.h"
  11
  12 #define API_CMD_CHAIN_CELL_SIZE_SHIFT   6U
  13
  14 #define API_CMD_CELL_DESC_SIZE          8
  15 #define API_CMD_CELL_DATA_ADDR_SIZE     8
  16
  17 #define API_CHAIN_NUM_CELLS             32
  18 #define API_CHAIN_CELL_SIZE             128
  19 #define API_CHAIN_RSP_DATA_SIZE         128
  20
  21 #define API_CHAIN_CELL_ALIGNMENT        8
  22
  23 #define API_CMD_TIMEOUT                 10000
  24
  25 #define API_CMD_BUF_SIZE                2048UL
  26
  27 #define API_CMD_NODE_ALIGN_SIZE         512UL
  28 #define API_PAYLOAD_ALIGN_SIZE          64
  29
  30 #define API_CHAIN_RESP_ALIGNMENT        64ULL
  31
  32 #define COMPLETION_TIMEOUT_DEFAULT              1000UL
  33 #define POLLING_COMPLETION_TIMEOUT_DEFAULT      1000U
  34
  35 #define API_CMD_RESPONSE_DATA_PADDR(val)        be64_to_cpu(*((u64 *)(val)))
  36
  37 #define READ_API_CMD_PRIV_DATA(id, token)       (((id) << 16) + (token))
  38 #define WRITE_API_CMD_PRIV_DATA(id)             ((id) << 16)
  39
  40 #define MASKED_IDX(chain, idx)          ((idx) & ((chain)->num_cells - 1))
  41
  42 #undef  SIZE_4BYTES
  43 #undef  SIZE_8BYTES
  44 #define SIZE_4BYTES(size)               (ALIGN((u32)(size), 4U) >> 2)
  45 #define SIZE_8BYTES(size)               (ALIGN((u32)(size), 8U) >> 3)
  46
  47 enum api_cmd_data_format {
  48         SGL_DATA     = 1,
  49 };
  50
  51 enum api_cmd_type {
  52         API_CMD_WRITE_TYPE = 0,
  53         API_CMD_READ_TYPE = 1,
  54 };
  55
  56 enum api_cmd_bypass {
  57         NOT_BYPASS = 0,
  58         BYPASS = 1,
  59 };
  60
  61 enum api_cmd_resp_aeq {
  62         NOT_TRIGGER = 0,
  63         TRIGGER     = 1,
  64 };
  65
  66 static u8 xor_chksum_set(void *data)
  67 {
  68         int idx;
  69         u8 checksum = 0;
  70         u8 *val = (u8 *)data;
  71
  72         for (idx = 0; idx < 7; idx++)
  73                 checksum ^= val[idx];
  74
  75         return checksum;
  76 }
  77
  78 static void set_prod_idx(struct hinic_api_cmd_chain *chain)
  79 {
  80         enum hinic_api_cmd_chain_type chain_type = chain->chain_type;
  81         struct hinic_hwif *hwif = chain->hwdev->hwif;
  82         u32 hw_prod_idx_addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type);
  83         u32 prod_idx = chain->prod_idx;
  84
  85         hinic_hwif_write_reg(hwif, hw_prod_idx_addr, prod_idx);
  86 }
  87
  88 static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)
  89 {
  90         u32 addr, val;
  91
  92         addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type);
  93         val  = hinic_hwif_read_reg(chain->hwdev->hwif, addr);
  94
  95         return HINIC_API_CMD_STATUS_GET(val, CONS_IDX);
  96 }
  97
  98 static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain)
  99 {
 100         u32 addr, val;
 101
 102         addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type);
 103         val  = hinic_hwif_read_reg(chain->hwdev->hwif, addr);
 104
 105         PMD_DRV_LOG(ERR, "chain type: 0x%x", chain->chain_type);
 106         PMD_DRV_LOG(ERR, "chain hw cpld error: 0x%x",
 107                 HINIC_API_CMD_STATUS_GET(val, CPLD_ERR));
 108         PMD_DRV_LOG(ERR, "chain hw check error: 0x%x",
 109                 HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR));
 110         PMD_DRV_LOG(ERR, "chain hw current fsm: 0x%x",
 111                 HINIC_API_CMD_STATUS_GET(val, FSM));
 112         PMD_DRV_LOG(ERR, "chain hw current ci: 0x%x",
 113                 HINIC_API_CMD_STATUS_GET(val, CONS_IDX));
 114
 115         addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type);
 116         val  = hinic_hwif_read_reg(chain->hwdev->hwif, addr);
 117         PMD_DRV_LOG(ERR, "Chain hw current pi: 0x%x", val);
 118 }
 119
 120 /**
 121  * chain_busy - check if the chain is still processing last requests
 122  * @chain: chain to check
 123  */
 124 static int chain_busy(struct hinic_api_cmd_chain *chain)
 125 {
 126         switch (chain->chain_type) {
 127         case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
 128         case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
 129                 chain->cons_idx = get_hw_cons_idx(chain);
 130                 if (chain->cons_idx == MASKED_IDX(chain, chain->prod_idx + 1)) {
 131                         PMD_DRV_LOG(ERR, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d",
 132                                 chain->chain_type, chain->cons_idx,
 133                                 chain->prod_idx);
 134                         dump_api_chain_reg(chain);
 135                         return -EBUSY;
 136                 }
 137                 break;
 138         default:
 139                 PMD_DRV_LOG(ERR, "Unknown Chain type");
 140                 return -EINVAL;
 141         }
 142
 143         return 0;
 144 }
 145
 146 /**
 147  * get_cell_data_size - get the data size of specific cell type
 148  * @type: chain type
 149  */
 150 static u16 get_cell_data_size(enum hinic_api_cmd_chain_type type,
 151                                 __rte_unused u16 cmd_size)
 152 {
 153         u16 cell_data_size = 0;
 154
 155         switch (type) {
 156         case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
 157         case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
 158                 cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE +
 159                                         API_CMD_CELL_DATA_ADDR_SIZE,
 160                                         API_CHAIN_CELL_ALIGNMENT);
 161                 break;
 162         default:
 163                 break;
 164         }
 165
 166         return cell_data_size;
 167 }
 168
 169 /**
 170  * prepare_cell_ctrl - prepare the ctrl of the cell for the command
 171  * @cell_ctrl: the control of the cell to set the control into it
 172  * @cell_len: the size of the cell
 173  */
 174 static void prepare_cell_ctrl(u64 *cell_ctrl, u16 cell_len)
 175 {
 176         u64 ctrl;
 177         u8 chksum;
 178
 179         /* Read Modify Write */
 180         ctrl = be64_to_cpu(*cell_ctrl);
 181         ctrl = HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, CELL_LEN) &
 182                 HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, RD_DMA_ATTR_OFF) &
 183                 HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, WR_DMA_ATTR_OFF) &
 184                 HINIC_API_CMD_CELL_CTRL_CLEAR(ctrl, XOR_CHKSUM);
 185
 186         ctrl |=  HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(cell_len), CELL_LEN) |
 187                 HINIC_API_CMD_CELL_CTRL_SET(0ULL, RD_DMA_ATTR_OFF) |
 188                 HINIC_API_CMD_CELL_CTRL_SET(0ULL, WR_DMA_ATTR_OFF);
 189
 190         chksum = xor_chksum_set(&ctrl);
 191
 192         ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM);
 193
 194         /* The data in the HW should be in Big Endian Format */
 195         *cell_ctrl = cpu_to_be64(ctrl);
 196 }
 197
 198 /**
 199  * prepare_api_cmd - prepare API CMD command
 200  * @chain: chain for the command
 201  * @cell: the cell of the command
 202  * @dest: destination node on the card that will receive the command
 203  * @cmd: command data
 204  * @cmd_size: the command size
 205  */
 206 static void prepare_api_cmd(struct hinic_api_cmd_chain *chain,
 207                                 struct hinic_api_cmd_cell *cell,
 208                                 enum hinic_node_id dest,
 209                                 void *cmd, u16 cmd_size)
 210 {
 211         struct hinic_api_cmd_cell_ctxt  *cell_ctxt;
 212         u32 priv;
 213
 214         cell_ctxt = &chain->cell_ctxt[chain->prod_idx];
 215
 216         /* Clear all the members before changes */
 217         cell->desc = HINIC_API_CMD_DESC_CLEAR(cell->desc, API_TYPE) &
 218                         HINIC_API_CMD_DESC_CLEAR(cell->desc, RD_WR) &
 219                         HINIC_API_CMD_DESC_CLEAR(cell->desc, MGMT_BYPASS) &
 220                         HINIC_API_CMD_DESC_CLEAR(cell->desc, RESP_AEQE_EN) &
 221                         HINIC_API_CMD_DESC_CLEAR(cell->desc, DEST) &
 222                         HINIC_API_CMD_DESC_CLEAR(cell->desc, SIZE) &
 223                         HINIC_API_CMD_DESC_CLEAR(cell->desc, XOR_CHKSUM);
 224
 225         switch (chain->chain_type) {
 226         case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
 227         case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
 228                 priv =  WRITE_API_CMD_PRIV_DATA(chain->chain_type);
 229                 cell->desc = HINIC_API_CMD_DESC_SET(SGL_DATA, API_TYPE) |
 230                         HINIC_API_CMD_DESC_SET(API_CMD_WRITE_TYPE, RD_WR) |
 231                         HINIC_API_CMD_DESC_SET(NOT_BYPASS, MGMT_BYPASS) |
 232                         HINIC_API_CMD_DESC_SET(TRIGGER, RESP_AEQE_EN)   |
 233                         HINIC_API_CMD_DESC_SET(priv, PRIV_DATA);
 234                 break;
 235         default:
 236                 PMD_DRV_LOG(ERR, "Unknown Chain type");
 237                 return;
 238         }
 239
 240         cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST)         |
 241                         HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE);
 242         cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc),
 243                                                 XOR_CHKSUM);
 244
 245         /* The data in the HW should be in Big Endian Format */
 246         cell->desc = cpu_to_be64(cell->desc);
 247
 248         memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size);
 249 }
 250
 251 /**
 252  * prepare_cell - prepare cell ctrl and cmd in the current producer cell
 253  * @chain: chain for the command
 254  * @dest: destination node on the card that will receive the command
 255  * @cmd: command data
 256  * @cmd_size: the command size
 257  */
 258 static void prepare_cell(struct hinic_api_cmd_chain *chain,
 259                          enum  hinic_node_id dest,
 260                          void *cmd, u16 cmd_size)
 261 {
 262         struct hinic_api_cmd_cell *curr_node;
 263         u16 cell_size;
 264
 265         curr_node = chain->curr_node;
 266
 267         cell_size = get_cell_data_size(chain->chain_type, cmd_size);
 268
 269         prepare_cell_ctrl(&curr_node->ctrl, cell_size);
 270         prepare_api_cmd(chain, curr_node, dest, cmd, cmd_size);
 271 }
 272
 273 static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain)
 274 {
 275         chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1);
 276 }
 277
 278 static void issue_api_cmd(struct hinic_api_cmd_chain *chain)
 279 {
 280         set_prod_idx(chain);
 281 }
 282
 283 /**
 284  * api_cmd_status_update - update the status of the chain
 285  * @chain: chain to update
 286  */
 287 static void api_cmd_status_update(struct hinic_api_cmd_chain *chain)
 288 {
 289         struct hinic_api_cmd_status *wb_status;
 290         enum hinic_api_cmd_chain_type chain_type;
 291         u64     status_header;
 292         u32     buf_desc;
 293
 294         wb_status = chain->wb_status;
 295
 296         buf_desc = be32_to_cpu(wb_status->buf_desc);
 297         if (HINIC_API_CMD_STATUS_GET(buf_desc, CHKSUM_ERR)) {
 298                 PMD_DRV_LOG(ERR, "API CMD status Xor check error");
 299                 return;
 300         }
 301
 302         status_header = be64_to_cpu(wb_status->header);
 303         chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID);
 304         if (chain_type >= HINIC_API_CMD_MAX)
 305                 return;
 306
 307         if (chain_type != chain->chain_type)
 308                 return;
 309
 310         chain->cons_idx = HINIC_API_CMD_STATUS_GET(buf_desc, CONS_IDX);
 311 }
 312
 313 /**
 314  * wait_for_status_poll - wait for write to mgmt command to complete
 315  * @chain: the chain of the command
 316  * Return: 0 - success, negative - failure
 317  */
 318 static int wait_for_status_poll(struct hinic_api_cmd_chain *chain)
 319 {
 320         unsigned long end;
 321         int err = -ETIMEDOUT;
 322
 323         end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
 324         do {
 325                 api_cmd_status_update(chain);
 326
 327                 /* SYNC API CMD cmd should start after prev cmd finished */
 328                 if (chain->cons_idx == chain->prod_idx) {
 329                         err = 0;
 330                         break;
 331                 }
 332
 333                 rte_delay_us(10);
 334         } while (time_before(jiffies, end));
 335
 336         return err;
 337 }
 338
 339 /**
 340  * wait_for_api_cmd_completion - wait for command to complete
 341  * @chain: chain for the command
 342  * Return: 0 - success, negative - failure
 343  */
 344 static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain,
 345                        __rte_unused struct hinic_api_cmd_cell_ctxt *ctxt,
 346                        __rte_unused void *ack, __rte_unused u16 ack_size)
 347 {
 348         int err = 0;
 349
 350         /* poll api cmd status for debug*/
 351         switch (chain->chain_type) {
 352         case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
 353                 err = wait_for_status_poll(chain);
 354                 if (err)
 355                         PMD_DRV_LOG(ERR, "API CMD poll status timeout");
 356                 break;
 357         case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
 358                 break;
 359         default:
 360                 PMD_DRV_LOG(ERR, "Unknown API CMD chain type");
 361                 err = -EINVAL;
 362                 break;
 363         }
 364
 365         if (err)
 366                 dump_api_chain_reg(chain);
 367
 368         return err;
 369 }
 370
 371 static inline void update_api_cmd_ctxt(struct hinic_api_cmd_chain *chain,
 372                                        struct hinic_api_cmd_cell_ctxt *ctxt)
 373 {
 374         ctxt->status = 1;
 375         ctxt->saved_prod_idx = chain->prod_idx;
 376 }
 377
 378 /**
 379  * api_cmd - API CMD command
 380  * @chain: chain for the command
 381  * @dest: destination node on the card that will receive the command
 382  * @cmd: command data
 383  * @cmd_size: the command size
 384  * @ack: pointer to messages to response
 385  * @ack_size: the size of ack message
 386  * Return: 0 - success, negative - failure
 387  */
 388 static int api_cmd(struct hinic_api_cmd_chain *chain,
 389                    enum hinic_node_id dest,
 390                    void *cmd, u16 cmd_size, void *ack, u16 ack_size)
 391 {
 392         struct hinic_api_cmd_cell_ctxt *ctxt;
 393
 394         spin_lock(&chain->async_lock);
 395
 396         ctxt = &chain->cell_ctxt[chain->prod_idx];
 397         if (chain_busy(chain)) {
 398                 spin_unlock(&chain->async_lock);
 399                 return -EBUSY;
 400         }
 401         update_api_cmd_ctxt(chain, ctxt);
 402
 403         prepare_cell(chain, dest, cmd, cmd_size);
 404
 405         cmd_chain_prod_idx_inc(chain);
 406
 407         rte_wmb();/* issue the command */
 408
 409         issue_api_cmd(chain);
 410
 411         /* incremented prod idx, update ctxt */
 412         chain->curr_node = chain->cell_ctxt[chain->prod_idx].cell_vaddr;
 413
 414         spin_unlock(&chain->async_lock);
 415
 416         return wait_for_api_cmd_completion(chain, ctxt, ack, ack_size);
 417 }
 418
 419 /**
 420  * hinic_api_cmd_write - Write API CMD command
 421  * @chain: chain for write command
 422  * @dest: destination node on the card that will receive the command
 423  * @cmd: command data
 424  * @size: the command size
 425  * Return: 0 - success, negative - failure
 426  */
 427 int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain,
 428                         enum hinic_node_id dest, void *cmd, u16 size)
 429 {
 430         /* Verify the chain type */
 431         return api_cmd(chain, dest, cmd, size, NULL, 0);
 432 }
 433
 434 /**
 435  * api_cmd_hw_restart - restart the chain in the HW
 436  * @chain: the API CMD specific chain to restart
 437  */
 438 static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain)
 439 {
 440         struct hinic_hwif *hwif = chain->hwdev->hwif;
 441         unsigned long end;
 442         u32 reg_addr, val;
 443         int err;
 444
 445         /* Read Modify Write */
 446         reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type);
 447         val = hinic_hwif_read_reg(hwif, reg_addr);
 448
 449         val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART);
 450         val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART);
 451
 452         hinic_hwif_write_reg(hwif, reg_addr, val);
 453
 454         end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
 455         err = -ETIMEDOUT;
 456         do {
 457                 val = hinic_hwif_read_reg(hwif, reg_addr);
 458
 459                 if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) {
 460                         err = 0;
 461                         break;
 462                 }
 463
 464                 rte_delay_ms(1);
 465         } while (time_before(jiffies, end));
 466
 467         return err;
 468 }
 469
 470 /**
 471  * api_cmd_ctrl_init - set the control register of a chain
 472  * @chain: the API CMD specific chain to set control register for
 473  */
 474 static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain)
 475 {
 476         struct hinic_hwif *hwif = chain->hwdev->hwif;
 477         u32 reg_addr, ctrl;
 478         u32 cell_size;
 479
 480         /* Read Modify Write */
 481         reg_addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
 482
 483         cell_size = (u32)ilog2(chain->cell_size >>
 484                                API_CMD_CHAIN_CELL_SIZE_SHIFT);
 485
 486         ctrl = hinic_hwif_read_reg(hwif, reg_addr);
 487
 488         ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) &
 489                 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
 490
 491         ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(0, AEQE_EN) |
 492                 HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE);
 493
 494         hinic_hwif_write_reg(hwif, reg_addr, ctrl);
 495 }
 496
 497 /**
 498  * api_cmd_set_status_addr - set the status address of a chain in the HW
 499  * @chain: the API CMD specific chain to set status address for
 500  */
 501 static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain)
 502 {
 503         struct hinic_hwif *hwif = chain->hwdev->hwif;
 504         u32 addr, val;
 505
 506         addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type);
 507         val = upper_32_bits(chain->wb_status_paddr);
 508         hinic_hwif_write_reg(hwif, addr, val);
 509
 510         addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type);
 511         val = lower_32_bits(chain->wb_status_paddr);
 512         hinic_hwif_write_reg(hwif, addr, val);
 513 }
 514
 515 /**
 516  * api_cmd_set_num_cells - set the number cells of a chain in the HW
 517  * @chain: the API CMD specific chain to set the number of cells for
 518  */
 519 static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain)
 520 {
 521         struct hinic_hwif *hwif = chain->hwdev->hwif;
 522         u32 addr, val;
 523
 524         addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type);
 525         val  = chain->num_cells;
 526         hinic_hwif_write_reg(hwif, addr, val);
 527 }
 528
 529 /**
 530  * api_cmd_head_init - set the head cell of a chain in the HW
 531  * @chain: the API CMD specific chain to set the head for
 532  */
 533 static void api_cmd_head_init(struct hinic_api_cmd_chain *chain)
 534 {
 535         struct hinic_hwif *hwif = chain->hwdev->hwif;
 536         u32 addr, val;
 537
 538         addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type);
 539         val = upper_32_bits(chain->head_cell_paddr);
 540         hinic_hwif_write_reg(hwif, addr, val);
 541
 542         addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type);
 543         val = lower_32_bits(chain->head_cell_paddr);
 544         hinic_hwif_write_reg(hwif, addr, val);
 545 }
 546
 547 /**
 548  * wait_for_ready_chain - wait for the chain to be ready
 549  * @chain: the API CMD specific chain to wait for
 550  * Return: 0 - success, negative - failure
 551  */
 552 static int wait_for_ready_chain(struct hinic_api_cmd_chain *chain)
 553 {
 554         struct hinic_hwif *hwif = chain->hwdev->hwif;
 555         unsigned long end;
 556         u32 addr, val;
 557         u32 hw_cons_idx;
 558         int err;
 559
 560         end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT);
 561
 562         addr = HINIC_CSR_API_CMD_STATUS_0_ADDR(chain->chain_type);
 563         err = -ETIMEDOUT;
 564         do {
 565                 val = hinic_hwif_read_reg(hwif, addr);
 566                 hw_cons_idx = HINIC_API_CMD_STATUS_GET(val, CONS_IDX);
 567
 568                 /* Wait for HW cons idx to be updated */
 569                 if (hw_cons_idx == chain->cons_idx) {
 570                         err = 0;
 571                         break;
 572                 }
 573
 574                 rte_delay_ms(1);
 575         } while (time_before(jiffies, end));
 576
 577         return err;
 578 }
 579
 580 /**
 581  * api_cmd_chain_hw_clean - clean the HW
 582  * @chain: the API CMD specific chain
 583  */
 584 static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain)
 585 {
 586         struct hinic_hwif *hwif = chain->hwdev->hwif;
 587         u32 addr, ctrl;
 588
 589         addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type);
 590
 591         ctrl = hinic_hwif_read_reg(hwif, addr);
 592         ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_EN) &
 593                HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR)    &
 594                HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN)    &
 595                HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) &
 596                HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE);
 597
 598         hinic_hwif_write_reg(hwif, addr, ctrl);
 599 }
 600
 601 /**
 602  * api_cmd_chain_hw_init - initialize the chain in the HW
 603  *(initialize API command csr)
 604  * @chain: the API CMD specific chain to initialize in HW
 605  * Return: 0 - success, negative - failure
 606  */
 607 static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain)
 608 {
 609         api_cmd_chain_hw_clean(chain);
 610
 611         api_cmd_set_status_addr(chain);
 612
 613         if (api_cmd_hw_restart(chain)) {
 614                 PMD_DRV_LOG(ERR, "Restart api_cmd_hw failed");
 615                 return -EBUSY;
 616         }
 617
 618         api_cmd_ctrl_init(chain);
 619         api_cmd_set_num_cells(chain);
 620         api_cmd_head_init(chain);
 621
 622         return wait_for_ready_chain(chain);
 623 }
 624
 625 /**
 626  * free_cmd_buf - free the dma buffer of API CMD command
 627  * @chain: the API CMD specific chain of the cmd
 628  * @cell_idx: the cell index of the cmd
 629  */
 630 static void free_cmd_buf(struct hinic_api_cmd_chain *chain, u32 cell_idx)
 631 {
 632         struct hinic_api_cmd_cell_ctxt *cell_ctxt;
 633         void *dev = chain->hwdev;
 634
 635         cell_ctxt = &chain->cell_ctxt[cell_idx];
 636
 637         dma_free_coherent(dev, (API_CMD_BUF_SIZE + API_PAYLOAD_ALIGN_SIZE),
 638                           cell_ctxt->api_cmd_vaddr_free,
 639                           cell_ctxt->api_cmd_paddr_free);
 640 }
 641
 642 /**
 643  * alloc_cmd_buf - allocate a dma buffer for API CMD command
 644  * @chain: the API CMD specific chain for the cmd
 645  * @cell: the cell in the HW for the cmd
 646  * @cell_idx: the index of the cell
 647  * Return: 0 - success, negative - failure
 648  */
 649 static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain,
 650                          struct hinic_api_cmd_cell *cell, u32 cell_idx)
 651 {
 652         void *dev = chain->hwdev;
 653         struct hinic_api_cmd_cell_ctxt *cell_ctxt;
 654         dma_addr_t cmd_paddr = 0;
 655         void *cmd_vaddr;
 656         void *cmd_vaddr_alloc;
 657         int err = 0;
 658
 659         cmd_vaddr_alloc = dma_zalloc_coherent(dev, (API_CMD_BUF_SIZE +
 660                                               API_PAYLOAD_ALIGN_SIZE),
 661                                               &cmd_paddr, SOCKET_ID_ANY);
 662         if (!cmd_vaddr_alloc) {
 663                 PMD_DRV_LOG(ERR, "Allocate API CMD dma memory failed");
 664                 return -ENOMEM;
 665         }
 666
 667         cell_ctxt = &chain->cell_ctxt[cell_idx];
 668
 669         cell_ctxt->api_cmd_paddr_free = cmd_paddr;
 670         cell_ctxt->api_cmd_vaddr_free = cmd_vaddr_alloc;
 671         cmd_vaddr = PTR_ALIGN(cmd_vaddr_alloc, API_PAYLOAD_ALIGN_SIZE);
 672         cmd_paddr = cmd_paddr + ((u64)cmd_vaddr - (u64)cmd_vaddr_alloc);
 673
 674         cell_ctxt->api_cmd_vaddr = cmd_vaddr;
 675         cell_ctxt->api_cmd_paddr = cmd_paddr;
 676
 677         /* set the cmd DMA address in the cell */
 678         switch (chain->chain_type) {
 679         case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
 680         case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
 681                 cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr);
 682                 break;
 683         default:
 684                 PMD_DRV_LOG(ERR, "Unknown API CMD chain type");
 685                 free_cmd_buf(chain, cell_idx);
 686                 err = -EINVAL;
 687                 break;
 688         }
 689
 690         return err;
 691 }
 692
 693 /**
 694  * api_cmd_create_cell - create API CMD cell of specific chain
 695  * @chain: the API CMD specific chain to create its cell
 696  * @cell_idx: the cell index to create
 697  * @pre_node: previous cell
 698  * @node_vaddr: the virt addr of the cell
 699  * Return: 0 - success, negative - failure
 700  */
 701 static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain,
 702                                u32 cell_idx,
 703                                struct hinic_api_cmd_cell *pre_node,
 704                                struct hinic_api_cmd_cell **node_vaddr)
 705 {
 706         void *dev = chain->hwdev;
 707         struct hinic_api_cmd_cell_ctxt *cell_ctxt;
 708         struct hinic_api_cmd_cell *node;
 709         dma_addr_t node_paddr = 0;
 710         void *node_vaddr_alloc;
 711         int err = 0;
 712
 713         node_vaddr_alloc = dma_zalloc_coherent(dev, (chain->cell_size +
 714                                                API_CMD_NODE_ALIGN_SIZE),
 715                                                &node_paddr, SOCKET_ID_ANY);
 716         if (!node_vaddr_alloc) {
 717                 PMD_DRV_LOG(ERR, "Allocate dma API CMD cell failed");
 718                 return -ENOMEM;
 719         }
 720
 721         cell_ctxt = &chain->cell_ctxt[cell_idx];
 722
 723         cell_ctxt->cell_vaddr_free = node_vaddr_alloc;
 724         cell_ctxt->cell_paddr_free = node_paddr;
 725         node = (struct hinic_api_cmd_cell *)PTR_ALIGN(node_vaddr_alloc,
 726                 API_CMD_NODE_ALIGN_SIZE);
 727         node_paddr = node_paddr + ((u64)node - (u64)node_vaddr_alloc);
 728
 729         node->read.hw_wb_resp_paddr = 0;
 730
 731         cell_ctxt->cell_vaddr = node;
 732         cell_ctxt->cell_paddr = node_paddr;
 733
 734         if (!pre_node) {
 735                 chain->head_node = node;
 736                 chain->head_cell_paddr = node_paddr;
 737         } else {
 738                 /* The data in the HW should be in Big Endian Format */
 739                 pre_node->next_cell_paddr = cpu_to_be64(node_paddr);
 740         }
 741
 742         /* Driver software should make sure that there is an empty
 743          * API command cell at the end the chain
 744          */
 745         node->next_cell_paddr = 0;
 746
 747         switch (chain->chain_type) {
 748         case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
 749         case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
 750                 err = alloc_cmd_buf(chain, node, cell_idx);
 751                 if (err) {
 752                         PMD_DRV_LOG(ERR, "Allocate cmd buffer failed");
 753                         goto alloc_cmd_buf_err;
 754                 }
 755                 break;
 756         default:
 757                 PMD_DRV_LOG(ERR, "Unsupported API CMD chain type");
 758                 err = -EINVAL;
 759                 goto alloc_cmd_buf_err;
 760         }
 761
 762         *node_vaddr = node;
 763
 764         return 0;
 765
 766 alloc_cmd_buf_err:
 767         dma_free_coherent(dev, (chain->cell_size + API_CMD_NODE_ALIGN_SIZE),
 768                           node_vaddr_alloc, cell_ctxt->cell_paddr_free);
 769
 770         return err;
 771 }
 772
 773 /**
 774  * api_cmd_destroy_cell - destroy API CMD cell of specific chain
 775  * @chain: the API CMD specific chain to destroy its cell
 776  * @cell_idx: the cell to destroy
 777  */
 778 static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain,
 779                                  u32 cell_idx)
 780 {
 781         void *dev = chain->hwdev;
 782         struct hinic_api_cmd_cell_ctxt *cell_ctxt;
 783         struct hinic_api_cmd_cell *node;
 784         dma_addr_t node_paddr;
 785
 786         cell_ctxt = &chain->cell_ctxt[cell_idx];
 787
 788         node = (struct hinic_api_cmd_cell *)(cell_ctxt->cell_vaddr_free);
 789         node_paddr = cell_ctxt->cell_paddr_free;
 790
 791         if (cell_ctxt->api_cmd_vaddr) {
 792                 switch (chain->chain_type) {
 793                 case HINIC_API_CMD_PMD_WRITE_TO_MGMT:
 794                 case HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU:
 795                         free_cmd_buf(chain, cell_idx);
 796                         break;
 797                 default:
 798                         break;
 799                 }
 800
 801                 dma_free_coherent(dev, (chain->cell_size +
 802                                   API_CMD_NODE_ALIGN_SIZE),
 803                                   node, node_paddr);
 804         }
 805 }
 806
 807 /**
 808  * api_cmd_destroy_cells - destroy API CMD cells of specific chain
 809  * @chain: the API CMD specific chain to destroy its cells
 810  * @num_cells: number of cells to destroy
 811  */
 812 static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain,
 813                                          u32 num_cells)
 814 {
 815         u32 cell_idx;
 816
 817         for (cell_idx = 0; cell_idx < num_cells; cell_idx++)
 818                 api_cmd_destroy_cell(chain, cell_idx);
 819 }
 820
 821 /**
 822  * api_cmd_create_cells - create API CMD cells for specific chain
 823  * @chain: the API CMD specific chain
 824  * Return: 0 - success, negative - failure
 825  */
 826 static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain)
 827 {
 828         struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL;
 829         u32 cell_idx;
 830         int err;
 831
 832         for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) {
 833                 err = api_cmd_create_cell(chain, cell_idx, pre_node, &node);
 834                 if (err) {
 835                         PMD_DRV_LOG(ERR, "Create API CMD cell failed");
 836                         goto create_cell_err;
 837                 }
 838
 839                 pre_node = node;
 840         }
 841
 842         if (!node) {
 843                 err = -EFAULT;
 844                 goto create_cell_err;
 845         }
 846
 847         /* set the Final node to point on the start */
 848         node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr);
 849
 850         /* set the current node to be the head */
 851         chain->curr_node = chain->head_node;
 852         return 0;
 853
 854 create_cell_err:
 855         api_cmd_destroy_cells(chain, cell_idx);
 856         return err;
 857 }
 858
 859 /**
 860  * api_chain_init - initialize API CMD specific chain
 861  * @chain: the API CMD specific chain to initialize
 862  * @attr: attributes to set in the chain
 863  * Return: 0 - success, negative - failure
 864  */
 865 static int api_chain_init(struct hinic_api_cmd_chain *chain,
 866                           struct hinic_api_cmd_chain_attr *attr)
 867 {
 868         void *dev = chain->hwdev;
 869         size_t cell_ctxt_size;
 870         int err;
 871
 872         chain->chain_type  = attr->chain_type;
 873         chain->num_cells = attr->num_cells;
 874         chain->cell_size = attr->cell_size;
 875         chain->rsp_size = attr->rsp_size;
 876
 877         chain->prod_idx  = 0;
 878         chain->cons_idx  = 0;
 879
 880         spin_lock_init(&chain->async_lock);
 881
 882         cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt);
 883         chain->cell_ctxt = kzalloc(cell_ctxt_size, GFP_KERNEL);
 884         if (!chain->cell_ctxt) {
 885                 PMD_DRV_LOG(ERR, "Allocate cell contexts for a chain failed");
 886                 err = -ENOMEM;
 887                 goto alloc_cell_ctxt_err;
 888         }
 889
 890         chain->wb_status = (struct hinic_api_cmd_status *)
 891                            dma_zalloc_coherent(dev, sizeof(*chain->wb_status),
 892                                 &chain->wb_status_paddr, SOCKET_ID_ANY);
 893         if (!chain->wb_status) {
 894                 PMD_DRV_LOG(ERR, "Allocate DMA wb status failed");
 895                 err = -ENOMEM;
 896                 goto alloc_wb_status_err;
 897         }
 898
 899         return 0;
 900
 901 alloc_wb_status_err:
 902         kfree(chain->cell_ctxt);
 903
 904 alloc_cell_ctxt_err:
 905
 906         return err;
 907 }
 908
 909 /**
 910  * api_chain_free - free API CMD specific chain
 911  * @chain: the API CMD specific chain to free
 912  */
 913 static void api_chain_free(struct hinic_api_cmd_chain *chain)
 914 {
 915         void *dev = chain->hwdev;
 916
 917         dma_free_coherent(dev, sizeof(*chain->wb_status),
 918                           chain->wb_status, chain->wb_status_paddr);
 919         kfree(chain->cell_ctxt);
 920 }
 921
 922 /**
 923  * api_cmd_create_chain - create API CMD specific chain
 924  * @cmd_chain: the API CMD specific chain to create
 925  * @attr: attributes to set in the chain
 926  * Return: 0 - success, negative - failure
 927  */
 928 static int api_cmd_create_chain(struct hinic_api_cmd_chain **cmd_chain,
 929                                 struct hinic_api_cmd_chain_attr *attr)
 930 {
 931         struct hinic_hwdev *hwdev = attr->hwdev;
 932         struct hinic_api_cmd_chain *chain;
 933         int err;
 934
 935         if (attr->num_cells & (attr->num_cells - 1)) {
 936                 PMD_DRV_LOG(ERR, "Invalid number of cells, must be power of 2");
 937                 return -EINVAL;
 938         }
 939
 940         chain = kzalloc(sizeof(*chain), GFP_KERNEL);
 941         if (!chain) {
 942                 PMD_DRV_LOG(ERR, "Allocate memory for the chain failed");
 943                 return -ENOMEM;
 944         }
 945
 946         chain->hwdev = hwdev;
 947
 948         err = api_chain_init(chain, attr);
 949         if (err) {
 950                 PMD_DRV_LOG(ERR, "Initialize chain failed");
 951                 goto chain_init_err;
 952         }
 953
 954         err = api_cmd_create_cells(chain);
 955         if (err) {
 956                 PMD_DRV_LOG(ERR, "Create cells for API CMD chain failed");
 957                 goto create_cells_err;
 958         }
 959
 960         err = api_cmd_chain_hw_init(chain);
 961         if (err) {
 962                 PMD_DRV_LOG(ERR, "Initialize chain hw info failed");
 963                 goto chain_hw_init_err;
 964         }
 965
 966         *cmd_chain = chain;
 967         return 0;
 968
 969 chain_hw_init_err:
 970         api_cmd_destroy_cells(chain, chain->num_cells);
 971
 972 create_cells_err:
 973         api_chain_free(chain);
 974
 975 chain_init_err:
 976         kfree(chain);
 977         return err;
 978 }
 979
 980 /**
 981  * api_cmd_destroy_chain - destroy API CMD specific chain
 982  * @chain: the API CMD specific chain to destroy
 983  */
 984 static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain)
 985 {
 986         api_cmd_destroy_cells(chain, chain->num_cells);
 987         api_chain_free(chain);
 988         kfree(chain);
 989 }
 990
 991 /**
 992  * hinic_api_cmd_init - Initialize all the API CMD chains
 993  * @hwdev: the hardware interface of a pci function device
 994  * @chain: the API CMD chains that will be initialized
 995  * Return: 0 - success, negative - failure
 996  */
 997 int hinic_api_cmd_init(struct hinic_hwdev *hwdev,
 998                        struct hinic_api_cmd_chain **chain)
 999 {
1000         struct hinic_api_cmd_chain_attr attr;
1001         enum hinic_api_cmd_chain_type chain_type, i;
1002         int err;
1003
1004         attr.hwdev = hwdev;
1005         attr.num_cells  = API_CHAIN_NUM_CELLS;
1006         attr.cell_size  = API_CHAIN_CELL_SIZE;
1007         attr.rsp_size   = API_CHAIN_RSP_DATA_SIZE;
1008
1009         chain_type = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU;
1010         for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) {
1011                 attr.chain_type = chain_type;
1012                 err = api_cmd_create_chain(&chain[chain_type], &attr);
1013                 if (err) {
1014                         PMD_DRV_LOG(ERR, "Create chain %d failed",
1015                                 chain_type);
1016                         goto create_chain_err;
1017                 }
1018         }
1019
1020         return 0;
1021
1022 create_chain_err:
1023         i = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU;
1024         for (; i < chain_type; i++)
1025                 api_cmd_destroy_chain(chain[i]);
1026
1027         return err;
1028 }
1029
1030 /**
1031  * hinic_api_cmd_free - free the API CMD chains
1032  * @chain: the API CMD chains that will be freed
1033  */
1034 void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain)
1035 {
1036         enum hinic_api_cmd_chain_type chain_type;
1037
1038         chain_type = HINIC_API_CMD_WRITE_ASYNC_TO_MGMT_CPU;
1039         for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++)
1040                 api_cmd_destroy_chain(chain[chain_type]);
1041 }