drivers/net/hinic/base/hinic_pmd_nicio.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4 #include<rte_bus_pci.h>
   5
   6 #include "hinic_compat.h"
   7 #include "hinic_pmd_hwdev.h"
   8 #include "hinic_pmd_hwif.h"
   9 #include "hinic_pmd_wq.h"
  10 #include "hinic_pmd_mgmt.h"
  11 #include "hinic_pmd_cmdq.h"
  12 #include "hinic_pmd_cfg.h"
  13 #include "hinic_pmd_niccfg.h"
  14 #include "hinic_pmd_nicio.h"
  15
  16 #define WQ_PREFETCH_MAX                 6
  17 #define WQ_PREFETCH_MIN                 1
  18 #define WQ_PREFETCH_THRESHOLD           256
  19
  20 #define DEFAULT_RX_BUF_SIZE             ((u16)0xB)
  21
  22 enum {
  23         RECYCLE_MODE_NIC = 0x0,
  24         RECYCLE_MODE_DPDK = 0x1,
  25 };
  26
  27 /* Queue buffer related define */
  28 enum hinic_rx_buf_size {
  29         HINIC_RX_BUF_SIZE_32B = 0x20,
  30         HINIC_RX_BUF_SIZE_64B = 0x40,
  31         HINIC_RX_BUF_SIZE_96B = 0x60,
  32         HINIC_RX_BUF_SIZE_128B = 0x80,
  33         HINIC_RX_BUF_SIZE_192B = 0xC0,
  34         HINIC_RX_BUF_SIZE_256B = 0x100,
  35         HINIC_RX_BUF_SIZE_384B = 0x180,
  36         HINIC_RX_BUF_SIZE_512B = 0x200,
  37         HINIC_RX_BUF_SIZE_768B = 0x300,
  38         HINIC_RX_BUF_SIZE_1K = 0x400,
  39         HINIC_RX_BUF_SIZE_1_5K = 0x600,
  40         HINIC_RX_BUF_SIZE_2K = 0x800,
  41         HINIC_RX_BUF_SIZE_3K = 0xC00,
  42         HINIC_RX_BUF_SIZE_4K = 0x1000,
  43         HINIC_RX_BUF_SIZE_8K = 0x2000,
  44         HINIC_RX_BUF_SIZE_16K = 0x4000,
  45 };
  46
  47 const u32 hinic_hw_rx_buf_size[] = {
  48         HINIC_RX_BUF_SIZE_32B,
  49         HINIC_RX_BUF_SIZE_64B,
  50         HINIC_RX_BUF_SIZE_96B,
  51         HINIC_RX_BUF_SIZE_128B,
  52         HINIC_RX_BUF_SIZE_192B,
  53         HINIC_RX_BUF_SIZE_256B,
  54         HINIC_RX_BUF_SIZE_384B,
  55         HINIC_RX_BUF_SIZE_512B,
  56         HINIC_RX_BUF_SIZE_768B,
  57         HINIC_RX_BUF_SIZE_1K,
  58         HINIC_RX_BUF_SIZE_1_5K,
  59         HINIC_RX_BUF_SIZE_2K,
  60         HINIC_RX_BUF_SIZE_3K,
  61         HINIC_RX_BUF_SIZE_4K,
  62         HINIC_RX_BUF_SIZE_8K,
  63         HINIC_RX_BUF_SIZE_16K,
  64 };
  65
  66 struct hinic_qp_ctxt_header {
  67         u16     num_queues;
  68         u16     queue_type;
  69         u32     addr_offset;
  70 };
  71
  72 struct hinic_sq_ctxt {
  73         u32     ceq_attr;
  74
  75         u32     ci_owner;
  76
  77         u32     wq_pfn_hi;
  78         u32     wq_pfn_lo;
  79
  80         u32     pref_cache;
  81         u32     pref_owner;
  82         u32     pref_wq_pfn_hi_ci;
  83         u32     pref_wq_pfn_lo;
  84
  85         u32     rsvd8;
  86         u32     rsvd9;
  87
  88         u32     wq_block_pfn_hi;
  89         u32     wq_block_pfn_lo;
  90 };
  91
  92 struct hinic_rq_ctxt {
  93         u32     ceq_attr;
  94
  95         u32     pi_intr_attr;
  96
  97         u32     wq_pfn_hi_ci;
  98         u32     wq_pfn_lo;
  99
 100         u32     pref_cache;
 101         u32     pref_owner;
 102
 103         u32     pref_wq_pfn_hi_ci;
 104         u32     pref_wq_pfn_lo;
 105
 106         u32     pi_paddr_hi;
 107         u32     pi_paddr_lo;
 108
 109         u32     wq_block_pfn_hi;
 110         u32     wq_block_pfn_lo;
 111 };
 112
 113 struct hinic_sq_ctxt_block {
 114         struct hinic_qp_ctxt_header     cmdq_hdr;
 115         struct hinic_sq_ctxt            sq_ctxt[HINIC_Q_CTXT_MAX];
 116 };
 117
 118 struct hinic_rq_ctxt_block {
 119         struct hinic_qp_ctxt_header     cmdq_hdr;
 120         struct hinic_rq_ctxt            rq_ctxt[HINIC_Q_CTXT_MAX];
 121 };
 122
 123 struct hinic_clean_queue_ctxt {
 124         struct hinic_qp_ctxt_header     cmdq_hdr;
 125         u32                             ctxt_size;
 126 };
 127
 128
 129 static void
 130 hinic_qp_prepare_cmdq_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr,
 131                              enum hinic_qp_ctxt_type ctxt_type,
 132                              u16 num_queues, u16 max_queues, u16 q_id)
 133 {
 134         qp_ctxt_hdr->queue_type = ctxt_type;
 135         qp_ctxt_hdr->num_queues = num_queues;
 136
 137         if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ)
 138                 qp_ctxt_hdr->addr_offset =
 139                                 SQ_CTXT_OFFSET(max_queues, max_queues, q_id);
 140         else
 141                 qp_ctxt_hdr->addr_offset =
 142                                 RQ_CTXT_OFFSET(max_queues, max_queues, q_id);
 143
 144         qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset);
 145
 146         hinic_cpu_to_be32(qp_ctxt_hdr, sizeof(*qp_ctxt_hdr));
 147 }
 148
 149 static void hinic_sq_prepare_ctxt(struct hinic_sq *sq, u16 global_qpn,
 150                            struct hinic_sq_ctxt *sq_ctxt)
 151 {
 152         struct hinic_wq *wq = sq->wq;
 153         u64 wq_page_addr;
 154         u64 wq_page_pfn, wq_block_pfn;
 155         u32 wq_page_pfn_hi, wq_page_pfn_lo;
 156         u32 wq_block_pfn_hi, wq_block_pfn_lo;
 157         u16 pi_start, ci_start;
 158
 159         ci_start = (u16)(wq->cons_idx);
 160         pi_start = (u16)(wq->prod_idx);
 161
 162         /* read the first page from the HW table */
 163         wq_page_addr = wq->queue_buf_paddr;
 164
 165         wq_page_pfn = WQ_PAGE_PFN(wq_page_addr);
 166         wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
 167         wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
 168
 169         wq_block_pfn = WQ_BLOCK_PFN(wq_page_addr);
 170         wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
 171         wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
 172
 173         /* must config as ceq disabled */
 174         sq_ctxt->ceq_attr = SQ_CTXT_CEQ_ATTR_SET(global_qpn, GLOBAL_SQ_ID) |
 175                                 SQ_CTXT_CEQ_ATTR_SET(0, ARM) |
 176                                 SQ_CTXT_CEQ_ATTR_SET(0, CEQ_ID) |
 177                                 SQ_CTXT_CEQ_ATTR_SET(0, EN);
 178
 179         sq_ctxt->ci_owner = SQ_CTXT_CI_SET(ci_start, IDX) |
 180                                 SQ_CTXT_CI_SET(1, OWNER);
 181
 182         sq_ctxt->wq_pfn_hi =
 183                         SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
 184                         SQ_CTXT_WQ_PAGE_SET(pi_start, PI);
 185
 186         sq_ctxt->wq_pfn_lo = wq_page_pfn_lo;
 187
 188         sq_ctxt->pref_cache =
 189                 SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
 190                 SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
 191                 SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
 192
 193         sq_ctxt->pref_owner = 1;
 194
 195         sq_ctxt->pref_wq_pfn_hi_ci =
 196                 SQ_CTXT_PREF_SET(ci_start, CI) |
 197                 SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_PFN_HI);
 198
 199         sq_ctxt->pref_wq_pfn_lo = wq_page_pfn_lo;
 200
 201         sq_ctxt->wq_block_pfn_hi =
 202                 SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, PFN_HI);
 203
 204         sq_ctxt->wq_block_pfn_lo = wq_block_pfn_lo;
 205
 206         hinic_cpu_to_be32(sq_ctxt, sizeof(*sq_ctxt));
 207 }
 208
 209 static void hinic_rq_prepare_ctxt(struct hinic_rq *rq,
 210                         struct hinic_rq_ctxt *rq_ctxt)
 211 {
 212         struct hinic_wq *wq = rq->wq;
 213         u64 wq_page_addr;
 214         u64 wq_page_pfn, wq_block_pfn;
 215         u32 wq_page_pfn_hi, wq_page_pfn_lo;
 216         u32 wq_block_pfn_hi, wq_block_pfn_lo;
 217         u16 pi_start, ci_start;
 218
 219         ci_start = (u16)(wq->cons_idx);
 220         pi_start = (u16)(wq->prod_idx);
 221
 222         /* read the first page from the HW table */
 223         wq_page_addr = wq->queue_buf_paddr;
 224
 225         wq_page_pfn = WQ_PAGE_PFN(wq_page_addr);
 226         wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
 227         wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
 228
 229         wq_block_pfn = WQ_BLOCK_PFN(wq_page_addr);
 230         wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
 231         wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
 232
 233         /* config as ceq disable, but must set msix state disable */
 234         rq_ctxt->ceq_attr = RQ_CTXT_CEQ_ATTR_SET(0, EN) |
 235                             RQ_CTXT_CEQ_ATTR_SET(1, OWNER);
 236
 237         rq_ctxt->pi_intr_attr = RQ_CTXT_PI_SET(pi_start, IDX) |
 238                                 RQ_CTXT_PI_SET(rq->msix_entry_idx, INTR) |
 239                                 RQ_CTXT_PI_SET(0, CEQ_ARM);
 240
 241         rq_ctxt->wq_pfn_hi_ci = RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
 242                                 RQ_CTXT_WQ_PAGE_SET(ci_start, CI);
 243
 244         rq_ctxt->wq_pfn_lo = wq_page_pfn_lo;
 245
 246         rq_ctxt->pref_cache =
 247                 RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
 248                 RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
 249                 RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
 250
 251         rq_ctxt->pref_owner = 1;
 252
 253         rq_ctxt->pref_wq_pfn_hi_ci =
 254                 RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_PFN_HI) |
 255                 RQ_CTXT_PREF_SET(ci_start, CI);
 256
 257         rq_ctxt->pref_wq_pfn_lo = wq_page_pfn_lo;
 258
 259         rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr);
 260         rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr);
 261
 262         rq_ctxt->wq_block_pfn_hi =
 263                 RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, PFN_HI);
 264
 265         rq_ctxt->wq_block_pfn_lo = wq_block_pfn_lo;
 266
 267         hinic_cpu_to_be32(rq_ctxt, sizeof(*rq_ctxt));
 268 }
 269
 270 static int init_sq_ctxts(struct hinic_nic_io *nic_io)
 271 {
 272         struct hinic_hwdev *hwdev = nic_io->hwdev;
 273         struct hinic_sq_ctxt_block *sq_ctxt_block;
 274         struct hinic_sq_ctxt *sq_ctxt;
 275         struct hinic_cmd_buf *cmd_buf;
 276         struct hinic_qp *qp;
 277         u64 out_param = EIO;
 278         u16 q_id, curr_id, global_qpn, max_ctxts, i;
 279         int err = 0;
 280
 281         cmd_buf = hinic_alloc_cmd_buf(hwdev);
 282         if (!cmd_buf) {
 283                 PMD_DRV_LOG(ERR, "Failed to allocate cmd buf");
 284                 return -ENOMEM;
 285         }
 286
 287         q_id = 0;
 288         /* sq and rq number may not equal */
 289         while (q_id < nic_io->num_sqs) {
 290                 sq_ctxt_block = cmd_buf->buf;
 291                 sq_ctxt = sq_ctxt_block->sq_ctxt;
 292
 293                 max_ctxts = (nic_io->num_sqs - q_id) > HINIC_Q_CTXT_MAX ?
 294                                 HINIC_Q_CTXT_MAX : (nic_io->num_sqs - q_id);
 295
 296                 hinic_qp_prepare_cmdq_header(&sq_ctxt_block->cmdq_hdr,
 297                                              HINIC_QP_CTXT_TYPE_SQ, max_ctxts,
 298                                              nic_io->max_qps, q_id);
 299
 300                 for (i = 0; i < max_ctxts; i++) {
 301                         curr_id = q_id + i;
 302                         qp = &nic_io->qps[curr_id];
 303                         global_qpn = nic_io->global_qpn + curr_id;
 304
 305                         hinic_sq_prepare_ctxt(&qp->sq, global_qpn, &sq_ctxt[i]);
 306                 }
 307
 308                 cmd_buf->size = SQ_CTXT_SIZE(max_ctxts);
 309
 310                 err = hinic_cmdq_direct_resp(hwdev, HINIC_ACK_TYPE_CMDQ,
 311                                              HINIC_MOD_L2NIC,
 312                                              HINIC_UCODE_CMD_MDY_QUEUE_CONTEXT,
 313                                              cmd_buf, &out_param, 0);
 314                 if (err || out_param != 0) {
 315                         PMD_DRV_LOG(ERR, "Failed to set SQ ctxts, err: %d",
 316                                 err);
 317                         err = -EIO;
 318                         break;
 319                 }
 320
 321                 q_id += max_ctxts;
 322         }
 323
 324         hinic_free_cmd_buf(hwdev, cmd_buf);
 325
 326         return err;
 327 }
 328
 329 static int init_rq_ctxts(struct hinic_nic_io *nic_io)
 330 {
 331         struct hinic_hwdev *hwdev = nic_io->hwdev;
 332         struct hinic_rq_ctxt_block *rq_ctxt_block;
 333         struct hinic_rq_ctxt *rq_ctxt;
 334         struct hinic_cmd_buf *cmd_buf;
 335         struct hinic_qp *qp;
 336         u64 out_param = 0;
 337         u16 q_id, curr_id, max_ctxts, i;
 338         int err = 0;
 339
 340         cmd_buf = hinic_alloc_cmd_buf(hwdev);
 341         if (!cmd_buf) {
 342                 PMD_DRV_LOG(ERR, "Failed to allocate cmd buf");
 343                 return -ENOMEM;
 344         }
 345
 346         q_id = 0;
 347         /* sq and rq number may not equal */
 348         while (q_id < nic_io->num_rqs) {
 349                 rq_ctxt_block = cmd_buf->buf;
 350                 rq_ctxt = rq_ctxt_block->rq_ctxt;
 351
 352                 max_ctxts = (nic_io->num_rqs - q_id) > HINIC_Q_CTXT_MAX ?
 353                                 HINIC_Q_CTXT_MAX : (nic_io->num_rqs - q_id);
 354
 355                 hinic_qp_prepare_cmdq_header(&rq_ctxt_block->cmdq_hdr,
 356                                              HINIC_QP_CTXT_TYPE_RQ, max_ctxts,
 357                                              nic_io->max_qps, q_id);
 358
 359                 for (i = 0; i < max_ctxts; i++) {
 360                         curr_id = q_id + i;
 361                         qp = &nic_io->qps[curr_id];
 362
 363                         hinic_rq_prepare_ctxt(&qp->rq, &rq_ctxt[i]);
 364                 }
 365
 366                 cmd_buf->size = RQ_CTXT_SIZE(max_ctxts);
 367
 368                 err = hinic_cmdq_direct_resp(hwdev, HINIC_ACK_TYPE_CMDQ,
 369                                              HINIC_MOD_L2NIC,
 370                                              HINIC_UCODE_CMD_MDY_QUEUE_CONTEXT,
 371                                              cmd_buf, &out_param, 0);
 372                 if ((err) || out_param != 0) {
 373                         PMD_DRV_LOG(ERR, "Failed to set RQ ctxts");
 374                         err = -EIO;
 375                         break;
 376                 }
 377
 378                 q_id += max_ctxts;
 379         }
 380
 381         hinic_free_cmd_buf(hwdev, cmd_buf);
 382
 383         return err;
 384 }
 385
 386 static int init_qp_ctxts(struct hinic_nic_io *nic_io)
 387 {
 388         return (init_sq_ctxts(nic_io) || init_rq_ctxts(nic_io));
 389 }
 390
 391 static int clean_queue_offload_ctxt(struct hinic_nic_io *nic_io,
 392                                     enum hinic_qp_ctxt_type ctxt_type)
 393 {
 394         struct hinic_hwdev *hwdev = nic_io->hwdev;
 395         struct hinic_clean_queue_ctxt *ctxt_block;
 396         struct hinic_cmd_buf *cmd_buf;
 397         u64 out_param = 0;
 398         int err;
 399
 400         cmd_buf = hinic_alloc_cmd_buf(hwdev);
 401         if (!cmd_buf) {
 402                 PMD_DRV_LOG(ERR, "Failed to allocate cmd buf");
 403                 return -ENOMEM;
 404         }
 405
 406         ctxt_block = cmd_buf->buf;
 407         ctxt_block->cmdq_hdr.num_queues = nic_io->max_qps;
 408         ctxt_block->cmdq_hdr.queue_type = ctxt_type;
 409         ctxt_block->cmdq_hdr.addr_offset = 0;
 410
 411         /* TSO/LRO ctxt size: 0x0:0B; 0x1:160B; 0x2:200B; 0x3:240B */
 412         ctxt_block->ctxt_size = 0x3;
 413
 414         hinic_cpu_to_be32(ctxt_block, sizeof(*ctxt_block));
 415
 416         cmd_buf->size = sizeof(*ctxt_block);
 417
 418         err = hinic_cmdq_direct_resp(hwdev, HINIC_ACK_TYPE_CMDQ,
 419                                      HINIC_MOD_L2NIC,
 420                                      HINIC_UCODE_CMD_CLEAN_QUEUE_CONTEXT,
 421                                      cmd_buf, &out_param, 0);
 422
 423         if ((err) || (out_param)) {
 424                 PMD_DRV_LOG(ERR, "Failed to clean queue offload ctxts");
 425                 err = -EIO;
 426         }
 427
 428         hinic_free_cmd_buf(hwdev, cmd_buf);
 429
 430         return err;
 431 }
 432
 433 static int clean_qp_offload_ctxt(struct hinic_nic_io *nic_io)
 434 {
 435         /* clean LRO/TSO context space */
 436         return (clean_queue_offload_ctxt(nic_io, HINIC_QP_CTXT_TYPE_SQ) ||
 437                 clean_queue_offload_ctxt(nic_io, HINIC_QP_CTXT_TYPE_RQ));
 438 }
 439
 440 /**
 441  * get_hw_rx_buf_size - translate rx_buf_size into hw_rx_buf_size
 442  * @rx_buf_sz: receive buffer size
 443  * @return
 444  *   hw rx buffer size
 445  */
 446 static u16 get_hw_rx_buf_size(u32 rx_buf_sz)
 447 {
 448         u16 num_hw_types = sizeof(hinic_hw_rx_buf_size)
 449                            / sizeof(hinic_hw_rx_buf_size[0]);
 450         u16 i;
 451
 452         for (i = 0; i < num_hw_types; i++) {
 453                 if (hinic_hw_rx_buf_size[i] == rx_buf_sz)
 454                         return i;
 455         }
 456
 457         PMD_DRV_LOG(ERR, "Hw can't support rx buf size of %u", rx_buf_sz);
 458
 459         return DEFAULT_RX_BUF_SIZE;     /* default 2K */
 460 }
 461
 462 /**
 463  * hinic_set_root_ctxt - init root context in NIC
 464  * @hwdev: the hardware interface of a nic device
 465  * @rq_depth: the depth of receive queue
 466  * @sq_depth: the depth of transmit queue
 467  * @rx_buf_sz: receive buffer size from app
 468  * Return: 0 on success, negative error value otherwise.
 469  */
 470 static int
 471 hinic_set_root_ctxt(void *hwdev, u16 rq_depth, u16 sq_depth, int rx_buf_sz)
 472 {
 473         struct hinic_root_ctxt root_ctxt;
 474         u16 out_size = sizeof(root_ctxt);
 475         int err;
 476
 477         memset(&root_ctxt, 0, sizeof(root_ctxt));
 478         root_ctxt.mgmt_msg_head.resp_aeq_num = HINIC_AEQ1;
 479         root_ctxt.func_idx = hinic_global_func_id(hwdev);
 480         root_ctxt.ppf_idx = hinic_ppf_idx(hwdev);
 481         root_ctxt.set_cmdq_depth = 0;
 482         root_ctxt.cmdq_depth = 0;
 483         root_ctxt.lro_en = 1;
 484         root_ctxt.rq_depth  = (u16)ilog2(rq_depth);
 485         root_ctxt.rx_buf_sz = get_hw_rx_buf_size(rx_buf_sz);
 486         root_ctxt.sq_depth  = (u16)ilog2(sq_depth);
 487
 488         err = hinic_msg_to_mgmt_sync(hwdev, HINIC_MOD_COMM,
 489                                      HINIC_MGMT_CMD_VAT_SET,
 490                                      &root_ctxt, sizeof(root_ctxt),
 491                                      &root_ctxt, &out_size, 0);
 492         if (err || !out_size || root_ctxt.mgmt_msg_head.status) {
 493                 PMD_DRV_LOG(ERR,
 494                         "Set root context failed, err: %d, status: 0x%x, out_size: 0x%x",
 495                         err, root_ctxt.mgmt_msg_head.status, out_size);
 496                 return -EIO;
 497         }
 498
 499         return 0;
 500 }
 501
 502 /**
 503  * hinic_clean_root_ctxt - clean root context table in NIC
 504  * @hwdev: the hardware interface of a nic device
 505  * @return
 506  *   0 on success,
 507  *   negative error value otherwise.
 508  */
 509 static int hinic_clean_root_ctxt(void *hwdev)
 510 {
 511         struct hinic_root_ctxt root_ctxt;
 512         u16 out_size = sizeof(root_ctxt);
 513         int err;
 514
 515         memset(&root_ctxt, 0, sizeof(root_ctxt));
 516         root_ctxt.mgmt_msg_head.resp_aeq_num = HINIC_AEQ1;
 517         root_ctxt.func_idx = hinic_global_func_id(hwdev);
 518         root_ctxt.ppf_idx = hinic_ppf_idx(hwdev);
 519         root_ctxt.set_cmdq_depth = 0;
 520         root_ctxt.cmdq_depth = 0;
 521         root_ctxt.lro_en = 0;
 522         root_ctxt.rq_depth  = 0;
 523         root_ctxt.rx_buf_sz = 0;
 524         root_ctxt.sq_depth  = 0;
 525
 526         err = hinic_msg_to_mgmt_sync(hwdev, HINIC_MOD_COMM,
 527                                      HINIC_MGMT_CMD_VAT_SET,
 528                                      &root_ctxt, sizeof(root_ctxt),
 529                                      &root_ctxt, &out_size, 0);
 530         if (err || !out_size || root_ctxt.mgmt_msg_head.status) {
 531                 PMD_DRV_LOG(ERR,
 532                         "Clean root context failed, err: %d, status: 0x%x, out_size: 0x%x",
 533                         err, root_ctxt.mgmt_msg_head.status, out_size);
 534                 return -EIO;
 535         }
 536
 537         return 0;
 538 }
 539
 540 /* init qps ctxt and set sq ci attr and arm all sq and set vat page_size */
 541 int hinic_init_qp_ctxts(struct hinic_hwdev *hwdev)
 542 {
 543         struct hinic_nic_io *nic_io = hwdev->nic_io;
 544         struct hinic_sq_attr sq_attr;
 545         u16 q_id;
 546         int err, rx_buf_sz;
 547
 548         /* set vat page size to max queue depth page_size */
 549         err = hinic_set_pagesize(hwdev, HINIC_PAGE_SIZE_DPDK);
 550         if (err != HINIC_OK) {
 551                 PMD_DRV_LOG(ERR, "Set vat page size: %d failed, rc: %d",
 552                         HINIC_PAGE_SIZE_DPDK, err);
 553                 return err;
 554         }
 555
 556         if (hwdev->cmdqs->status & HINIC_CMDQ_SET_FAIL) {
 557                 err = hinic_reinit_cmdq_ctxts(hwdev);
 558                 if (err) {
 559                         PMD_DRV_LOG(ERR, "Reinit cmdq context failed when dev start, err: %d",
 560                                 err);
 561                         return err;
 562                 }
 563         }
 564
 565         err = init_qp_ctxts(nic_io);
 566         if (err) {
 567                 PMD_DRV_LOG(ERR, "Init QP ctxts failed, rc: %d", err);
 568                 return err;
 569         }
 570
 571         /* clean LRO/TSO context space */
 572         err = clean_qp_offload_ctxt(nic_io);
 573         if (err) {
 574                 PMD_DRV_LOG(ERR, "Clean qp offload ctxts failed, rc: %d", err);
 575                 return err;
 576         }
 577
 578         rx_buf_sz = nic_io->rq_buf_size;
 579
 580         /* update rx buf size to function table */
 581         err = hinic_set_rx_vhd_mode(hwdev, HINIC_VHD_TYPE_0B, rx_buf_sz);
 582         if (err) {
 583                 PMD_DRV_LOG(ERR, "Set rx vhd mode failed, rc: %d", err);
 584                 return err;
 585         }
 586
 587         err = hinic_set_root_ctxt(hwdev, nic_io->rq_depth,
 588                                   nic_io->sq_depth, rx_buf_sz);
 589         if (err) {
 590                 PMD_DRV_LOG(ERR, "Set root context failed, rc: %d", err);
 591                 return err;
 592         }
 593
 594         for (q_id = 0; q_id < nic_io->num_sqs; q_id++) {
 595                 sq_attr.ci_dma_base =
 596                         HINIC_CI_PADDR(nic_io->ci_dma_base, q_id) >> 2;
 597                 /* performance: sq ci update threshold as 8 */
 598                 sq_attr.pending_limit = 1;
 599                 sq_attr.coalescing_time = 1;
 600                 sq_attr.intr_en = 0;
 601                 sq_attr.l2nic_sqn = q_id;
 602                 sq_attr.dma_attr_off = 0;
 603                 err = hinic_set_ci_table(hwdev, q_id, &sq_attr);
 604                 if (err) {
 605                         PMD_DRV_LOG(ERR, "Set ci table failed, rc: %d", err);
 606                         goto set_cons_idx_table_err;
 607                 }
 608         }
 609
 610         return 0;
 611
 612 set_cons_idx_table_err:
 613         (void)hinic_clean_root_ctxt(hwdev);
 614         return err;
 615 }
 616
 617 void hinic_free_qp_ctxts(struct hinic_hwdev *hwdev)
 618 {
 619         int err;
 620
 621         err = hinic_clean_root_ctxt(hwdev);
 622         if (err)
 623                 PMD_DRV_LOG(ERR, "Failed to clean root ctxt");
 624 }
 625
 626 static int hinic_init_nic_hwdev(struct hinic_hwdev *hwdev)
 627 {
 628         struct hinic_nic_io *nic_io = hwdev->nic_io;
 629         u16 global_qpn, rx_buf_sz;
 630         int err;
 631
 632         err = hinic_get_base_qpn(hwdev, &global_qpn);
 633         if (err) {
 634                 PMD_DRV_LOG(ERR, "Failed to get base qpn");
 635                 goto err_init_nic_hwdev;
 636         }
 637
 638         nic_io->global_qpn = global_qpn;
 639         rx_buf_sz = HINIC_IS_VF(hwdev) ? RX_BUF_LEN_1_5K : RX_BUF_LEN_16K;
 640         err = hinic_init_function_table(hwdev, rx_buf_sz);
 641         if (err) {
 642                 PMD_DRV_LOG(ERR, "Failed to init function table");
 643                 goto err_init_nic_hwdev;
 644         }
 645
 646         err = hinic_vf_func_init(hwdev);
 647         if (err) {
 648                 PMD_DRV_LOG(ERR, "Failed to init nic mbox");
 649                 goto err_init_nic_hwdev;
 650         }
 651
 652         err = hinic_set_fast_recycle_mode(hwdev, RECYCLE_MODE_DPDK);
 653         if (err) {
 654                 PMD_DRV_LOG(ERR, "Failed to set fast recycle mode");
 655                 goto err_init_nic_hwdev;
 656         }
 657
 658         return 0;
 659
 660 err_init_nic_hwdev:
 661         return err;
 662 }
 663
 664 static void hinic_free_nic_hwdev(struct hinic_hwdev *hwdev)
 665 {
 666         hinic_vf_func_free(hwdev);
 667 }
 668
 669 int hinic_rx_tx_flush(struct hinic_hwdev *hwdev)
 670 {
 671         return hinic_func_rx_tx_flush(hwdev);
 672 }
 673
 674 int hinic_get_sq_free_wqebbs(struct hinic_hwdev *hwdev, u16 q_id)
 675 {
 676         struct hinic_nic_io *nic_io = hwdev->nic_io;
 677         struct hinic_wq *wq = &nic_io->sq_wq[q_id];
 678
 679         return (wq->delta) - 1;
 680 }
 681
 682 int hinic_get_rq_free_wqebbs(struct hinic_hwdev *hwdev, u16 q_id)
 683 {
 684         struct hinic_nic_io *nic_io = hwdev->nic_io;
 685         struct hinic_wq *wq = &nic_io->rq_wq[q_id];
 686
 687         return (wq->delta) - 1;
 688 }
 689
 690 u16 hinic_get_sq_local_ci(struct hinic_hwdev *hwdev, u16 q_id)
 691 {
 692         struct hinic_nic_io *nic_io = hwdev->nic_io;
 693         struct hinic_wq *wq = &nic_io->sq_wq[q_id];
 694
 695         return (wq->cons_idx) & wq->mask;
 696 }
 697
 698 void hinic_return_sq_wqe(struct hinic_hwdev *hwdev, u16 q_id,
 699                          int num_wqebbs, u16 owner)
 700 {
 701         struct hinic_nic_io *nic_io = hwdev->nic_io;
 702         struct hinic_sq *sq = &nic_io->qps[q_id].sq;
 703
 704         if (owner != sq->owner)
 705                 sq->owner = owner;
 706
 707         sq->wq->delta += num_wqebbs;
 708         sq->wq->prod_idx -= num_wqebbs;
 709 }
 710
 711 void hinic_update_sq_local_ci(struct hinic_hwdev *hwdev,
 712                               u16 q_id, int wqebb_cnt)
 713 {
 714         struct hinic_nic_io *nic_io = hwdev->nic_io;
 715         struct hinic_sq *sq = &nic_io->qps[q_id].sq;
 716
 717         hinic_put_wqe(sq->wq, wqebb_cnt);
 718 }
 719
 720 void *hinic_get_rq_wqe(struct hinic_hwdev *hwdev, u16 q_id, u16 *pi)
 721 {
 722         struct hinic_nic_io *nic_io = hwdev->nic_io;
 723         struct hinic_rq *rq = &nic_io->qps[q_id].rq;
 724
 725         return hinic_get_wqe(rq->wq, 1, pi);
 726 }
 727
 728 void hinic_return_rq_wqe(struct hinic_hwdev *hwdev, u16 q_id, int num_wqebbs)
 729 {
 730         struct hinic_nic_io *nic_io = hwdev->nic_io;
 731         struct hinic_rq *rq = &nic_io->qps[q_id].rq;
 732
 733         rq->wq->delta += num_wqebbs;
 734         rq->wq->prod_idx -= num_wqebbs;
 735 }
 736
 737 u16 hinic_get_rq_local_ci(struct hinic_hwdev *hwdev, u16 q_id)
 738 {
 739         struct hinic_nic_io *nic_io = hwdev->nic_io;
 740         struct hinic_wq *wq = &nic_io->rq_wq[q_id];
 741
 742         return (wq->cons_idx) & wq->mask;
 743 }
 744
 745 void hinic_update_rq_local_ci(struct hinic_hwdev *hwdev, u16 q_id, int wqe_cnt)
 746 {
 747         struct hinic_nic_io *nic_io = hwdev->nic_io;
 748         struct hinic_rq *rq = &nic_io->qps[q_id].rq;
 749
 750         hinic_put_wqe(rq->wq, wqe_cnt);
 751 }
 752
 753 static int hinic_alloc_nicio(struct hinic_hwdev *hwdev)
 754 {
 755         struct hinic_nic_io *nic_io = hwdev->nic_io;
 756         struct rte_pci_device *pdev = hwdev->pcidev_hdl;
 757         u16 max_qps, num_qp;
 758         int err;
 759
 760         max_qps = hinic_func_max_qnum(hwdev);
 761
 762         nic_io->max_qps = max_qps;
 763         nic_io->num_qps = max_qps;
 764         num_qp = max_qps;
 765
 766         nic_io->qps = kzalloc_aligned(num_qp * sizeof(*nic_io->qps),
 767                                       GFP_KERNEL);
 768         if (!nic_io->qps) {
 769                 PMD_DRV_LOG(ERR, "Failed to allocate qps");
 770                 err = -ENOMEM;
 771                 goto alloc_qps_err;
 772         }
 773
 774         nic_io->ci_vaddr_base = dma_zalloc_coherent(hwdev,
 775                                     CI_TABLE_SIZE(num_qp, HINIC_PAGE_SIZE),
 776                                     &nic_io->ci_dma_base,
 777                                     pdev->device.numa_node);
 778         if (!nic_io->ci_vaddr_base) {
 779                 PMD_DRV_LOG(ERR, "Failed to allocate ci area");
 780                 err = -ENOMEM;
 781                 goto ci_base_err;
 782         }
 783
 784         nic_io->sq_wq = kzalloc_aligned(num_qp * sizeof(*nic_io->sq_wq),
 785                                         GFP_KERNEL);
 786         if (!nic_io->sq_wq) {
 787                 PMD_DRV_LOG(ERR, "Failed to allocate sq wq array");
 788                 err = -ENOMEM;
 789                 goto sq_wq_err;
 790         }
 791
 792         nic_io->rq_wq = kzalloc_aligned(num_qp * sizeof(*nic_io->rq_wq),
 793                                         GFP_KERNEL);
 794         if (!nic_io->rq_wq) {
 795                 PMD_DRV_LOG(ERR, "Failed to allocate rq wq array");
 796                 err = -ENOMEM;
 797                 goto rq_wq_err;
 798         }
 799
 800         return HINIC_OK;
 801
 802 rq_wq_err:
 803         kfree(nic_io->sq_wq);
 804
 805 sq_wq_err:
 806         dma_free_coherent(hwdev, CI_TABLE_SIZE(num_qp, HINIC_PAGE_SIZE),
 807                           nic_io->ci_vaddr_base, nic_io->ci_dma_base);
 808
 809 ci_base_err:
 810         kfree(nic_io->qps);
 811
 812 alloc_qps_err:
 813         return err;
 814 }
 815
 816 static void hinic_free_nicio(struct hinic_hwdev *hwdev)
 817 {
 818         struct hinic_nic_io *nic_io = hwdev->nic_io;
 819
 820         /* nic_io->rq_wq */
 821         kfree(nic_io->rq_wq);
 822
 823         /* nic_io->sq_wq */
 824         kfree(nic_io->sq_wq);
 825
 826         /* nic_io->ci_vaddr_base */
 827         dma_free_coherent(hwdev,
 828                           CI_TABLE_SIZE(nic_io->max_qps, HINIC_PAGE_SIZE),
 829                           nic_io->ci_vaddr_base, nic_io->ci_dma_base);
 830
 831         /* nic_io->qps */
 832         kfree(nic_io->qps);
 833 }
 834
 835 /* alloc nic hwdev and init function table */
 836 int hinic_init_nicio(struct hinic_hwdev *hwdev)
 837 {
 838         int rc;
 839
 840         hwdev->nic_io = rte_zmalloc("hinic_nicio", sizeof(*hwdev->nic_io),
 841                                       RTE_CACHE_LINE_SIZE);
 842         if (!hwdev->nic_io) {
 843                 PMD_DRV_LOG(ERR, "Allocate nic_io failed, dev_name: %s",
 844                             hwdev->pcidev_hdl->name);
 845                 return -ENOMEM;
 846         }
 847         hwdev->nic_io->hwdev = hwdev;
 848
 849         /* alloc root working queue set */
 850         rc = hinic_alloc_nicio(hwdev);
 851         if (rc) {
 852                 PMD_DRV_LOG(ERR, "Allocate nic_io failed, dev_name: %s",
 853                             hwdev->pcidev_hdl->name);
 854                 goto allc_nicio_fail;
 855         }
 856
 857         rc = hinic_init_nic_hwdev(hwdev);
 858         if (rc) {
 859                 PMD_DRV_LOG(ERR, "Initialize hwdev failed, dev_name: %s",
 860                             hwdev->pcidev_hdl->name);
 861                 goto init_nic_hwdev_fail;
 862         }
 863
 864         return 0;
 865
 866 init_nic_hwdev_fail:
 867         hinic_free_nicio(hwdev);
 868
 869 allc_nicio_fail:
 870         rte_free(hwdev->nic_io);
 871         return rc;
 872 }
 873
 874 void hinic_deinit_nicio(struct hinic_hwdev *hwdev)
 875 {
 876         hinic_free_nicio(hwdev);
 877
 878         hinic_free_nic_hwdev(hwdev);
 879
 880         rte_free(hwdev->nic_io);
 881         hwdev->nic_io = NULL;
 882 }
 883
 884 /**
 885  * hinic_convert_rx_buf_size - convert rx buffer size to hw size
 886  * @rx_buf_sz: receive buffer size of mbuf
 887  * @match_sz: receive buffer size of hardware
 888  * @return
 889  *   0 on success,
 890  *   negative error value otherwise.
 891  */
 892 int hinic_convert_rx_buf_size(u32 rx_buf_sz, u32 *match_sz)
 893 {
 894         u32 i, num_hw_types, best_match_sz;
 895
 896         if (unlikely(!match_sz || rx_buf_sz < HINIC_RX_BUF_SIZE_32B))
 897                 return -EINVAL;
 898
 899         if (rx_buf_sz >= HINIC_RX_BUF_SIZE_16K) {
 900                 best_match_sz =  HINIC_RX_BUF_SIZE_16K;
 901                 goto size_matched;
 902         }
 903
 904         num_hw_types = sizeof(hinic_hw_rx_buf_size) /
 905                 sizeof(hinic_hw_rx_buf_size[0]);
 906         best_match_sz = hinic_hw_rx_buf_size[0];
 907         for (i = 0; i < num_hw_types; i++) {
 908                 if (rx_buf_sz == hinic_hw_rx_buf_size[i]) {
 909                         best_match_sz = hinic_hw_rx_buf_size[i];
 910                         break;
 911                 } else if (rx_buf_sz < hinic_hw_rx_buf_size[i]) {
 912                         break;
 913                 }
 914                 best_match_sz = hinic_hw_rx_buf_size[i];
 915         }
 916
 917 size_matched:
 918         *match_sz = best_match_sz;
 919
 920         return 0;
 921 }