drivers/net/hinic/base/hinic_pmd_hwif.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_bus_pci.h>
   6
   7 #include "hinic_compat.h"
   8 #include "hinic_csr.h"
   9 #include "hinic_pmd_hwdev.h"
  10 #include "hinic_pmd_hwif.h"
  11
  12 #define HINIC_CFG_REGS_BAR      0
  13 #define HINIC_INTR_MSI_BAR      2
  14 #define HINIC_DB_MEM_BAR        4
  15
  16 #define PAGE_SIZE_4K            0x1000
  17 #define PAGE_SIZE_64K           0x10000
  18
  19 #define HINIC_MSIX_CNT_RESEND_TIMER_SHIFT       29
  20 #define HINIC_MSIX_CNT_RESEND_TIMER_MASK        0x7U
  21
  22 #define HINIC_MSIX_CNT_SET(val, member)         \
  23                 (((val) & HINIC_MSIX_CNT_##member##_MASK) << \
  24                 HINIC_MSIX_CNT_##member##_SHIFT)
  25
  26 /**
  27  * hwif_ready - test if the HW initialization passed
  28  * @hwdev: the pointer to the private hardware device object
  29  * Return: 0 - success, negative - failure
  30  */
  31 static int hwif_ready(struct hinic_hwdev *hwdev)
  32 {
  33         u32 addr, attr0, attr1;
  34
  35         addr   = HINIC_CSR_FUNC_ATTR1_ADDR;
  36         attr1  = hinic_hwif_read_reg(hwdev->hwif, addr);
  37         if (!HINIC_AF1_GET(attr1, MGMT_INIT_STATUS))
  38                 return -EBUSY;
  39
  40         addr   = HINIC_CSR_FUNC_ATTR0_ADDR;
  41         attr0  = hinic_hwif_read_reg(hwdev->hwif, addr);
  42         if ((HINIC_AF0_GET(attr0, FUNC_TYPE) == TYPE_VF) &&
  43              !HINIC_AF1_GET(attr1, PF_INIT_STATUS))
  44                 return -EBUSY;
  45
  46         return 0;
  47 }
  48
  49 /**
  50  * set_hwif_attr - set the attributes as members in hwif
  51  * @hwif: the hardware interface of a pci function device
  52  * @attr0: the first attribute that was read from the hw
  53  * @attr1: the second attribute that was read from the hw
  54  * @attr2: the third attribute that was read from the hw
  55  */
  56 static void set_hwif_attr(struct hinic_hwif *hwif, u32 attr0, u32 attr1,
  57                           u32 attr2)
  58 {
  59         hwif->attr.func_global_idx = HINIC_AF0_GET(attr0, FUNC_GLOBAL_IDX);
  60         hwif->attr.port_to_port_idx = HINIC_AF0_GET(attr0, P2P_IDX);
  61         hwif->attr.pci_intf_idx = HINIC_AF0_GET(attr0, PCI_INTF_IDX);
  62         hwif->attr.vf_in_pf = HINIC_AF0_GET(attr0, VF_IN_PF);
  63         hwif->attr.func_type = HINIC_AF0_GET(attr0, FUNC_TYPE);
  64
  65         hwif->attr.ppf_idx = HINIC_AF1_GET(attr1, PPF_IDX);
  66
  67         hwif->attr.num_aeqs = BIT(HINIC_AF1_GET(attr1, AEQS_PER_FUNC));
  68         hwif->attr.num_ceqs = BIT(HINIC_AF1_GET(attr1, CEQS_PER_FUNC));
  69         hwif->attr.num_irqs = BIT(HINIC_AF1_GET(attr1, IRQS_PER_FUNC));
  70         hwif->attr.num_dma_attr = BIT(HINIC_AF1_GET(attr1, DMA_ATTR_PER_FUNC));
  71
  72         hwif->attr.global_vf_id_of_pf = HINIC_AF2_GET(attr2,
  73                                                       GLOBAL_VF_ID_OF_PF);
  74 }
  75
  76 /**
  77  * get_hwif_attr - read and set the attributes as members in hwif
  78  * @hwif: the hardware interface of a pci function device
  79  */
  80 static void get_hwif_attr(struct hinic_hwif *hwif)
  81 {
  82         u32 addr, attr0, attr1, attr2;
  83
  84         addr   = HINIC_CSR_FUNC_ATTR0_ADDR;
  85         attr0  = hinic_hwif_read_reg(hwif, addr);
  86
  87         addr   = HINIC_CSR_FUNC_ATTR1_ADDR;
  88         attr1  = hinic_hwif_read_reg(hwif, addr);
  89
  90         addr   = HINIC_CSR_FUNC_ATTR2_ADDR;
  91         attr2  = hinic_hwif_read_reg(hwif, addr);
  92
  93         set_hwif_attr(hwif, attr0, attr1, attr2);
  94 }
  95
  96 void hinic_set_pf_status(struct hinic_hwif *hwif, enum hinic_pf_status status)
  97 {
  98         u32 attr5 = HINIC_AF5_SET(status, PF_STATUS);
  99         u32 addr  = HINIC_CSR_FUNC_ATTR5_ADDR;
 100
 101         if (hwif->attr.func_type == TYPE_VF) {
 102                 PMD_DRV_LOG(ERR, "VF doesn't support set attr5");
 103                 return;
 104         }
 105
 106         hinic_hwif_write_reg(hwif, addr, attr5);
 107 }
 108
 109 enum hinic_pf_status hinic_get_pf_status(struct hinic_hwif *hwif)
 110 {
 111         u32 attr5 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR5_ADDR);
 112
 113         return HINIC_AF5_GET(attr5, PF_STATUS);
 114 }
 115
 116 static enum hinic_doorbell_ctrl
 117 hinic_get_doorbell_ctrl_status(struct hinic_hwif *hwif)
 118 {
 119         u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR);
 120
 121         return HINIC_AF4_GET(attr4, DOORBELL_CTRL);
 122 }
 123
 124 static enum hinic_outbound_ctrl
 125 hinic_get_outbound_ctrl_status(struct hinic_hwif *hwif)
 126 {
 127         u32 attr4 = hinic_hwif_read_reg(hwif, HINIC_CSR_FUNC_ATTR4_ADDR);
 128
 129         return HINIC_AF4_GET(attr4, OUTBOUND_CTRL);
 130 }
 131
 132 void hinic_enable_doorbell(struct hinic_hwif *hwif)
 133 {
 134         u32 addr, attr4;
 135
 136         addr = HINIC_CSR_FUNC_ATTR4_ADDR;
 137         attr4 = hinic_hwif_read_reg(hwif, addr);
 138
 139         attr4 = HINIC_AF4_CLEAR(attr4, DOORBELL_CTRL);
 140         attr4 |= HINIC_AF4_SET(ENABLE_DOORBELL, DOORBELL_CTRL);
 141
 142         hinic_hwif_write_reg(hwif, addr, attr4);
 143 }
 144
 145 void hinic_disable_doorbell(struct hinic_hwif *hwif)
 146 {
 147         u32 addr, attr4;
 148
 149         addr = HINIC_CSR_FUNC_ATTR4_ADDR;
 150         attr4 = hinic_hwif_read_reg(hwif, addr);
 151
 152         attr4 = HINIC_AF4_CLEAR(attr4, DOORBELL_CTRL);
 153         attr4 |= HINIC_AF4_SET(DISABLE_DOORBELL, DOORBELL_CTRL);
 154
 155         hinic_hwif_write_reg(hwif, addr, attr4);
 156 }
 157
 158 /**
 159  * set_ppf - try to set hwif as ppf and set the type of hwif in this case
 160  * @hwif: the hardware interface of a pci function device
 161  */
 162 static void set_ppf(struct hinic_hwif *hwif)
 163 {
 164         struct hinic_func_attr *attr = &hwif->attr;
 165         u32 addr, val, ppf_election;
 166
 167         /* Read Modify Write */
 168         addr  = HINIC_CSR_PPF_ELECTION_ADDR;
 169
 170         val = hinic_hwif_read_reg(hwif, addr);
 171         val = HINIC_PPF_ELECTION_CLEAR(val, IDX);
 172
 173         ppf_election =  HINIC_PPF_ELECTION_SET(attr->func_global_idx, IDX);
 174         val |= ppf_election;
 175
 176         hinic_hwif_write_reg(hwif, addr, val);
 177
 178         /* Check PPF */
 179         val = hinic_hwif_read_reg(hwif, addr);
 180
 181         attr->ppf_idx = HINIC_PPF_ELECTION_GET(val, IDX);
 182         if (attr->ppf_idx == attr->func_global_idx)
 183                 attr->func_type = TYPE_PPF;
 184 }
 185
 186 static void init_db_area_idx(struct hinic_hwif *hwif)
 187 {
 188         struct hinic_free_db_area *free_db_area = &hwif->free_db_area;
 189         u32 db_max_areas = hwif->db_max_areas;
 190         u32 i;
 191
 192         for (i = 0; i < db_max_areas; i++)
 193                 free_db_area->db_idx[i] = i;
 194
 195         free_db_area->alloc_pos = 0;
 196         free_db_area->return_pos = 0;
 197
 198         free_db_area->num_free = db_max_areas;
 199
 200         spin_lock_init(&free_db_area->idx_lock);
 201 }
 202
 203 static int get_db_idx(struct hinic_hwif *hwif, u32 *idx)
 204 {
 205         struct hinic_free_db_area *free_db_area = &hwif->free_db_area;
 206         u32 pos;
 207         u32 pg_idx;
 208
 209         spin_lock(&free_db_area->idx_lock);
 210
 211         if (free_db_area->num_free == 0) {
 212                 spin_unlock(&free_db_area->idx_lock);
 213                 return -ENOMEM;
 214         }
 215
 216         free_db_area->num_free--;
 217
 218         pos = free_db_area->alloc_pos++;
 219         pos &= (hwif->db_max_areas - 1);
 220
 221         pg_idx = free_db_area->db_idx[pos];
 222
 223         free_db_area->db_idx[pos] = 0xFFFFFFFF;
 224
 225         spin_unlock(&free_db_area->idx_lock);
 226
 227         *idx = pg_idx;
 228
 229         return 0;
 230 }
 231
 232 static void free_db_idx(struct hinic_hwif *hwif, u32 idx)
 233 {
 234         struct hinic_free_db_area *free_db_area = &hwif->free_db_area;
 235         u32 pos;
 236
 237         spin_lock(&free_db_area->idx_lock);
 238
 239         pos = free_db_area->return_pos++;
 240         pos &= (hwif->db_max_areas - 1);
 241
 242         free_db_area->db_idx[pos] = idx;
 243
 244         free_db_area->num_free++;
 245
 246         spin_unlock(&free_db_area->idx_lock);
 247 }
 248
 249 void hinic_free_db_addr(void *hwdev, void __iomem *db_base)
 250 {
 251         struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
 252         u32 idx = DB_IDX(db_base, hwif->db_base);
 253
 254         free_db_idx(hwif, idx);
 255 }
 256
 257 int hinic_alloc_db_addr(void *hwdev, void __iomem **db_base)
 258 {
 259         struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
 260         u32 idx;
 261         int err;
 262
 263         err = get_db_idx(hwif, &idx);
 264         if (err)
 265                 return -EFAULT;
 266
 267         *db_base = hwif->db_base + idx * HINIC_DB_PAGE_SIZE;
 268
 269         return 0;
 270 }
 271
 272 void hinic_set_msix_state(void *hwdev, u16 msix_idx, enum hinic_msix_state flag)
 273 {
 274         struct hinic_hwdev *hw = hwdev;
 275         struct hinic_hwif *hwif = hw->hwif;
 276         u32 offset = msix_idx * HINIC_PCI_MSIX_ENTRY_SIZE
 277                 + HINIC_PCI_MSIX_ENTRY_VECTOR_CTRL;
 278         u32 mask_bits;
 279
 280         /* vfio-pci does not mmap msi-x vector table to user space,
 281          * we can not access the space when kernel driver is vfio-pci
 282          */
 283         if (hw->pcidev_hdl->kdrv == RTE_KDRV_VFIO)
 284                 return;
 285
 286         mask_bits = readl(hwif->intr_regs_base + offset);
 287         mask_bits &= ~HINIC_PCI_MSIX_ENTRY_CTRL_MASKBIT;
 288         if (flag)
 289                 mask_bits |= HINIC_PCI_MSIX_ENTRY_CTRL_MASKBIT;
 290
 291         writel(mask_bits, hwif->intr_regs_base + offset);
 292 }
 293
 294 static void disable_all_msix(struct hinic_hwdev *hwdev)
 295 {
 296         u16 num_irqs = hwdev->hwif->attr.num_irqs;
 297         u16 i;
 298
 299         for (i = 0; i < num_irqs; i++)
 300                 hinic_set_msix_state(hwdev, i, HINIC_MSIX_DISABLE);
 301 }
 302
 303 /**
 304  * Wait for up enable or disable doorbell flush finished.
 305  * @hwif: the hardware interface of a pci function device.
 306  * @states: Disable or Enable.
 307  */
 308 int wait_until_doorbell_flush_states(struct hinic_hwif *hwif,
 309                                         enum hinic_doorbell_ctrl states)
 310 {
 311         unsigned long end;
 312         enum hinic_doorbell_ctrl db_ctrl;
 313
 314         end = jiffies +
 315                 msecs_to_jiffies(HINIC_WAIT_DOORBELL_AND_OUTBOUND_TIMEOUT);
 316         do {
 317                 db_ctrl = hinic_get_doorbell_ctrl_status(hwif);
 318                 if (db_ctrl == states)
 319                         return 0;
 320
 321                 rte_delay_ms(1);
 322         } while (time_before(jiffies, end));
 323
 324         return -EFAULT;
 325 }
 326
 327 static int wait_until_doorbell_and_outbound_enabled(struct hinic_hwif *hwif)
 328 {
 329         unsigned long end;
 330         enum hinic_doorbell_ctrl db_ctrl;
 331         enum hinic_outbound_ctrl outbound_ctrl;
 332
 333         end = jiffies +
 334                 msecs_to_jiffies(HINIC_WAIT_DOORBELL_AND_OUTBOUND_TIMEOUT);
 335         do {
 336                 db_ctrl = hinic_get_doorbell_ctrl_status(hwif);
 337                 outbound_ctrl = hinic_get_outbound_ctrl_status(hwif);
 338
 339                 if (outbound_ctrl == ENABLE_OUTBOUND &&
 340                     db_ctrl == ENABLE_DOORBELL)
 341                         return 0;
 342
 343                 rte_delay_ms(1);
 344         } while (time_before(jiffies, end));
 345
 346         return -EFAULT;
 347 }
 348
 349 u16 hinic_global_func_id(void *hwdev)
 350 {
 351         struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
 352
 353         return hwif->attr.func_global_idx;
 354 }
 355
 356 enum func_type hinic_func_type(void *hwdev)
 357 {
 358         struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
 359
 360         return hwif->attr.func_type;
 361 }
 362
 363 u8 hinic_ppf_idx(void *hwdev)
 364 {
 365         struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
 366
 367         return hwif->attr.ppf_idx;
 368 }
 369
 370 /**
 371  * hinic_dma_attr_entry_num - get number id of DMA attribute table.
 372  * @hwdev: the pointer to the private hardware device object.
 373  * Return: The number id of DMA attribute table.
 374  */
 375 u8 hinic_dma_attr_entry_num(void *hwdev)
 376 {
 377         struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
 378         return hwif->attr.num_dma_attr;
 379 }
 380
 381 /**
 382  * hinic_init_hwif - initialize the hw interface
 383  * @hwdev: the pointer to the private hardware device object
 384  * @cfg_reg_base: base physical address of configuration registers
 385  * @intr_reg_base: base physical address of msi-x vector table
 386  * @db_base_phy: base physical address of doorbell registers
 387  * @db_base: base virtual address of doorbell registers
 388  * @dwqe_mapping: direct wqe io mapping address
 389  * Return: 0 - success, negative - failure
 390  */
 391 static int hinic_init_hwif(struct hinic_hwdev *hwdev, void *cfg_reg_base,
 392                     void *intr_reg_base, u64 db_base_phy,
 393                     void *db_base, __rte_unused void *dwqe_mapping)
 394 {
 395         struct hinic_hwif *hwif;
 396         struct rte_pci_device *pci_dev;
 397         u64 db_bar_len;
 398         int err;
 399
 400         pci_dev = (struct rte_pci_device *)(hwdev->pcidev_hdl);
 401         db_bar_len = pci_dev->mem_resource[HINIC_DB_MEM_BAR].len;
 402
 403         hwif = hwdev->hwif;
 404
 405         hwif->cfg_regs_base = (u8 __iomem *)cfg_reg_base;
 406         hwif->intr_regs_base = (u8 __iomem *)intr_reg_base;
 407
 408         hwif->db_base_phy = db_base_phy;
 409         hwif->db_base = (u8 __iomem *)db_base;
 410         hwif->db_max_areas = db_bar_len / HINIC_DB_PAGE_SIZE;
 411         if (hwif->db_max_areas > HINIC_DB_MAX_AREAS)
 412                 hwif->db_max_areas = HINIC_DB_MAX_AREAS;
 413
 414         init_db_area_idx(hwif);
 415
 416         get_hwif_attr(hwif);
 417
 418         err = hwif_ready(hwdev);
 419         if (err) {
 420                 PMD_DRV_LOG(ERR, "Hwif is not ready");
 421                 goto hwif_ready_err;
 422         }
 423
 424         err = wait_until_doorbell_and_outbound_enabled(hwif);
 425         if (err) {
 426                 PMD_DRV_LOG(ERR, "Hw doorbell/outbound is disabled");
 427                 goto hwif_ready_err;
 428         }
 429
 430         if (!HINIC_IS_VF(hwdev))
 431                 set_ppf(hwif);
 432
 433         /* disable mgmt cpu report any event */
 434         hinic_set_pf_status(hwdev->hwif, HINIC_PF_STATUS_INIT);
 435
 436         return 0;
 437
 438 hwif_ready_err:
 439         spin_lock_deinit(&hwif->free_db_area.idx_lock);
 440
 441         return err;
 442 }
 443
 444 #define HINIC_HWIF_ATTR_REG_PRINT_NUM        (6)
 445 #define HINIC_HWIF_APICMD_REG_PRINT_NUM      (2)
 446 #define HINIC_HWIF_EQ_REG_PRINT_NUM          (2)
 447
 448 static void hinic_parse_hwif_attr(struct hinic_hwdev *hwdev)
 449 {
 450         struct hinic_hwif *hwif = hwdev->hwif;
 451
 452         PMD_DRV_LOG(INFO, "Device %s hwif attribute:", hwdev->pcidev_hdl->name);
 453         PMD_DRV_LOG(INFO, "func_idx:%u, p2p_idx:%u, pciintf_idx:%u, "
 454                     "vf_in_pf:%u, ppf_idx:%u, global_vf_id:%u, func_type:%u",
 455                     hwif->attr.func_global_idx,
 456                     hwif->attr.port_to_port_idx, hwif->attr.pci_intf_idx,
 457                     hwif->attr.vf_in_pf, hwif->attr.ppf_idx,
 458                     hwif->attr.global_vf_id_of_pf, hwif->attr.func_type);
 459         PMD_DRV_LOG(INFO, "num_aeqs:%u, num_ceqs:%u, num_irqs:%u, dma_attr:%u",
 460                     hwif->attr.num_aeqs, hwif->attr.num_ceqs,
 461                     hwif->attr.num_irqs, hwif->attr.num_dma_attr);
 462 }
 463
 464 static void hinic_get_mmio(struct hinic_hwdev *hwdev, void **cfg_regs_base,
 465                            void **intr_base, void **db_base)
 466 {
 467         struct rte_pci_device *pci_dev = hwdev->pcidev_hdl;
 468         uint64_t bar0_size;
 469         uint64_t bar2_size;
 470         uint64_t bar0_phy_addr;
 471         uint64_t pagesize = sysconf(_SC_PAGESIZE);
 472
 473         *cfg_regs_base = pci_dev->mem_resource[HINIC_CFG_REGS_BAR].addr;
 474         *intr_base = pci_dev->mem_resource[HINIC_INTR_MSI_BAR].addr;
 475         *db_base = pci_dev->mem_resource[HINIC_DB_MEM_BAR].addr;
 476
 477         bar0_size = pci_dev->mem_resource[HINIC_CFG_REGS_BAR].len;
 478         bar2_size = pci_dev->mem_resource[HINIC_INTR_MSI_BAR].len;
 479
 480         if (pagesize == PAGE_SIZE_64K && (bar0_size % pagesize != 0)) {
 481                 bar0_phy_addr =
 482                         pci_dev->mem_resource[HINIC_CFG_REGS_BAR].phys_addr;
 483                 if (bar0_phy_addr % pagesize != 0 &&
 484                 (bar0_size + bar2_size <= pagesize) &&
 485                 bar2_size >= bar0_size) {
 486                         *cfg_regs_base = (void *)((uint8_t *)(*intr_base)
 487                                 + bar2_size);
 488                 }
 489         }
 490 }
 491
 492 void hinic_hwif_res_free(struct hinic_hwdev *hwdev)
 493 {
 494         rte_free(hwdev->hwif);
 495         hwdev->hwif = NULL;
 496 }
 497
 498 int hinic_hwif_res_init(struct hinic_hwdev *hwdev)
 499 {
 500         int err = HINIC_ERROR;
 501         void *cfg_regs_base, *db_base, *intr_base = NULL;
 502
 503         /* hinic related init */
 504         hwdev->hwif = rte_zmalloc("hinic_hwif", sizeof(*hwdev->hwif),
 505                                   RTE_CACHE_LINE_SIZE);
 506         if (!hwdev->hwif) {
 507                 PMD_DRV_LOG(ERR, "Allocate hwif failed, dev_name: %s",
 508                             hwdev->pcidev_hdl->name);
 509                 return -ENOMEM;
 510         }
 511
 512         hinic_get_mmio(hwdev, &cfg_regs_base, &intr_base, &db_base);
 513
 514         err = hinic_init_hwif(hwdev, cfg_regs_base,
 515                               intr_base, 0, db_base, NULL);
 516         if (err) {
 517                 PMD_DRV_LOG(ERR, "Initialize hwif failed, dev_name: %s",
 518                             hwdev->pcidev_hdl->name);
 519                 goto init_hwif_err;
 520         }
 521
 522         /* disable msix interrupt in hw device */
 523         disable_all_msix(hwdev);
 524
 525         /* print hwif attributes */
 526         hinic_parse_hwif_attr(hwdev);
 527
 528         return HINIC_OK;
 529
 530 init_hwif_err:
 531         rte_free(hwdev->hwif);
 532         hwdev->hwif = NULL;
 533
 534         return err;
 535 }
 536
 537 /**
 538  * hinic_misx_intr_clear_resend_bit - clear interrupt resend configuration
 539  * @hwdev: the hardware interface of a nic device
 540  * @msix_idx: Index of msix interrupt
 541  * @clear_resend_en: enable flag of clear resend configuration
 542  */
 543 void hinic_misx_intr_clear_resend_bit(void *hwdev, u16 msix_idx,
 544                                       u8 clear_resend_en)
 545 {
 546         struct hinic_hwif *hwif = ((struct hinic_hwdev *)hwdev)->hwif;
 547         u32 msix_ctrl = 0, addr;
 548
 549         msix_ctrl = HINIC_MSIX_CNT_SET(clear_resend_en, RESEND_TIMER);
 550
 551         addr = HINIC_CSR_MSIX_CNT_ADDR(msix_idx);
 552
 553         hinic_hwif_write_reg(hwif, addr, msix_ctrl);
 554 }