drivers/raw/ntb/ntb_hw_intel.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation.
   3  */
   4 #include <stdint.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7
   8 #include <rte_io.h>
   9 #include <rte_eal.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_rawdev.h>
  13 #include <rte_rawdev_pmd.h>
  14
  15 #include "ntb.h"
  16 #include "ntb_hw_intel.h"
  17
  18 enum xeon_ntb_bar {
  19         XEON_NTB_BAR23 = 2,
  20         XEON_NTB_BAR45 = 4,
  21 };
  22
  23 static enum xeon_ntb_bar intel_ntb_bar[] = {
  24         XEON_NTB_BAR23,
  25         XEON_NTB_BAR45,
  26 };
  27
  28 static inline int
  29 is_gen3_ntb(const struct ntb_hw *hw)
  30 {
  31         if (hw->pci_dev->id.device_id == NTB_INTEL_DEV_ID_B2B_SKX)
  32                 return 1;
  33
  34         return 0;
  35 }
  36
  37 static inline int
  38 is_gen4_ntb(const struct ntb_hw *hw)
  39 {
  40         if (hw->pci_dev->id.device_id == NTB_INTEL_DEV_ID_B2B_ICX)
  41                 return 1;
  42
  43         return 0;
  44 }
  45
  46 static int
  47 intel_ntb3_check_ppd(struct ntb_hw *hw)
  48 {
  49         uint8_t reg_val;
  50         int ret;
  51
  52         ret = rte_pci_read_config(hw->pci_dev, &reg_val,
  53                                   sizeof(reg_val), XEON_PPD_OFFSET);
  54         if (ret < 0) {
  55                 NTB_LOG(ERR, "Cannot get NTB PPD (PCIe port definition).");
  56                 return -EIO;
  57         }
  58
  59         /* Check connection topo type. Only support B2B. */
  60         switch (reg_val & XEON_PPD_CONN_MASK) {
  61         case XEON_PPD_CONN_B2B:
  62                 NTB_LOG(INFO, "Topo B2B (back to back) is using.");
  63                 break;
  64         case XEON_PPD_CONN_TRANSPARENT:
  65         case XEON_PPD_CONN_RP:
  66         default:
  67                 NTB_LOG(ERR, "Not supported conn topo. Please use B2B.");
  68                 return -EINVAL;
  69         }
  70
  71         /* Check device type. */
  72         if (reg_val & XEON_PPD_DEV_DSD) {
  73                 NTB_LOG(INFO, "DSD, Downstream Device.");
  74                 hw->topo = NTB_TOPO_B2B_DSD;
  75         } else {
  76                 NTB_LOG(INFO, "USD, Upstream device.");
  77                 hw->topo = NTB_TOPO_B2B_USD;
  78         }
  79
  80         /* Check if bar4 is split. Do not support split bar. */
  81         if (reg_val & XEON_PPD_SPLIT_BAR_MASK) {
  82                 NTB_LOG(ERR, "Do not support split bar.");
  83                 return -EINVAL;
  84         }
  85
  86         return 0;
  87 }
  88
  89 static int
  90 intel_ntb4_check_ppd(struct ntb_hw *hw)
  91 {
  92         uint32_t reg_val;
  93
  94         reg_val = rte_read32(hw->hw_addr + XEON_GEN4_PPD1_OFFSET);
  95
  96         /* Check connection topo type. Only support B2B. */
  97         switch (reg_val & XEON_GEN4_PPD_CONN_MASK) {
  98         case XEON_GEN4_PPD_CONN_B2B:
  99                 NTB_LOG(INFO, "Topo B2B (back to back) is using.");
 100                 break;
 101         default:
 102                 NTB_LOG(ERR, "Not supported conn topo. Please use B2B.");
 103                 return -EINVAL;
 104         }
 105
 106         /* Check device type. */
 107         if (reg_val & XEON_GEN4_PPD_DEV_DSD) {
 108                 NTB_LOG(INFO, "DSD, Downstream Device.");
 109                 hw->topo = NTB_TOPO_B2B_DSD;
 110         } else {
 111                 NTB_LOG(INFO, "USD, Upstream device.");
 112                 hw->topo = NTB_TOPO_B2B_USD;
 113         }
 114
 115         return 0;
 116 }
 117
 118 static int
 119 intel_ntb_dev_init(const struct rte_rawdev *dev)
 120 {
 121         struct ntb_hw *hw = dev->dev_private;
 122         uint8_t bar;
 123         int ret, i;
 124
 125         if (hw == NULL) {
 126                 NTB_LOG(ERR, "Invalid device.");
 127                 return -EINVAL;
 128         }
 129
 130         hw->hw_addr = (char *)hw->pci_dev->mem_resource[0].addr;
 131
 132         if (is_gen3_ntb(hw))
 133                 ret = intel_ntb3_check_ppd(hw);
 134         else if (is_gen4_ntb(hw))
 135                 /* PPD is in MMIO but not config space for NTB Gen4 */
 136                 ret = intel_ntb4_check_ppd(hw);
 137         else {
 138                 NTB_LOG(ERR, "Cannot init device for unsupported device.");
 139                 return -ENOTSUP;
 140         }
 141
 142         if (ret)
 143                 return ret;
 144
 145         hw->mw_cnt = XEON_MW_COUNT;
 146         hw->db_cnt = XEON_DB_COUNT;
 147         hw->spad_cnt = XEON_SPAD_COUNT;
 148
 149         hw->mw_size = rte_zmalloc("ntb_mw_size",
 150                                   hw->mw_cnt * sizeof(uint64_t), 0);
 151         if (hw->mw_size == NULL) {
 152                 NTB_LOG(ERR, "Cannot allocate memory for mw size.");
 153                 return -ENOMEM;
 154         }
 155
 156         for (i = 0; i < hw->mw_cnt; i++) {
 157                 bar = intel_ntb_bar[i];
 158                 hw->mw_size[i] = hw->pci_dev->mem_resource[bar].len;
 159         }
 160
 161         /* Reserve the last 2 spad registers for users. */
 162         for (i = 0; i < NTB_SPAD_USER_MAX_NUM; i++)
 163                 hw->spad_user_list[i] = hw->spad_cnt;
 164         hw->spad_user_list[0] = hw->spad_cnt - 2;
 165         hw->spad_user_list[1] = hw->spad_cnt - 1;
 166
 167         return 0;
 168 }
 169
 170 static void *
 171 intel_ntb_get_peer_mw_addr(const struct rte_rawdev *dev, int mw_idx)
 172 {
 173         struct ntb_hw *hw = dev->dev_private;
 174         uint8_t bar;
 175
 176         if (hw == NULL) {
 177                 NTB_LOG(ERR, "Invalid device.");
 178                 return 0;
 179         }
 180
 181         if (mw_idx < 0 || mw_idx >= hw->mw_cnt) {
 182                 NTB_LOG(ERR, "Invalid memory window index (0 - %u).",
 183                         hw->mw_cnt - 1);
 184                 return 0;
 185         }
 186
 187         bar = intel_ntb_bar[mw_idx];
 188
 189         return hw->pci_dev->mem_resource[bar].addr;
 190 }
 191
 192 static int
 193 intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,
 194                        uint64_t addr, uint64_t size)
 195 {
 196         struct ntb_hw *hw = dev->dev_private;
 197         void *xlat_addr, *limit_addr;
 198         uint64_t xlat_off, limit_off;
 199         uint64_t base, limit;
 200         uint8_t bar;
 201
 202         if (hw == NULL) {
 203                 NTB_LOG(ERR, "Invalid device.");
 204                 return -EINVAL;
 205         }
 206
 207         if (mw_idx < 0 || mw_idx >= hw->mw_cnt) {
 208                 NTB_LOG(ERR, "Invalid memory window index (0 - %u).",
 209                         hw->mw_cnt - 1);
 210                 return -EINVAL;
 211         }
 212
 213         bar = intel_ntb_bar[mw_idx];
 214
 215         xlat_off = XEON_IMBAR1XBASE_OFFSET + mw_idx * XEON_BAR_INTERVAL_OFFSET;
 216         limit_off = XEON_IMBAR1XLMT_OFFSET + mw_idx * XEON_BAR_INTERVAL_OFFSET;
 217         xlat_addr = hw->hw_addr + xlat_off;
 218         limit_addr = hw->hw_addr + limit_off;
 219
 220         /* Limit reg val should be EMBAR base address plus MW size. */
 221         base = addr;
 222         limit = hw->pci_dev->mem_resource[bar].phys_addr + size;
 223         rte_write64(base, xlat_addr);
 224         rte_write64(limit, limit_addr);
 225
 226         if (is_gen3_ntb(hw)) {
 227                 /* Setup the external point so that remote can access. */
 228                 xlat_off = XEON_EMBAR1_OFFSET + 8 * mw_idx;
 229                 xlat_addr = hw->hw_addr + xlat_off;
 230                 limit_off = XEON_EMBAR1XLMT_OFFSET +
 231                             mw_idx * XEON_BAR_INTERVAL_OFFSET;
 232                 limit_addr = hw->hw_addr + limit_off;
 233                 base = rte_read64(xlat_addr);
 234                 base &= ~0xf;
 235                 limit = base + size;
 236                 rte_write64(limit, limit_addr);
 237         } else if (is_gen4_ntb(hw)) {
 238                 /* Set translate base address index register */
 239                 xlat_off = XEON_GEN4_IM1XBASEIDX_OFFSET +
 240                            mw_idx * XEON_GEN4_XBASEIDX_INTERVAL;
 241                 xlat_addr = hw->hw_addr + xlat_off;
 242                 rte_write16(rte_log2_u64(size), xlat_addr);
 243         } else {
 244                 NTB_LOG(ERR, "Cannot set translation of memory windows for unsupported device.");
 245                 rte_write64(base, limit_addr);
 246                 rte_write64(0, xlat_addr);
 247                 return -ENOTSUP;
 248         }
 249
 250         return 0;
 251 }
 252
 253 static void *
 254 intel_ntb_ioremap(const struct rte_rawdev *dev, uint64_t addr)
 255 {
 256         struct ntb_hw *hw = dev->dev_private;
 257         void *mapped = NULL;
 258         void *base;
 259         int i;
 260
 261         for (i = 0; i < hw->peer_used_mws; i++) {
 262                 if (addr >= hw->peer_mw_base[i] &&
 263                     addr <= hw->peer_mw_base[i] + hw->mw_size[i]) {
 264                         base = intel_ntb_get_peer_mw_addr(dev, i);
 265                         mapped = (void *)(size_t)(addr - hw->peer_mw_base[i] +
 266                                  (size_t)base);
 267                         break;
 268                 }
 269         }
 270
 271         return mapped;
 272 }
 273
 274 static int
 275 intel_ntb_get_link_status(const struct rte_rawdev *dev)
 276 {
 277         struct ntb_hw *hw = dev->dev_private;
 278         uint16_t reg_val, reg_off;
 279         int ret;
 280
 281         if (hw == NULL) {
 282                 NTB_LOG(ERR, "Invalid device.");
 283                 return -EINVAL;
 284         }
 285
 286         if (is_gen3_ntb(hw)) {
 287                 reg_off = XEON_GEN3_LINK_STATUS_OFFSET;
 288                 ret = rte_pci_read_config(hw->pci_dev, &reg_val,
 289                                           sizeof(reg_val), reg_off);
 290                 if (ret < 0) {
 291                         NTB_LOG(ERR, "Unable to get link status.");
 292                         return -EIO;
 293                 }
 294         } else if (is_gen4_ntb(hw)) {
 295                 reg_off = XEON_GEN4_LINK_STATUS_OFFSET;
 296                 reg_val = rte_read16(hw->hw_addr + reg_off);
 297         } else {
 298                 NTB_LOG(ERR, "Cannot get link status for unsupported device.");
 299                 return -ENOTSUP;
 300         }
 301
 302         hw->link_status = NTB_LNK_STA_ACTIVE(reg_val);
 303
 304         if (hw->link_status) {
 305                 hw->link_speed = NTB_LNK_STA_SPEED(reg_val);
 306                 hw->link_width = NTB_LNK_STA_WIDTH(reg_val);
 307         } else {
 308                 hw->link_speed = NTB_SPEED_NONE;
 309                 hw->link_width = NTB_WIDTH_NONE;
 310         }
 311
 312         return 0;
 313 }
 314
 315 static int
 316 intel_ntb_gen3_set_link(const struct ntb_hw *hw, bool up)
 317 {
 318         uint32_t ntb_ctrl, reg_off;
 319         void *reg_addr;
 320
 321         reg_off = XEON_NTBCNTL_OFFSET;
 322         reg_addr = hw->hw_addr + reg_off;
 323         ntb_ctrl = rte_read32(reg_addr);
 324
 325         if (up) {
 326                 ntb_ctrl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
 327                 ntb_ctrl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
 328                 ntb_ctrl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
 329         } else {
 330                 ntb_ctrl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
 331                 ntb_ctrl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
 332                 ntb_ctrl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
 333         }
 334
 335         rte_write32(ntb_ctrl, reg_addr);
 336
 337         return 0;
 338 }
 339
 340 static int
 341 intel_ntb_gen4_set_link(const struct ntb_hw *hw, bool up)
 342 {
 343         uint32_t ntb_ctrl, ppd0;
 344         uint16_t link_ctrl;
 345         void *reg_addr;
 346
 347         if (up) {
 348                 reg_addr = hw->hw_addr + XEON_NTBCNTL_OFFSET;
 349                 ntb_ctrl = NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
 350                 ntb_ctrl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
 351                 rte_write32(ntb_ctrl, reg_addr);
 352
 353                 reg_addr = hw->hw_addr + XEON_GEN4_LINK_CTRL_OFFSET;
 354                 link_ctrl = rte_read16(reg_addr);
 355                 link_ctrl &= ~XEON_GEN4_LINK_CTRL_LINK_DIS;
 356                 rte_write16(link_ctrl, reg_addr);
 357
 358                 /* start link training */
 359                 reg_addr = hw->hw_addr + XEON_GEN4_PPD0_OFFSET;
 360                 ppd0 = rte_read32(reg_addr);
 361                 ppd0 |= XEON_GEN4_PPD_LINKTRN;
 362                 rte_write32(ppd0, reg_addr);
 363
 364                 /* make sure link training has started */
 365                 ppd0 = rte_read32(reg_addr);
 366                 if (!(ppd0 & XEON_GEN4_PPD_LINKTRN)) {
 367                         NTB_LOG(ERR, "Link is not training.");
 368                         return -EINVAL;
 369                 }
 370         } else {
 371                 reg_addr = hw->hw_addr + XEON_NTBCNTL_OFFSET;
 372                 ntb_ctrl = rte_read32(reg_addr);
 373                 ntb_ctrl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
 374                 ntb_ctrl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
 375                 rte_write32(ntb_ctrl, reg_addr);
 376
 377                 reg_addr = hw->hw_addr + XEON_GEN4_LINK_CTRL_OFFSET;
 378                 link_ctrl = rte_read16(reg_addr);
 379                 link_ctrl |= XEON_GEN4_LINK_CTRL_LINK_DIS;
 380                 rte_write16(link_ctrl, reg_addr);
 381         }
 382
 383         return 0;
 384 }
 385
 386 static int
 387 intel_ntb_set_link(const struct rte_rawdev *dev, bool up)
 388 {
 389         struct ntb_hw *hw = dev->dev_private;
 390         int ret = 0;
 391
 392         if (is_gen3_ntb(hw))
 393                 ret = intel_ntb_gen3_set_link(hw, up);
 394         else if (is_gen4_ntb(hw))
 395                 ret = intel_ntb_gen4_set_link(hw, up);
 396         else {
 397                 NTB_LOG(ERR, "Cannot set link for unsupported device.");
 398                 ret = -ENOTSUP;
 399         }
 400
 401         return ret;
 402 }
 403
 404 static uint32_t
 405 intel_ntb_spad_read(const struct rte_rawdev *dev, int spad, bool peer)
 406 {
 407         struct ntb_hw *hw = dev->dev_private;
 408         uint32_t spad_v, reg_off;
 409         void *reg_addr;
 410
 411         if (spad < 0 || spad >= hw->spad_cnt) {
 412                 NTB_LOG(ERR, "Invalid spad reg index.");
 413                 return 0;
 414         }
 415
 416         /* When peer is true, read peer spad reg */
 417         if (is_gen3_ntb(hw))
 418                 reg_off = peer ? XEON_GEN3_B2B_SPAD_OFFSET :
 419                                 XEON_IM_SPAD_OFFSET;
 420         else if (is_gen4_ntb(hw))
 421                 reg_off = peer ? XEON_GEN4_B2B_SPAD_OFFSET :
 422                                 XEON_IM_SPAD_OFFSET;
 423         else {
 424                 NTB_LOG(ERR, "Cannot read spad for unsupported device.");
 425                 return -ENOTSUP;
 426         }
 427         reg_addr = hw->hw_addr + reg_off + (spad << 2);
 428         spad_v = rte_read32(reg_addr);
 429
 430         return spad_v;
 431 }
 432
 433 static int
 434 intel_ntb_spad_write(const struct rte_rawdev *dev, int spad,
 435                      bool peer, uint32_t spad_v)
 436 {
 437         struct ntb_hw *hw = dev->dev_private;
 438         uint32_t reg_off;
 439         void *reg_addr;
 440
 441         if (spad < 0 || spad >= hw->spad_cnt) {
 442                 NTB_LOG(ERR, "Invalid spad reg index.");
 443                 return -EINVAL;
 444         }
 445
 446         /* When peer is true, write peer spad reg */
 447         if (is_gen3_ntb(hw))
 448                 reg_off = peer ? XEON_GEN3_B2B_SPAD_OFFSET :
 449                                 XEON_IM_SPAD_OFFSET;
 450         else if (is_gen4_ntb(hw))
 451                 reg_off = peer ? XEON_GEN4_B2B_SPAD_OFFSET :
 452                                 XEON_IM_SPAD_OFFSET;
 453         else {
 454                 NTB_LOG(ERR, "Cannot write spad for unsupported device.");
 455                 return -ENOTSUP;
 456         }
 457         reg_addr = hw->hw_addr + reg_off + (spad << 2);
 458
 459         rte_write32(spad_v, reg_addr);
 460
 461         return 0;
 462 }
 463
 464 static uint64_t
 465 intel_ntb_db_read(const struct rte_rawdev *dev)
 466 {
 467         struct ntb_hw *hw = dev->dev_private;
 468         uint64_t db_off, db_bits;
 469         void *db_addr;
 470
 471         db_off = XEON_IM_INT_STATUS_OFFSET;
 472         db_addr = hw->hw_addr + db_off;
 473
 474         db_bits = rte_read64(db_addr);
 475
 476         return db_bits;
 477 }
 478
 479 static int
 480 intel_ntb_db_clear(const struct rte_rawdev *dev, uint64_t db_bits)
 481 {
 482         struct ntb_hw *hw = dev->dev_private;
 483         uint64_t db_off;
 484         void *db_addr;
 485
 486         db_off = XEON_IM_INT_STATUS_OFFSET;
 487         db_addr = hw->hw_addr + db_off;
 488
 489         if (is_gen4_ntb(hw))
 490                 rte_write16(XEON_GEN4_SLOTSTS_DLLSCS,
 491                             hw->hw_addr + XEON_GEN4_SLOTSTS);
 492         rte_write64(db_bits, db_addr);
 493
 494         return 0;
 495 }
 496
 497 static int
 498 intel_ntb_db_set_mask(const struct rte_rawdev *dev, uint64_t db_mask)
 499 {
 500         struct ntb_hw *hw = dev->dev_private;
 501         uint64_t db_m_off;
 502         void *db_m_addr;
 503
 504         db_m_off = XEON_IM_INT_DISABLE_OFFSET;
 505         db_m_addr = hw->hw_addr + db_m_off;
 506
 507         db_mask |= hw->db_mask;
 508
 509         rte_write64(db_mask, db_m_addr);
 510
 511         hw->db_mask = db_mask;
 512
 513         return 0;
 514 }
 515
 516 static int
 517 intel_ntb_peer_db_set(const struct rte_rawdev *dev, uint8_t db_idx)
 518 {
 519         struct ntb_hw *hw = dev->dev_private;
 520         uint32_t db_off;
 521         void *db_addr;
 522
 523         if (((uint64_t)1 << db_idx) & ~hw->db_valid_mask) {
 524                 NTB_LOG(ERR, "Invalid doorbell.");
 525                 return -EINVAL;
 526         }
 527
 528         db_off = XEON_IM_DOORBELL_OFFSET + db_idx * 4;
 529         db_addr = hw->hw_addr + db_off;
 530
 531         rte_write32(1, db_addr);
 532
 533         return 0;
 534 }
 535
 536 static int
 537 intel_ntb_vector_bind(const struct rte_rawdev *dev, uint8_t intr, uint8_t msix)
 538 {
 539         struct ntb_hw *hw = dev->dev_private;
 540         uint8_t reg_off;
 541         void *reg_addr;
 542
 543         if (intr >= hw->db_cnt) {
 544                 NTB_LOG(ERR, "Invalid intr source.");
 545                 return -EINVAL;
 546         }
 547
 548         /* Bind intr source to msix vector */
 549         if (is_gen3_ntb(hw))
 550                 reg_off = XEON_GEN3_INTVEC_OFFSET;
 551         else if (is_gen4_ntb(hw))
 552                 reg_off = XEON_GEN4_INTVEC_OFFSET;
 553         else {
 554                 NTB_LOG(ERR, "Cannot bind vectors for unsupported device.");
 555                 return -ENOTSUP;
 556         }
 557         reg_addr = hw->hw_addr + reg_off + intr;
 558
 559         rte_write8(msix, reg_addr);
 560
 561         return 0;
 562 }
 563
 564 /* operations for primary side of local ntb */
 565 const struct ntb_dev_ops intel_ntb_ops = {
 566         .ntb_dev_init       = intel_ntb_dev_init,
 567         .get_peer_mw_addr   = intel_ntb_get_peer_mw_addr,
 568         .mw_set_trans       = intel_ntb_mw_set_trans,
 569         .ioremap            = intel_ntb_ioremap,
 570         .get_link_status    = intel_ntb_get_link_status,
 571         .set_link           = intel_ntb_set_link,
 572         .spad_read          = intel_ntb_spad_read,
 573         .spad_write         = intel_ntb_spad_write,
 574         .db_read            = intel_ntb_db_read,
 575         .db_clear           = intel_ntb_db_clear,
 576         .db_set_mask        = intel_ntb_db_set_mask,
 577         .peer_db_set        = intel_ntb_peer_db_set,
 578         .vector_bind        = intel_ntb_vector_bind,
 579 };