ntb_hw_intel.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation.
   3  */
   4 #include <stdint.h>
   5 #include <stdio.h>
   6 #include <errno.h>
   7
   8 #include <rte_io.h>
   9 #include <rte_eal.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_rawdev.h>
  13 #include <rte_rawdev_pmd.h>
  14
  15 #include "ntb.h"
  16 #include "ntb_hw_intel.h"
  17
  18 enum xeon_ntb_bar {
  19         XEON_NTB_BAR23 = 2,
  20         XEON_NTB_BAR45 = 4,
  21 };
  22
  23 static enum xeon_ntb_bar intel_ntb_bar[] = {
  24         XEON_NTB_BAR23,
  25         XEON_NTB_BAR45,
  26 };
  27
  28 static int
  29 intel_ntb_dev_init(const struct rte_rawdev *dev)
  30 {
  31         struct ntb_hw *hw = dev->dev_private;
  32         uint8_t reg_val, bar;
  33         int ret, i;
  34
  35         if (hw == NULL) {
  36                 NTB_LOG(ERR, "Invalid device.");
  37                 return -EINVAL;
  38         }
  39
  40         ret = rte_pci_read_config(hw->pci_dev, &reg_val,
  41                                   sizeof(reg_val), XEON_PPD_OFFSET);
  42         if (ret < 0) {
  43                 NTB_LOG(ERR, "Cannot get NTB PPD (PCIe port definition).");
  44                 return -EIO;
  45         }
  46
  47         /* Check connection topo type. Only support B2B. */
  48         switch (reg_val & XEON_PPD_CONN_MASK) {
  49         case XEON_PPD_CONN_B2B:
  50                 NTB_LOG(INFO, "Topo B2B (back to back) is using.");
  51                 break;
  52         case XEON_PPD_CONN_TRANSPARENT:
  53         case XEON_PPD_CONN_RP:
  54         default:
  55                 NTB_LOG(ERR, "Not supported conn topo. Please use B2B.");
  56                 return -EINVAL;
  57         }
  58
  59         /* Check device type. */
  60         if (reg_val & XEON_PPD_DEV_DSD) {
  61                 NTB_LOG(INFO, "DSD, Downstream Device.");
  62                 hw->topo = NTB_TOPO_B2B_DSD;
  63         } else {
  64                 NTB_LOG(INFO, "USD, Upstream device.");
  65                 hw->topo = NTB_TOPO_B2B_USD;
  66         }
  67
  68         /* Check if bar4 is split. Do not support split bar. */
  69         if (reg_val & XEON_PPD_SPLIT_BAR_MASK) {
  70                 NTB_LOG(ERR, "Do not support split bar.");
  71                 return -EINVAL;
  72         }
  73
  74         hw->hw_addr = (char *)hw->pci_dev->mem_resource[0].addr;
  75
  76         hw->mw_cnt = XEON_MW_COUNT;
  77         hw->db_cnt = XEON_DB_COUNT;
  78         hw->spad_cnt = XEON_SPAD_COUNT;
  79
  80         hw->mw_size = rte_zmalloc("ntb_mw_size",
  81                                   hw->mw_cnt * sizeof(uint64_t), 0);
  82         for (i = 0; i < hw->mw_cnt; i++) {
  83                 bar = intel_ntb_bar[i];
  84                 hw->mw_size[i] = hw->pci_dev->mem_resource[bar].len;
  85         }
  86
  87         /* Reserve the last 2 spad registers for users. */
  88         for (i = 0; i < NTB_SPAD_USER_MAX_NUM; i++)
  89                 hw->spad_user_list[i] = hw->spad_cnt;
  90         hw->spad_user_list[0] = hw->spad_cnt - 2;
  91         hw->spad_user_list[1] = hw->spad_cnt - 1;
  92
  93         return 0;
  94 }
  95
  96 static void *
  97 intel_ntb_get_peer_mw_addr(const struct rte_rawdev *dev, int mw_idx)
  98 {
  99         struct ntb_hw *hw = dev->dev_private;
 100         uint8_t bar;
 101
 102         if (hw == NULL) {
 103                 NTB_LOG(ERR, "Invalid device.");
 104                 return 0;
 105         }
 106
 107         if (mw_idx < 0 || mw_idx >= hw->mw_cnt) {
 108                 NTB_LOG(ERR, "Invalid memory window index (0 - %u).",
 109                         hw->mw_cnt - 1);
 110                 return 0;
 111         }
 112
 113         bar = intel_ntb_bar[mw_idx];
 114
 115         return hw->pci_dev->mem_resource[bar].addr;
 116 }
 117
 118 static int
 119 intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,
 120                        uint64_t addr, uint64_t size)
 121 {
 122         struct ntb_hw *hw = dev->dev_private;
 123         void *xlat_addr, *limit_addr;
 124         uint64_t xlat_off, limit_off;
 125         uint64_t base, limit;
 126         uint8_t bar;
 127
 128         if (hw == NULL) {
 129                 NTB_LOG(ERR, "Invalid device.");
 130                 return -EINVAL;
 131         }
 132
 133         if (mw_idx < 0 || mw_idx >= hw->mw_cnt) {
 134                 NTB_LOG(ERR, "Invalid memory window index (0 - %u).",
 135                         hw->mw_cnt - 1);
 136                 return -EINVAL;
 137         }
 138
 139         bar = intel_ntb_bar[mw_idx];
 140
 141         xlat_off = XEON_IMBAR1XBASE_OFFSET + mw_idx * XEON_BAR_INTERVAL_OFFSET;
 142         limit_off = XEON_IMBAR1XLMT_OFFSET + mw_idx * XEON_BAR_INTERVAL_OFFSET;
 143         xlat_addr = hw->hw_addr + xlat_off;
 144         limit_addr = hw->hw_addr + limit_off;
 145
 146         /* Limit reg val should be EMBAR base address plus MW size. */
 147         base = addr;
 148         limit = hw->pci_dev->mem_resource[bar].phys_addr + size;
 149         rte_write64(base, xlat_addr);
 150         rte_write64(limit, limit_addr);
 151
 152         /* Setup the external point so that remote can access. */
 153         xlat_off = XEON_EMBAR1_OFFSET + 8 * mw_idx;
 154         xlat_addr = hw->hw_addr + xlat_off;
 155         limit_off = XEON_EMBAR1XLMT_OFFSET + mw_idx * XEON_BAR_INTERVAL_OFFSET;
 156         limit_addr = hw->hw_addr + limit_off;
 157         base = rte_read64(xlat_addr);
 158         base &= ~0xf;
 159         limit = base + size;
 160         rte_write64(limit, limit_addr);
 161
 162         return 0;
 163 }
 164
 165 static void *
 166 intel_ntb_ioremap(const struct rte_rawdev *dev, uint64_t addr)
 167 {
 168         struct ntb_hw *hw = dev->dev_private;
 169         void *mapped = NULL;
 170         void *base;
 171         int i;
 172
 173         for (i = 0; i < hw->peer_used_mws; i++) {
 174                 if (addr >= hw->peer_mw_base[i] &&
 175                     addr <= hw->peer_mw_base[i] + hw->mw_size[i]) {
 176                         base = intel_ntb_get_peer_mw_addr(dev, i);
 177                         mapped = (void *)(size_t)(addr - hw->peer_mw_base[i] +
 178                                  (size_t)base);
 179                         break;
 180                 }
 181         }
 182
 183         return mapped;
 184 }
 185
 186 static int
 187 intel_ntb_get_link_status(const struct rte_rawdev *dev)
 188 {
 189         struct ntb_hw *hw = dev->dev_private;
 190         uint16_t reg_val;
 191         int ret;
 192
 193         if (hw == NULL) {
 194                 NTB_LOG(ERR, "Invalid device.");
 195                 return -EINVAL;
 196         }
 197
 198         ret = rte_pci_read_config(hw->pci_dev, &reg_val,
 199                                   sizeof(reg_val), XEON_LINK_STATUS_OFFSET);
 200         if (ret < 0) {
 201                 NTB_LOG(ERR, "Unable to get link status.");
 202                 return -EIO;
 203         }
 204
 205         hw->link_status = NTB_LNK_STA_ACTIVE(reg_val);
 206
 207         if (hw->link_status) {
 208                 hw->link_speed = NTB_LNK_STA_SPEED(reg_val);
 209                 hw->link_width = NTB_LNK_STA_WIDTH(reg_val);
 210         } else {
 211                 hw->link_speed = NTB_SPEED_NONE;
 212                 hw->link_width = NTB_WIDTH_NONE;
 213         }
 214
 215         return 0;
 216 }
 217
 218 static int
 219 intel_ntb_set_link(const struct rte_rawdev *dev, bool up)
 220 {
 221         struct ntb_hw *hw = dev->dev_private;
 222         uint32_t ntb_ctrl, reg_off;
 223         void *reg_addr;
 224
 225         reg_off = XEON_NTBCNTL_OFFSET;
 226         reg_addr = hw->hw_addr + reg_off;
 227         ntb_ctrl = rte_read32(reg_addr);
 228
 229         if (up) {
 230                 ntb_ctrl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
 231                 ntb_ctrl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
 232                 ntb_ctrl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
 233         } else {
 234                 ntb_ctrl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
 235                 ntb_ctrl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
 236                 ntb_ctrl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
 237         }
 238
 239         rte_write32(ntb_ctrl, reg_addr);
 240
 241         return 0;
 242 }
 243
 244 static uint32_t
 245 intel_ntb_spad_read(const struct rte_rawdev *dev, int spad, bool peer)
 246 {
 247         struct ntb_hw *hw = dev->dev_private;
 248         uint32_t spad_v, reg_off;
 249         void *reg_addr;
 250
 251         if (spad < 0 || spad >= hw->spad_cnt) {
 252                 NTB_LOG(ERR, "Invalid spad reg index.");
 253                 return 0;
 254         }
 255
 256         /* When peer is true, read peer spad reg */
 257         reg_off = peer ? XEON_B2B_SPAD_OFFSET : XEON_IM_SPAD_OFFSET;
 258         reg_addr = hw->hw_addr + reg_off + (spad << 2);
 259         spad_v = rte_read32(reg_addr);
 260
 261         return spad_v;
 262 }
 263
 264 static int
 265 intel_ntb_spad_write(const struct rte_rawdev *dev, int spad,
 266                      bool peer, uint32_t spad_v)
 267 {
 268         struct ntb_hw *hw = dev->dev_private;
 269         uint32_t reg_off;
 270         void *reg_addr;
 271
 272         if (spad < 0 || spad >= hw->spad_cnt) {
 273                 NTB_LOG(ERR, "Invalid spad reg index.");
 274                 return -EINVAL;
 275         }
 276
 277         /* When peer is true, write peer spad reg */
 278         reg_off = peer ? XEON_B2B_SPAD_OFFSET : XEON_IM_SPAD_OFFSET;
 279         reg_addr = hw->hw_addr + reg_off + (spad << 2);
 280
 281         rte_write32(spad_v, reg_addr);
 282
 283         return 0;
 284 }
 285
 286 static uint64_t
 287 intel_ntb_db_read(const struct rte_rawdev *dev)
 288 {
 289         struct ntb_hw *hw = dev->dev_private;
 290         uint64_t db_off, db_bits;
 291         void *db_addr;
 292
 293         db_off = XEON_IM_INT_STATUS_OFFSET;
 294         db_addr = hw->hw_addr + db_off;
 295
 296         db_bits = rte_read64(db_addr);
 297
 298         return db_bits;
 299 }
 300
 301 static int
 302 intel_ntb_db_clear(const struct rte_rawdev *dev, uint64_t db_bits)
 303 {
 304         struct ntb_hw *hw = dev->dev_private;
 305         uint64_t db_off;
 306         void *db_addr;
 307
 308         db_off = XEON_IM_INT_STATUS_OFFSET;
 309         db_addr = hw->hw_addr + db_off;
 310
 311         rte_write64(db_bits, db_addr);
 312
 313         return 0;
 314 }
 315
 316 static int
 317 intel_ntb_db_set_mask(const struct rte_rawdev *dev, uint64_t db_mask)
 318 {
 319         struct ntb_hw *hw = dev->dev_private;
 320         uint64_t db_m_off;
 321         void *db_m_addr;
 322
 323         db_m_off = XEON_IM_INT_DISABLE_OFFSET;
 324         db_m_addr = hw->hw_addr + db_m_off;
 325
 326         db_mask |= hw->db_mask;
 327
 328         rte_write64(db_mask, db_m_addr);
 329
 330         hw->db_mask = db_mask;
 331
 332         return 0;
 333 }
 334
 335 static int
 336 intel_ntb_peer_db_set(const struct rte_rawdev *dev, uint8_t db_idx)
 337 {
 338         struct ntb_hw *hw = dev->dev_private;
 339         uint32_t db_off;
 340         void *db_addr;
 341
 342         if (((uint64_t)1 << db_idx) & ~hw->db_valid_mask) {
 343                 NTB_LOG(ERR, "Invalid doorbell.");
 344                 return -EINVAL;
 345         }
 346
 347         db_off = XEON_IM_DOORBELL_OFFSET + db_idx * 4;
 348         db_addr = hw->hw_addr + db_off;
 349
 350         rte_write32(1, db_addr);
 351
 352         return 0;
 353 }
 354
 355 static int
 356 intel_ntb_vector_bind(const struct rte_rawdev *dev, uint8_t intr, uint8_t msix)
 357 {
 358         struct ntb_hw *hw = dev->dev_private;
 359         uint8_t reg_off;
 360         void *reg_addr;
 361
 362         if (intr >= hw->db_cnt) {
 363                 NTB_LOG(ERR, "Invalid intr source.");
 364                 return -EINVAL;
 365         }
 366
 367         /* Bind intr source to msix vector */
 368         reg_off = XEON_INTVEC_OFFSET;
 369         reg_addr = hw->hw_addr + reg_off + intr;
 370
 371         rte_write8(msix, reg_addr);
 372
 373         return 0;
 374 }
 375
 376 /* operations for primary side of local ntb */
 377 const struct ntb_dev_ops intel_ntb_ops = {
 378         .ntb_dev_init       = intel_ntb_dev_init,
 379         .get_peer_mw_addr   = intel_ntb_get_peer_mw_addr,
 380         .mw_set_trans       = intel_ntb_mw_set_trans,
 381         .ioremap            = intel_ntb_ioremap,
 382         .get_link_status    = intel_ntb_get_link_status,
 383         .set_link           = intel_ntb_set_link,
 384         .spad_read          = intel_ntb_spad_read,
 385         .spad_write         = intel_ntb_spad_write,
 386         .db_read            = intel_ntb_db_read,
 387         .db_clear           = intel_ntb_db_clear,
 388         .db_set_mask        = intel_ntb_db_set_mask,
 389         .peer_db_set        = intel_ntb_peer_db_set,
 390         .vector_bind        = intel_ntb_vector_bind,
 391 };