1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation.
11 #include <rte_bus_pci.h>
12 #include <rte_rawdev.h>
13 #include <rte_rawdev_pmd.h>
16 #include "ntb_hw_intel.h"
23 static enum xeon_ntb_bar intel_ntb_bar[] = {
29 is_gen3_ntb(const struct ntb_hw *hw)
31 if (hw->pci_dev->id.device_id == NTB_INTEL_DEV_ID_B2B_SKX)
38 is_gen4_ntb(const struct ntb_hw *hw)
40 if (hw->pci_dev->id.device_id == NTB_INTEL_DEV_ID_B2B_ICX)
47 intel_ntb3_check_ppd(struct ntb_hw *hw)
52 ret = rte_pci_read_config(hw->pci_dev, ®_val,
53 sizeof(reg_val), XEON_PPD_OFFSET);
55 NTB_LOG(ERR, "Cannot get NTB PPD (PCIe port definition).");
59 /* Check connection topo type. Only support B2B. */
60 switch (reg_val & XEON_PPD_CONN_MASK) {
61 case XEON_PPD_CONN_B2B:
62 NTB_LOG(INFO, "Topo B2B (back to back) is using.");
64 case XEON_PPD_CONN_TRANSPARENT:
65 case XEON_PPD_CONN_RP:
67 NTB_LOG(ERR, "Not supported conn topo. Please use B2B.");
71 /* Check device type. */
72 if (reg_val & XEON_PPD_DEV_DSD) {
73 NTB_LOG(INFO, "DSD, Downstream Device.");
74 hw->topo = NTB_TOPO_B2B_DSD;
76 NTB_LOG(INFO, "USD, Upstream device.");
77 hw->topo = NTB_TOPO_B2B_USD;
80 /* Check if bar4 is split. Do not support split bar. */
81 if (reg_val & XEON_PPD_SPLIT_BAR_MASK) {
82 NTB_LOG(ERR, "Do not support split bar.");
90 intel_ntb4_check_ppd(struct ntb_hw *hw)
94 reg_val = rte_read32(hw->hw_addr + XEON_GEN4_PPD1_OFFSET);
96 /* Check connection topo type. Only support B2B. */
97 switch (reg_val & XEON_GEN4_PPD_CONN_MASK) {
98 case XEON_GEN4_PPD_CONN_B2B:
99 NTB_LOG(INFO, "Topo B2B (back to back) is using.");
102 NTB_LOG(ERR, "Not supported conn topo. Please use B2B.");
106 /* Check device type. */
107 if (reg_val & XEON_GEN4_PPD_DEV_DSD) {
108 NTB_LOG(INFO, "DSD, Downstream Device.");
109 hw->topo = NTB_TOPO_B2B_DSD;
111 NTB_LOG(INFO, "USD, Upstream device.");
112 hw->topo = NTB_TOPO_B2B_USD;
119 intel_ntb_dev_init(const struct rte_rawdev *dev)
121 struct ntb_hw *hw = dev->dev_private;
126 NTB_LOG(ERR, "Invalid device.");
130 hw->hw_addr = (char *)hw->pci_dev->mem_resource[0].addr;
133 ret = intel_ntb3_check_ppd(hw);
134 else if (is_gen4_ntb(hw))
135 /* PPD is in MMIO but not config space for NTB Gen4 */
136 ret = intel_ntb4_check_ppd(hw);
138 NTB_LOG(ERR, "Cannot init device for unsupported device.");
145 hw->mw_cnt = XEON_MW_COUNT;
146 hw->db_cnt = XEON_DB_COUNT;
147 hw->spad_cnt = XEON_SPAD_COUNT;
149 hw->mw_size = rte_zmalloc("ntb_mw_size",
150 hw->mw_cnt * sizeof(uint64_t), 0);
151 for (i = 0; i < hw->mw_cnt; i++) {
152 bar = intel_ntb_bar[i];
153 hw->mw_size[i] = hw->pci_dev->mem_resource[bar].len;
156 /* Reserve the last 2 spad registers for users. */
157 for (i = 0; i < NTB_SPAD_USER_MAX_NUM; i++)
158 hw->spad_user_list[i] = hw->spad_cnt;
159 hw->spad_user_list[0] = hw->spad_cnt - 2;
160 hw->spad_user_list[1] = hw->spad_cnt - 1;
166 intel_ntb_get_peer_mw_addr(const struct rte_rawdev *dev, int mw_idx)
168 struct ntb_hw *hw = dev->dev_private;
172 NTB_LOG(ERR, "Invalid device.");
176 if (mw_idx < 0 || mw_idx >= hw->mw_cnt) {
177 NTB_LOG(ERR, "Invalid memory window index (0 - %u).",
182 bar = intel_ntb_bar[mw_idx];
184 return hw->pci_dev->mem_resource[bar].addr;
188 intel_ntb_mw_set_trans(const struct rte_rawdev *dev, int mw_idx,
189 uint64_t addr, uint64_t size)
191 struct ntb_hw *hw = dev->dev_private;
192 void *xlat_addr, *limit_addr;
193 uint64_t xlat_off, limit_off;
194 uint64_t base, limit;
198 NTB_LOG(ERR, "Invalid device.");
202 if (mw_idx < 0 || mw_idx >= hw->mw_cnt) {
203 NTB_LOG(ERR, "Invalid memory window index (0 - %u).",
208 bar = intel_ntb_bar[mw_idx];
210 xlat_off = XEON_IMBAR1XBASE_OFFSET + mw_idx * XEON_BAR_INTERVAL_OFFSET;
211 limit_off = XEON_IMBAR1XLMT_OFFSET + mw_idx * XEON_BAR_INTERVAL_OFFSET;
212 xlat_addr = hw->hw_addr + xlat_off;
213 limit_addr = hw->hw_addr + limit_off;
215 /* Limit reg val should be EMBAR base address plus MW size. */
217 limit = hw->pci_dev->mem_resource[bar].phys_addr + size;
218 rte_write64(base, xlat_addr);
219 rte_write64(limit, limit_addr);
221 if (is_gen3_ntb(hw)) {
222 /* Setup the external point so that remote can access. */
223 xlat_off = XEON_EMBAR1_OFFSET + 8 * mw_idx;
224 xlat_addr = hw->hw_addr + xlat_off;
225 limit_off = XEON_EMBAR1XLMT_OFFSET +
226 mw_idx * XEON_BAR_INTERVAL_OFFSET;
227 limit_addr = hw->hw_addr + limit_off;
228 base = rte_read64(xlat_addr);
231 rte_write64(limit, limit_addr);
232 } else if (is_gen4_ntb(hw)) {
233 /* Set translate base address index register */
234 xlat_off = XEON_GEN4_IM1XBASEIDX_OFFSET +
235 mw_idx * XEON_GEN4_XBASEIDX_INTERVAL;
236 xlat_addr = hw->hw_addr + xlat_off;
237 rte_write16(rte_log2_u64(size), xlat_addr);
239 NTB_LOG(ERR, "Cannot set translation of memory windows for unsupported device.");
240 rte_write64(base, limit_addr);
241 rte_write64(0, xlat_addr);
249 intel_ntb_ioremap(const struct rte_rawdev *dev, uint64_t addr)
251 struct ntb_hw *hw = dev->dev_private;
256 for (i = 0; i < hw->peer_used_mws; i++) {
257 if (addr >= hw->peer_mw_base[i] &&
258 addr <= hw->peer_mw_base[i] + hw->mw_size[i]) {
259 base = intel_ntb_get_peer_mw_addr(dev, i);
260 mapped = (void *)(size_t)(addr - hw->peer_mw_base[i] +
270 intel_ntb_get_link_status(const struct rte_rawdev *dev)
272 struct ntb_hw *hw = dev->dev_private;
273 uint16_t reg_val, reg_off;
277 NTB_LOG(ERR, "Invalid device.");
281 if (is_gen3_ntb(hw)) {
282 reg_off = XEON_GEN3_LINK_STATUS_OFFSET;
283 ret = rte_pci_read_config(hw->pci_dev, ®_val,
284 sizeof(reg_val), reg_off);
286 NTB_LOG(ERR, "Unable to get link status.");
289 } else if (is_gen4_ntb(hw)) {
290 reg_off = XEON_GEN4_LINK_STATUS_OFFSET;
291 reg_val = rte_read16(hw->hw_addr + reg_off);
293 NTB_LOG(ERR, "Cannot get link status for unsupported device.");
297 hw->link_status = NTB_LNK_STA_ACTIVE(reg_val);
299 if (hw->link_status) {
300 hw->link_speed = NTB_LNK_STA_SPEED(reg_val);
301 hw->link_width = NTB_LNK_STA_WIDTH(reg_val);
303 hw->link_speed = NTB_SPEED_NONE;
304 hw->link_width = NTB_WIDTH_NONE;
311 intel_ntb_gen3_set_link(const struct ntb_hw *hw, bool up)
313 uint32_t ntb_ctrl, reg_off;
316 reg_off = XEON_NTBCNTL_OFFSET;
317 reg_addr = hw->hw_addr + reg_off;
318 ntb_ctrl = rte_read32(reg_addr);
321 ntb_ctrl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
322 ntb_ctrl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
323 ntb_ctrl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
325 ntb_ctrl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
326 ntb_ctrl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
327 ntb_ctrl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
330 rte_write32(ntb_ctrl, reg_addr);
336 intel_ntb_gen4_set_link(const struct ntb_hw *hw, bool up)
338 uint32_t ntb_ctrl, ppd0;
343 reg_addr = hw->hw_addr + XEON_NTBCNTL_OFFSET;
344 ntb_ctrl = NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
345 ntb_ctrl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
346 rte_write32(ntb_ctrl, reg_addr);
348 reg_addr = hw->hw_addr + XEON_GEN4_LINK_CTRL_OFFSET;
349 link_ctrl = rte_read16(reg_addr);
350 link_ctrl &= ~XEON_GEN4_LINK_CTRL_LINK_DIS;
351 rte_write16(link_ctrl, reg_addr);
353 /* start link training */
354 reg_addr = hw->hw_addr + XEON_GEN4_PPD0_OFFSET;
355 ppd0 = rte_read32(reg_addr);
356 ppd0 |= XEON_GEN4_PPD_LINKTRN;
357 rte_write32(ppd0, reg_addr);
359 /* make sure link training has started */
360 ppd0 = rte_read32(reg_addr);
361 if (!(ppd0 & XEON_GEN4_PPD_LINKTRN)) {
362 NTB_LOG(ERR, "Link is not training.");
366 reg_addr = hw->hw_addr + XEON_NTBCNTL_OFFSET;
367 ntb_ctrl = rte_read32(reg_addr);
368 ntb_ctrl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
369 ntb_ctrl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
370 rte_write32(ntb_ctrl, reg_addr);
372 reg_addr = hw->hw_addr + XEON_GEN4_LINK_CTRL_OFFSET;
373 link_ctrl = rte_read16(reg_addr);
374 link_ctrl |= XEON_GEN4_LINK_CTRL_LINK_DIS;
375 rte_write16(link_ctrl, reg_addr);
382 intel_ntb_set_link(const struct rte_rawdev *dev, bool up)
384 struct ntb_hw *hw = dev->dev_private;
388 ret = intel_ntb_gen3_set_link(hw, up);
389 else if (is_gen4_ntb(hw))
390 ret = intel_ntb_gen4_set_link(hw, up);
392 NTB_LOG(ERR, "Cannot set link for unsupported device.");
400 intel_ntb_spad_read(const struct rte_rawdev *dev, int spad, bool peer)
402 struct ntb_hw *hw = dev->dev_private;
403 uint32_t spad_v, reg_off;
406 if (spad < 0 || spad >= hw->spad_cnt) {
407 NTB_LOG(ERR, "Invalid spad reg index.");
411 /* When peer is true, read peer spad reg */
413 reg_off = peer ? XEON_GEN3_B2B_SPAD_OFFSET :
415 else if (is_gen4_ntb(hw))
416 reg_off = peer ? XEON_GEN4_B2B_SPAD_OFFSET :
419 NTB_LOG(ERR, "Cannot read spad for unsupported device.");
422 reg_addr = hw->hw_addr + reg_off + (spad << 2);
423 spad_v = rte_read32(reg_addr);
429 intel_ntb_spad_write(const struct rte_rawdev *dev, int spad,
430 bool peer, uint32_t spad_v)
432 struct ntb_hw *hw = dev->dev_private;
436 if (spad < 0 || spad >= hw->spad_cnt) {
437 NTB_LOG(ERR, "Invalid spad reg index.");
441 /* When peer is true, write peer spad reg */
443 reg_off = peer ? XEON_GEN3_B2B_SPAD_OFFSET :
445 else if (is_gen4_ntb(hw))
446 reg_off = peer ? XEON_GEN4_B2B_SPAD_OFFSET :
449 NTB_LOG(ERR, "Cannot write spad for unsupported device.");
452 reg_addr = hw->hw_addr + reg_off + (spad << 2);
454 rte_write32(spad_v, reg_addr);
460 intel_ntb_db_read(const struct rte_rawdev *dev)
462 struct ntb_hw *hw = dev->dev_private;
463 uint64_t db_off, db_bits;
466 db_off = XEON_IM_INT_STATUS_OFFSET;
467 db_addr = hw->hw_addr + db_off;
469 db_bits = rte_read64(db_addr);
475 intel_ntb_db_clear(const struct rte_rawdev *dev, uint64_t db_bits)
477 struct ntb_hw *hw = dev->dev_private;
481 db_off = XEON_IM_INT_STATUS_OFFSET;
482 db_addr = hw->hw_addr + db_off;
485 rte_write16(XEON_GEN4_SLOTSTS_DLLSCS,
486 hw->hw_addr + XEON_GEN4_SLOTSTS);
487 rte_write64(db_bits, db_addr);
493 intel_ntb_db_set_mask(const struct rte_rawdev *dev, uint64_t db_mask)
495 struct ntb_hw *hw = dev->dev_private;
499 db_m_off = XEON_IM_INT_DISABLE_OFFSET;
500 db_m_addr = hw->hw_addr + db_m_off;
502 db_mask |= hw->db_mask;
504 rte_write64(db_mask, db_m_addr);
506 hw->db_mask = db_mask;
512 intel_ntb_peer_db_set(const struct rte_rawdev *dev, uint8_t db_idx)
514 struct ntb_hw *hw = dev->dev_private;
518 if (((uint64_t)1 << db_idx) & ~hw->db_valid_mask) {
519 NTB_LOG(ERR, "Invalid doorbell.");
523 db_off = XEON_IM_DOORBELL_OFFSET + db_idx * 4;
524 db_addr = hw->hw_addr + db_off;
526 rte_write32(1, db_addr);
532 intel_ntb_vector_bind(const struct rte_rawdev *dev, uint8_t intr, uint8_t msix)
534 struct ntb_hw *hw = dev->dev_private;
538 if (intr >= hw->db_cnt) {
539 NTB_LOG(ERR, "Invalid intr source.");
543 /* Bind intr source to msix vector */
545 reg_off = XEON_GEN3_INTVEC_OFFSET;
546 else if (is_gen4_ntb(hw))
547 reg_off = XEON_GEN4_INTVEC_OFFSET;
549 NTB_LOG(ERR, "Cannot bind vectors for unsupported device.");
552 reg_addr = hw->hw_addr + reg_off + intr;
554 rte_write8(msix, reg_addr);
559 /* operations for primary side of local ntb */
560 const struct ntb_dev_ops intel_ntb_ops = {
561 .ntb_dev_init = intel_ntb_dev_init,
562 .get_peer_mw_addr = intel_ntb_get_peer_mw_addr,
563 .mw_set_trans = intel_ntb_mw_set_trans,
564 .ioremap = intel_ntb_ioremap,
565 .get_link_status = intel_ntb_get_link_status,
566 .set_link = intel_ntb_set_link,
567 .spad_read = intel_ntb_spad_read,
568 .spad_write = intel_ntb_spad_write,
569 .db_read = intel_ntb_db_read,
570 .db_clear = intel_ntb_db_clear,
571 .db_set_mask = intel_ntb_db_set_mask,
572 .peer_db_set = intel_ntb_peer_db_set,
573 .vector_bind = intel_ntb_vector_bind,