1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Netronome Systems, Inc.
8 * Authors: Vinayak Tammineedi <vinayak.tammineedi@netronome.com>
10 * Multiplexes the NFP BARs between NFP internal resources and
11 * implements the PCIe specific interface for generic CPP bus access.
13 * The BARs are managed and allocated if they are available.
14 * The generic CPP bus abstraction builds upon this BAR interface.
35 #include "nfp_target.h"
36 #include "nfp6000/nfp6000.h"
38 #define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0)
40 #define NFP_PCIE_BAR_PCIE2CPP_ACTION_BASEADDRESS(_x) (((_x) & 0x1f) << 16)
41 #define NFP_PCIE_BAR_PCIE2CPP_BASEADDRESS(_x) (((_x) & 0xffff) << 0)
42 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT(_x) (((_x) & 0x3) << 27)
43 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_32BIT 0
44 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_64BIT 1
45 #define NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_0BYTE 3
46 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE(_x) (((_x) & 0x7) << 29)
47 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_OF(_x) (((_x) >> 29) & 0x7)
48 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_FIXED 0
49 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_BULK 1
50 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_TARGET 2
51 #define NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_GENERAL 3
52 #define NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(_x) (((_x) & 0xf) << 23)
53 #define NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(_x) (((_x) & 0x3) << 21)
56 * Minimal size of the PCIe cfg memory we depend on being mapped,
57 * queue controller and DMA controller don't have to be covered.
59 #define NFP_PCI_MIN_MAP_SIZE 0x080000
61 #define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize)
62 #define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize)
63 #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2))
64 #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4))
65 #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4))
67 #define NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(bar, slot) \
68 (NFP_PCIE_BAR(0) + ((bar) * 8 + (slot)) * 4)
70 #define NFP_PCIE_CPP_BAR_PCIETOCPPEXPBAR(bar, slot) \
71 (((bar) * 8 + (slot)) * 4)
74 * Define to enable a bit more verbose debug output.
75 * Set to 1 to enable a bit more verbose debug output.
78 struct nfp6000_area_priv;
81 * struct nfp_bar - describes BAR configuration and usage
82 * @nfp: backlink to owner
83 * @barcfg: cached contents of BAR config CSR
84 * @base: the BAR's base CPP offset
85 * @mask: mask for the BAR aperture (read only)
86 * @bitsize: bitsize of BAR aperture (read only)
87 * @index: index of the BAR
88 * @lock: lock to specify if bar is in use
89 * @refcnt: number of current users
90 * @iomem: mapped IO memory
94 struct nfp_pcie_user *nfp;
96 uint64_t base; /* CPP address base */
97 uint64_t mask; /* Bit mask of the bar */
98 uint32_t bitsize; /* Bit size of the bar */
107 struct nfp_pcie_user {
108 struct nfp_bar bar[NFP_BAR_MAX];
112 char busdev[BUSDEV_SZ];
118 nfp_bar_maptype(struct nfp_bar *bar)
120 return NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_OF(bar->barcfg);
123 #define TARGET_WIDTH_32 4
124 #define TARGET_WIDTH_64 8
127 nfp_compute_bar(const struct nfp_bar *bar, uint32_t *bar_config,
128 uint64_t *bar_base, int tgt, int act, int tok,
129 uint64_t offset, size_t size, int width)
141 NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT
142 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_64BIT);
146 NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT
147 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_32BIT);
151 NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT
152 (NFP_PCIE_BAR_PCIE2CPP_LENGTHSELECT_0BYTE);
158 if (act != NFP_CPP_ACTION_RW && act != 0) {
159 /* Fixed CPP mapping with specific action */
160 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1);
163 NFP_PCIE_BAR_PCIE2CPP_MAPTYPE
164 (NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_FIXED);
165 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(tgt);
166 newcfg |= NFP_PCIE_BAR_PCIE2CPP_ACTION_BASEADDRESS(act);
167 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(tok);
169 if ((offset & mask) != ((offset + size - 1) & mask)) {
170 printf("BAR%d: Won't use for Fixed mapping\n",
172 printf("\t<%#llx,%#llx>, action=%d\n",
173 (unsigned long long)offset,
174 (unsigned long long)(offset + size), act);
175 printf("\tBAR too small (0x%llx).\n",
176 (unsigned long long)mask);
182 printf("BAR%d: Created Fixed mapping\n", bar->index);
183 printf("\t%d:%d:%d:0x%#llx-0x%#llx>\n", tgt, act, tok,
184 (unsigned long long)offset,
185 (unsigned long long)(offset + mask));
190 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1);
194 NFP_PCIE_BAR_PCIE2CPP_MAPTYPE
195 (NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_BULK);
197 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TARGET_BASEADDRESS(tgt);
198 newcfg |= NFP_PCIE_BAR_PCIE2CPP_TOKEN_BASEADDRESS(tok);
200 if ((offset & mask) != ((offset + size - 1) & mask)) {
201 printf("BAR%d: Won't use for bulk mapping\n",
203 printf("\t<%#llx,%#llx>\n", (unsigned long long)offset,
204 (unsigned long long)(offset + size));
205 printf("\ttarget=%d, token=%d\n", tgt, tok);
206 printf("\tBAR too small (%#llx) - (%#llx != %#llx).\n",
207 (unsigned long long)mask,
208 (unsigned long long)(offset & mask),
209 (unsigned long long)(offset + size - 1) & mask);
217 printf("BAR%d: Created bulk mapping %d:x:%d:%#llx-%#llx\n",
218 bar->index, tgt, tok, (unsigned long long)offset,
219 (unsigned long long)(offset + ~mask));
225 if (bar->bitsize < bitsize) {
226 printf("BAR%d: Too small for %d:%d:%d\n", bar->index, tgt, tok,
231 newcfg |= offset >> bitsize;
237 *bar_config = newcfg;
243 nfp_bar_write(struct nfp_pcie_user *nfp, struct nfp_bar *bar,
248 base = bar->index >> 3;
249 slot = bar->index & 7;
254 bar->csr = nfp->cfg +
255 NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(base, slot);
257 *(uint32_t *)(bar->csr) = newcfg;
259 bar->barcfg = newcfg;
261 printf("BAR%d: updated to 0x%08x\n", bar->index, newcfg);
268 nfp_reconfigure_bar(struct nfp_pcie_user *nfp, struct nfp_bar *bar, int tgt,
269 int act, int tok, uint64_t offset, size_t size, int width)
275 err = nfp_compute_bar(bar, &newcfg, &newbase, tgt, act, tok, offset,
282 return nfp_bar_write(nfp, bar, newcfg);
286 * Map all PCI bars. We assume that the BAR with the PCIe config block is
289 * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM)
292 nfp_enable_bars(struct nfp_pcie_user *nfp)
297 for (x = ARRAY_SIZE(nfp->bar); x > 0; x--) {
298 bar = &nfp->bar[x - 1];
302 bar->mask = (1 << (nfp->barsz - 3)) - 1;
303 bar->bitsize = nfp->barsz - 3;
307 bar->csr = nfp->cfg +
308 NFP_PCIE_CFG_BAR_PCIETOCPPEXPBAR(bar->index >> 3,
311 (char *)mmap(0, 1 << bar->bitsize, PROT_READ | PROT_WRITE,
312 MAP_SHARED, nfp->device,
313 bar->index << bar->bitsize);
315 if (bar->iomem == MAP_FAILED)
321 static struct nfp_bar *
322 nfp_alloc_bar(struct nfp_pcie_user *nfp)
327 for (x = ARRAY_SIZE(nfp->bar); x > 0; x--) {
328 bar = &nfp->bar[x - 1];
338 nfp_disable_bars(struct nfp_pcie_user *nfp)
343 for (x = ARRAY_SIZE(nfp->bar); x > 0; x--) {
344 bar = &nfp->bar[x - 1];
346 munmap(bar->iomem, 1 << (nfp->barsz - 3));
354 * Generic CPP bus access interface.
357 struct nfp6000_area_priv {
375 nfp6000_area_init(struct nfp_cpp_area *area, uint32_t dest,
376 unsigned long long address, unsigned long size)
378 struct nfp_pcie_user *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
379 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
380 uint32_t target = NFP_CPP_ID_TARGET_of(dest);
381 uint32_t action = NFP_CPP_ID_ACTION_of(dest);
382 uint32_t token = NFP_CPP_ID_TOKEN_of(dest);
385 pp = nfp6000_target_pushpull(NFP_CPP_ID(target, action, token),
390 priv->width.read = PUSH_WIDTH(pp);
391 priv->width.write = PULL_WIDTH(pp);
393 if (priv->width.read > 0 &&
394 priv->width.write > 0 && priv->width.read != priv->width.write)
397 if (priv->width.read > 0)
398 priv->width.bar = priv->width.read;
400 priv->width.bar = priv->width.write;
402 priv->bar = nfp_alloc_bar(nfp);
403 if (priv->bar == NULL)
406 priv->target = target;
407 priv->action = action;
409 priv->offset = address;
412 ret = nfp_reconfigure_bar(nfp, priv->bar, priv->target, priv->action,
413 priv->token, priv->offset, priv->size,
420 nfp6000_area_acquire(struct nfp_cpp_area *area)
422 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
424 /* Calculate offset into BAR. */
425 if (nfp_bar_maptype(priv->bar) ==
426 NFP_PCIE_BAR_PCIE2CPP_MAPTYPE_GENERAL) {
427 priv->bar_offset = priv->offset &
428 (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1);
430 NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(priv->bar,
433 NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(priv->bar, priv->token);
435 priv->bar_offset = priv->offset & priv->bar->mask;
438 /* Must have been too big. Sub-allocate. */
439 if (!priv->bar->iomem)
442 priv->iomem = priv->bar->iomem + priv->bar_offset;
448 nfp6000_area_mapped(struct nfp_cpp_area *area)
450 struct nfp6000_area_priv *area_priv = nfp_cpp_area_priv(area);
452 if (!area_priv->iomem)
455 return area_priv->iomem;
459 nfp6000_area_release(struct nfp_cpp_area *area)
461 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
468 nfp6000_area_iomem(struct nfp_cpp_area *area)
470 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
475 nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
476 unsigned long offset, unsigned int length)
478 uint64_t *wrptr64 = kernel_vaddr;
479 const volatile uint64_t *rdptr64;
480 struct nfp6000_area_priv *priv;
481 uint32_t *wrptr32 = kernel_vaddr;
482 const volatile uint32_t *rdptr32;
487 priv = nfp_cpp_area_priv(area);
488 rdptr64 = (uint64_t *)(priv->iomem + offset);
489 rdptr32 = (uint32_t *)(priv->iomem + offset);
491 if (offset + length > priv->size)
494 width = priv->width.read;
499 /* Unaligned? Translate to an explicit access */
500 if ((priv->offset + offset) & (width - 1)) {
501 printf("aread_read unaligned!!!\n");
505 is_64 = width == TARGET_WIDTH_64;
507 /* MU reads via a PCIe2CPP BAR supports 32bit (and other) lengths */
508 if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
509 priv->action == NFP_CPP_ACTION_RW) {
514 if (offset % sizeof(uint64_t) != 0 ||
515 length % sizeof(uint64_t) != 0)
518 if (offset % sizeof(uint32_t) != 0 ||
519 length % sizeof(uint32_t) != 0)
527 for (n = 0; n < length; n += sizeof(uint64_t)) {
533 for (n = 0; n < length; n += sizeof(uint32_t)) {
543 nfp6000_area_write(struct nfp_cpp_area *area, const void *kernel_vaddr,
544 unsigned long offset, unsigned int length)
546 const uint64_t *rdptr64 = kernel_vaddr;
548 const uint32_t *rdptr32 = kernel_vaddr;
549 struct nfp6000_area_priv *priv;
555 priv = nfp_cpp_area_priv(area);
556 wrptr64 = (uint64_t *)(priv->iomem + offset);
557 wrptr32 = (uint32_t *)(priv->iomem + offset);
559 if (offset + length > priv->size)
562 width = priv->width.write;
567 /* Unaligned? Translate to an explicit access */
568 if ((priv->offset + offset) & (width - 1))
571 is_64 = width == TARGET_WIDTH_64;
573 /* MU writes via a PCIe2CPP BAR supports 32bit (and other) lengths */
574 if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
575 priv->action == NFP_CPP_ACTION_RW)
579 if (offset % sizeof(uint64_t) != 0 ||
580 length % sizeof(uint64_t) != 0)
583 if (offset % sizeof(uint32_t) != 0 ||
584 length % sizeof(uint32_t) != 0)
592 for (n = 0; n < length; n += sizeof(uint64_t)) {
598 for (n = 0; n < length; n += sizeof(uint32_t)) {
607 #define PCI_DEVICES "/sys/bus/pci/devices"
610 nfp_acquire_process_lock(struct nfp_pcie_user *desc)
616 memset(&lock, 0, sizeof(lock));
618 snprintf(lockname, sizeof(lockname), "/var/lock/nfp_%s", desc->busdev);
619 desc->lock = open(lockname, O_RDWR | O_CREAT, 0666);
623 lock.l_type = F_WRLCK;
624 lock.l_whence = SEEK_SET;
627 rc = fcntl(desc->lock, F_SETLKW, &lock);
629 if (errno != EAGAIN && errno != EACCES) {
640 nfp6000_set_model(struct nfp_pcie_user *desc, struct nfp_cpp *cpp)
646 snprintf(tmp_str, sizeof(tmp_str), "%s/%s/config", PCI_DEVICES,
649 fp = open(tmp_str, O_RDONLY);
653 lseek(fp, 0x2e, SEEK_SET);
655 if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
656 printf("Error reading config file for model\n");
665 nfp_cpp_model_set(cpp, tmp);
671 nfp6000_set_interface(struct nfp_pcie_user *desc, struct nfp_cpp *cpp)
677 snprintf(tmp_str, sizeof(tmp_str), "%s/%s/config", PCI_DEVICES,
680 fp = open(tmp_str, O_RDONLY);
684 lseek(fp, 0x154, SEEK_SET);
686 if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
687 printf("error reading config file for interface\n");
694 nfp_cpp_interface_set(cpp, tmp);
699 #define PCI_CFG_SPACE_SIZE 256
700 #define PCI_CFG_SPACE_EXP_SIZE 4096
701 #define PCI_EXT_CAP_ID(header) (int)(header & 0x0000ffff)
702 #define PCI_EXT_CAP_NEXT(header) ((header >> 20) & 0xffc)
703 #define PCI_EXT_CAP_ID_DSN 0x03
705 nfp_pci_find_next_ext_capability(int fp, int cap)
709 int pos = PCI_CFG_SPACE_SIZE;
711 /* minimum 8 bytes per capability */
712 ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
714 lseek(fp, pos, SEEK_SET);
715 if (read(fp, &header, sizeof(header)) != sizeof(header)) {
716 printf("error reading config file for serial\n");
721 * If we have no capabilities, this is indicated by cap ID,
722 * cap version and next pointer all being 0.
728 if (PCI_EXT_CAP_ID(header) == cap)
731 pos = PCI_EXT_CAP_NEXT(header);
732 if (pos < PCI_CFG_SPACE_SIZE)
735 lseek(fp, pos, SEEK_SET);
736 if (read(fp, &header, sizeof(header)) != sizeof(header)) {
737 printf("error reading config file for serial\n");
746 nfp6000_set_serial(struct nfp_pcie_user *desc, struct nfp_cpp *cpp)
754 snprintf(tmp_str, sizeof(tmp_str), "%s/%s/config", PCI_DEVICES,
757 fp = open(tmp_str, O_RDONLY);
761 pos = nfp_pci_find_next_ext_capability(fp, PCI_EXT_CAP_ID_DSN);
763 printf("PCI_EXT_CAP_ID_DSN not found. Using default offset\n");
764 lseek(fp, 0x156, SEEK_SET);
766 lseek(fp, pos + 6, SEEK_SET);
769 if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
770 printf("error reading config file for serial\n");
774 serial[4] = (uint8_t)((tmp >> 8) & 0xff);
775 serial[5] = (uint8_t)(tmp & 0xff);
777 if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
778 printf("error reading config file for serial\n");
782 serial[2] = (uint8_t)((tmp >> 8) & 0xff);
783 serial[3] = (uint8_t)(tmp & 0xff);
785 if (read(fp, &tmp, sizeof(tmp)) != sizeof(tmp)) {
786 printf("error reading config file for serial\n");
790 serial[0] = (uint8_t)((tmp >> 8) & 0xff);
791 serial[1] = (uint8_t)(tmp & 0xff);
796 nfp_cpp_serial_set(cpp, serial, serial_len);
802 nfp6000_set_barsz(struct nfp_pcie_user *desc)
805 unsigned long start, end, flags, tmp;
809 snprintf(tmp_str, sizeof(tmp_str), "%s/%s/resource", PCI_DEVICES,
812 fp = fopen(tmp_str, "r");
816 if (fscanf(fp, "0x%lx 0x%lx 0x%lx", &start, &end, &flags) == 0) {
817 printf("error reading resource file for bar size\n");
821 if (fclose(fp) == -1)
824 tmp = (end - start) + 1;
833 nfp6000_init(struct nfp_cpp *cpp, const char *devname)
840 struct nfp_pcie_user *desc;
842 desc = malloc(sizeof(*desc));
847 memset(desc->busdev, 0, BUSDEV_SZ);
848 strncpy(desc->busdev, devname, strlen(devname));
850 ret = nfp_acquire_process_lock(desc);
854 snprintf(tmp_str, sizeof(tmp_str), "%s/%s/driver", PCI_DEVICES,
857 size = readlink(tmp_str, link, sizeof(link));
862 if (size == sizeof(link))
865 snprintf(tmp_str, sizeof(tmp_str), "%s/%s/resource0", PCI_DEVICES,
868 desc->device = open(tmp_str, O_RDWR);
869 if (desc->device == -1)
872 if (nfp6000_set_model(desc, cpp) < 0)
874 if (nfp6000_set_interface(desc, cpp) < 0)
876 if (nfp6000_set_serial(desc, cpp) < 0)
878 if (nfp6000_set_barsz(desc) < 0)
881 desc->cfg = (char *)mmap(0, 1 << (desc->barsz - 3),
882 PROT_READ | PROT_WRITE,
883 MAP_SHARED, desc->device, 0);
885 if (desc->cfg == MAP_FAILED)
888 nfp_enable_bars(desc);
890 nfp_cpp_priv_set(cpp, desc);
892 model = __nfp_cpp_model_autodetect(cpp);
893 nfp_cpp_model_set(cpp, model);
899 nfp6000_free(struct nfp_cpp *cpp)
901 struct nfp_pcie_user *desc = nfp_cpp_priv(cpp);
904 /* Unmap may cause if there are any pending transaxctions */
905 nfp_disable_bars(desc);
906 munmap(desc->cfg, 1 << (desc->barsz - 3));
908 for (x = ARRAY_SIZE(desc->bar); x > 0; x--) {
909 if (desc->bar[x - 1].iomem)
910 munmap(desc->bar[x - 1].iomem, 1 << (desc->barsz - 3));
917 static const struct nfp_cpp_operations nfp6000_pcie_ops = {
918 .init = nfp6000_init,
919 .free = nfp6000_free,
921 .area_priv_size = sizeof(struct nfp6000_area_priv),
922 .area_init = nfp6000_area_init,
923 .area_acquire = nfp6000_area_acquire,
924 .area_release = nfp6000_area_release,
925 .area_mapped = nfp6000_area_mapped,
926 .area_read = nfp6000_area_read,
927 .area_write = nfp6000_area_write,
928 .area_iomem = nfp6000_area_iomem,
932 nfp_cpp_operations *nfp_cpp_transport_operations(void)
934 return &nfp6000_pcie_ops;