drivers/raw/ifpga_rawdev/base/ifpga_enumerate.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include "opae_hw_api.h"
   6 #include "ifpga_api.h"
   7
   8 #include "ifpga_hw.h"
   9 #include "ifpga_enumerate.h"
  10 #include "ifpga_feature_dev.h"
  11
  12 struct build_feature_devs_info {
  13         struct opae_adapter_data_pci *pci_data;
  14
  15         struct ifpga_afu_info *acc_info;
  16
  17         void *fiu;
  18         enum fpga_id_type current_type;
  19         int current_port_id;
  20
  21         void *ioaddr;
  22         void *ioend;
  23         uint64_t phys_addr;
  24         int current_bar;
  25
  26         void *pfme_hdr;
  27
  28         struct ifpga_hw *hw;
  29 };
  30
  31 static int feature_revision(void __iomem *start)
  32 {
  33         struct feature_header header;
  34
  35         header.csr = readq(start);
  36
  37         return header.revision;
  38 }
  39
  40 static u32 feature_size(void __iomem *start)
  41 {
  42         struct feature_header header;
  43
  44         header.csr = readq(start);
  45
  46         /*the size of private feature is 4KB aligned*/
  47         return header.next_header_offset ? header.next_header_offset:4096;
  48 }
  49
  50 static u64 feature_id(void __iomem *start)
  51 {
  52         struct feature_header header;
  53
  54         header.csr = readq(start);
  55
  56         switch (header.type) {
  57         case FEATURE_TYPE_FIU:
  58                 return FEATURE_ID_FIU_HEADER;
  59         case FEATURE_TYPE_PRIVATE:
  60                 return header.id;
  61         case FEATURE_TYPE_AFU:
  62                 return FEATURE_ID_AFU;
  63         }
  64
  65         WARN_ON(1);
  66         return 0;
  67 }
  68
  69 static int
  70 build_info_add_sub_feature(struct build_feature_devs_info *binfo,
  71                 void __iomem *start, u64 fid, unsigned int size,
  72                 unsigned int vec_start,
  73                 unsigned int vec_cnt)
  74 {
  75         struct ifpga_hw *hw = binfo->hw;
  76         struct feature *feature = NULL;
  77         struct feature_irq_ctx *ctx = NULL;
  78         int port_id, ret = 0;
  79         unsigned int i;
  80
  81         fid = fid?fid:feature_id(start);
  82         size = size?size:feature_size(start);
  83
  84         feature = opae_malloc(sizeof(struct feature));
  85         if (!feature)
  86                 return -ENOMEM;
  87
  88         feature->state = IFPGA_FEATURE_ATTACHED;
  89         feature->addr = start;
  90         feature->id = fid;
  91         feature->size = size;
  92         feature->revision = feature_revision(start);
  93         feature->phys_addr = binfo->phys_addr +
  94                                 ((u8 *)start - (u8 *)binfo->ioaddr);
  95         feature->vec_start = vec_start;
  96         feature->vec_cnt = vec_cnt;
  97
  98         dev_debug(binfo, "%s: id=0x%llx, phys_addr=0x%llx, size=%u\n",
  99                         __func__, (unsigned long long)feature->id,
 100                         (unsigned long long)feature->phys_addr, size);
 101
 102         if (vec_cnt) {
 103                 if (vec_start + vec_cnt <= vec_start)
 104                         return -EINVAL;
 105
 106                 ctx = zmalloc(sizeof(*ctx) * vec_cnt);
 107                 if (!ctx)
 108                         return -ENOMEM;
 109
 110                 for (i = 0; i < vec_cnt; i++) {
 111                         ctx[i].eventfd = -1;
 112                         ctx[i].idx = vec_start + i;
 113                 }
 114         }
 115
 116         feature->ctx = ctx;
 117         feature->ctx_num = vec_cnt;
 118         feature->vfio_dev_fd = binfo->pci_data->vfio_dev_fd;
 119
 120         if (binfo->current_type == FME_ID) {
 121                 feature->parent = &hw->fme;
 122                 feature->type = FEATURE_FME_TYPE;
 123                 feature->name = get_fme_feature_name(fid);
 124                 TAILQ_INSERT_TAIL(&hw->fme.feature_list, feature, next);
 125         } else if (binfo->current_type == PORT_ID) {
 126                 port_id = binfo->current_port_id;
 127                 feature->parent = &hw->port[port_id];
 128                 feature->type = FEATURE_PORT_TYPE;
 129                 feature->name = get_port_feature_name(fid);
 130                 TAILQ_INSERT_TAIL(&hw->port[port_id].feature_list,
 131                                 feature, next);
 132         } else {
 133                 return -EFAULT;
 134         }
 135         return ret;
 136 }
 137
 138 static int
 139 create_feature_instance(struct build_feature_devs_info *binfo,
 140                         void __iomem *start, u64 fid,
 141                         unsigned int size, unsigned int vec_start,
 142                         unsigned int vec_cnt)
 143 {
 144         return build_info_add_sub_feature(binfo, start, fid, size, vec_start,
 145                         vec_cnt);
 146 }
 147
 148 /*
 149  * UAFU GUID is dynamic as it can be changed after FME downloads different
 150  * Green Bitstream to the port, so we treat the unknown GUIDs which are
 151  * attached on port's feature list as UAFU.
 152  */
 153 static bool feature_is_UAFU(struct build_feature_devs_info *binfo)
 154 {
 155         if (binfo->current_type != PORT_ID)
 156                 return false;
 157
 158         return true;
 159 }
 160
 161 static int parse_feature_port_uafu(struct build_feature_devs_info *binfo,
 162                                    struct feature_header *hdr)
 163 {
 164         u64 id = PORT_FEATURE_ID_UAFU;
 165         struct ifpga_afu_info *info;
 166         void *start = (void *)hdr;
 167         struct feature_port_header *port_hdr = binfo->ioaddr;
 168         struct feature_port_capability capability;
 169         int ret;
 170         int size;
 171
 172         capability.csr = readq(&port_hdr->capability);
 173
 174         size = capability.mmio_size << 10;
 175
 176         ret = create_feature_instance(binfo, hdr, id, size, 0, 0);
 177         if (ret)
 178                 return ret;
 179
 180         info = opae_malloc(sizeof(*info));
 181         if (!info)
 182                 return -ENOMEM;
 183
 184         info->region[0].addr = start;
 185         info->region[0].phys_addr = binfo->phys_addr +
 186                         (uint8_t *)start - (uint8_t *)binfo->ioaddr;
 187         info->region[0].len = size;
 188         info->num_regions = 1;
 189
 190         binfo->acc_info = info;
 191
 192         return ret;
 193 }
 194
 195 static int parse_feature_afus(struct build_feature_devs_info *binfo,
 196                               struct feature_header *hdr)
 197 {
 198         int ret;
 199         struct feature_afu_header *afu_hdr, header;
 200         u8 __iomem *start;
 201         u8 __iomem *end = binfo->ioend;
 202
 203         start = (u8 __iomem *)hdr;
 204         for (; start < end; start += header.next_afu) {
 205                 if ((unsigned int)(end - start) <
 206                         (unsigned int)(sizeof(*afu_hdr) + sizeof(*hdr)))
 207                         return -EINVAL;
 208
 209                 hdr = (struct feature_header *)start;
 210                 afu_hdr = (struct feature_afu_header *)(hdr + 1);
 211                 header.csr = readq(&afu_hdr->csr);
 212
 213                 if (feature_is_UAFU(binfo)) {
 214                         ret = parse_feature_port_uafu(binfo, hdr);
 215                         if (ret)
 216                                 return ret;
 217                 }
 218
 219                 if (!header.next_afu)
 220                         break;
 221         }
 222
 223         return 0;
 224 }
 225
 226 /* create and register proper private data */
 227 static int build_info_commit_dev(struct build_feature_devs_info *binfo)
 228 {
 229         struct ifpga_afu_info *info = binfo->acc_info;
 230         struct ifpga_hw *hw = binfo->hw;
 231         struct opae_manager *mgr;
 232         struct opae_bridge *br;
 233         struct opae_accelerator *acc;
 234         struct ifpga_port_hw *port;
 235         struct feature *feature;
 236
 237         if (!binfo->fiu)
 238                 return 0;
 239
 240         if (binfo->current_type == PORT_ID) {
 241                 /* return error if no valid acc info data structure */
 242                 if (!info)
 243                         return -EFAULT;
 244
 245                 br = opae_bridge_alloc(hw->adapter->name, &ifpga_br_ops,
 246                                        binfo->fiu);
 247                 if (!br)
 248                         return -ENOMEM;
 249
 250                 br->id = binfo->current_port_id;
 251
 252                 /* update irq info */
 253                 port = &hw->port[binfo->current_port_id];
 254                 feature = get_feature_by_id(&port->feature_list,
 255                                 PORT_FEATURE_ID_UINT);
 256                 if (feature)
 257                         info->num_irqs = feature->vec_cnt;
 258
 259                 acc = opae_accelerator_alloc(hw->adapter->name,
 260                                              &ifpga_acc_ops, info);
 261                 if (!acc) {
 262                         opae_bridge_free(br);
 263                         return -ENOMEM;
 264                 }
 265
 266                 acc->br = br;
 267                 acc->index = br->id;
 268
 269                 opae_adapter_add_acc(hw->adapter, acc);
 270
 271         } else if (binfo->current_type == FME_ID) {
 272                 mgr = opae_manager_alloc(hw->adapter->name, &ifpga_mgr_ops,
 273                                 binfo->fiu);
 274                 if (!mgr)
 275                         return -ENOMEM;
 276
 277                 mgr->adapter = hw->adapter;
 278                 hw->adapter->mgr = mgr;
 279         }
 280
 281         binfo->fiu = NULL;
 282
 283         return 0;
 284 }
 285
 286 static int
 287 build_info_create_dev(struct build_feature_devs_info *binfo,
 288                       enum fpga_id_type type, unsigned int index)
 289 {
 290         int ret;
 291
 292         ret = build_info_commit_dev(binfo);
 293         if (ret)
 294                 return ret;
 295
 296         binfo->current_type = type;
 297
 298         if (type == FME_ID) {
 299                 binfo->fiu = &binfo->hw->fme;
 300         } else if (type == PORT_ID) {
 301                 binfo->fiu = &binfo->hw->port[index];
 302                 binfo->current_port_id = index;
 303         }
 304
 305         return 0;
 306 }
 307
 308 static int parse_feature_fme(struct build_feature_devs_info *binfo,
 309                              struct feature_header *start)
 310 {
 311         struct ifpga_hw *hw = binfo->hw;
 312         struct ifpga_fme_hw *fme = &hw->fme;
 313         int ret;
 314
 315         ret = build_info_create_dev(binfo, FME_ID, 0);
 316         if (ret)
 317                 return ret;
 318
 319         /* Update FME states */
 320         fme->state = IFPGA_FME_IMPLEMENTED;
 321         fme->parent = hw;
 322         TAILQ_INIT(&fme->feature_list);
 323         spinlock_init(&fme->lock);
 324
 325         return create_feature_instance(binfo, start, 0, 0, 0, 0);
 326 }
 327
 328 static int parse_feature_port(struct build_feature_devs_info *binfo,
 329                               void __iomem *start)
 330 {
 331         struct feature_port_header *port_hdr;
 332         struct feature_port_capability capability;
 333         struct ifpga_hw *hw = binfo->hw;
 334         struct ifpga_port_hw *port;
 335         unsigned int port_id;
 336         int ret;
 337
 338         /* Get current port's id */
 339         port_hdr = (struct feature_port_header *)start;
 340         capability.csr = readq(&port_hdr->capability);
 341         port_id = capability.port_number;
 342
 343         ret = build_info_create_dev(binfo, PORT_ID, port_id);
 344         if (ret)
 345                 return ret;
 346
 347         /*found a Port device*/
 348         port = &hw->port[port_id];
 349         port->port_id = binfo->current_port_id;
 350         port->parent = hw;
 351         port->state = IFPGA_PORT_ATTACHED;
 352         spinlock_init(&port->lock);
 353         TAILQ_INIT(&port->feature_list);
 354
 355         return create_feature_instance(binfo, start, 0, 0, 0, 0);
 356 }
 357
 358 static void enable_port_uafu(struct build_feature_devs_info *binfo,
 359                              void __iomem *start)
 360 {
 361         struct ifpga_port_hw *port = &binfo->hw->port[binfo->current_port_id];
 362
 363         UNUSED(start);
 364
 365         fpga_port_reset(port);
 366 }
 367
 368 static int parse_feature_fiu(struct build_feature_devs_info *binfo,
 369                              struct feature_header *hdr)
 370 {
 371         struct feature_header header;
 372         struct feature_fiu_header *fiu_hdr, fiu_header;
 373         u8 __iomem *start = (u8 __iomem *)hdr;
 374         int ret;
 375
 376         header.csr = readq(hdr);
 377
 378         switch (header.id) {
 379         case FEATURE_FIU_ID_FME:
 380                 ret = parse_feature_fme(binfo, hdr);
 381                 binfo->pfme_hdr = hdr;
 382                 if (ret)
 383                         return ret;
 384                 break;
 385         case FEATURE_FIU_ID_PORT:
 386                 ret = parse_feature_port(binfo, hdr);
 387                 enable_port_uafu(binfo, hdr);
 388                 if (ret)
 389                         return ret;
 390
 391                 /* Check Port FIU's next_afu pointer to User AFU DFH */
 392                 fiu_hdr = (struct feature_fiu_header *)(hdr + 1);
 393                 fiu_header.csr = readq(&fiu_hdr->csr);
 394
 395                 if (fiu_header.next_afu) {
 396                         start += fiu_header.next_afu;
 397                         ret = parse_feature_afus(binfo,
 398                                                 (struct feature_header *)start);
 399                         if (ret)
 400                                 return ret;
 401                 } else {
 402                         dev_info(binfo, "No AFUs detected on Port\n");
 403                 }
 404
 405                 break;
 406         default:
 407                 dev_info(binfo, "FIU TYPE %d is not supported yet.\n",
 408                          header.id);
 409         }
 410
 411         return 0;
 412 }
 413
 414 static void parse_feature_irqs(struct build_feature_devs_info *binfo,
 415                 void __iomem *start, unsigned int *vec_start,
 416                 unsigned int *vec_cnt)
 417 {
 418         UNUSED(binfo);
 419         u64 id;
 420
 421         id = feature_id(start);
 422
 423         if (id == PORT_FEATURE_ID_UINT) {
 424                 struct feature_port_uint *port_uint = start;
 425                 struct feature_port_uint_cap uint_cap;
 426
 427                 uint_cap.csr = readq(&port_uint->capability);
 428                 if (uint_cap.intr_num) {
 429                         *vec_start = uint_cap.first_vec_num;
 430                         *vec_cnt = uint_cap.intr_num;
 431                 } else {
 432                         dev_debug(binfo, "UAFU doesn't support interrupt\n");
 433                 }
 434         } else if (id == PORT_FEATURE_ID_ERROR) {
 435                 struct feature_port_error *port_err = start;
 436                 struct feature_port_err_capability port_err_cap;
 437
 438                 port_err_cap.csr = readq(&port_err->error_capability);
 439                 if (port_err_cap.support_intr) {
 440                         *vec_start = port_err_cap.intr_vector_num;
 441                         *vec_cnt = 1;
 442                 } else {
 443                         dev_debug(&binfo, "Port error doesn't support interrupt\n");
 444                 }
 445
 446         } else if (id == FME_FEATURE_ID_GLOBAL_ERR) {
 447                 struct feature_fme_err *fme_err = start;
 448                 struct feature_fme_error_capability fme_err_cap;
 449
 450                 fme_err_cap.csr = readq(&fme_err->fme_err_capability);
 451                 if (fme_err_cap.support_intr) {
 452                         *vec_start = fme_err_cap.intr_vector_num;
 453                         *vec_cnt = 1;
 454                 } else {
 455                         dev_debug(&binfo, "FME error doesn't support interrupt\n");
 456                 }
 457         }
 458 }
 459
 460 static int parse_feature_fme_private(struct build_feature_devs_info *binfo,
 461                                      struct feature_header *hdr)
 462 {
 463         unsigned int vec_start = 0;
 464         unsigned int vec_cnt = 0;
 465
 466         parse_feature_irqs(binfo, hdr, &vec_start, &vec_cnt);
 467
 468         return create_feature_instance(binfo, hdr, 0, 0, vec_start, vec_cnt);
 469 }
 470
 471 static int parse_feature_port_private(struct build_feature_devs_info *binfo,
 472                                       struct feature_header *hdr)
 473 {
 474         unsigned int vec_start = 0;
 475         unsigned int vec_cnt = 0;
 476
 477         parse_feature_irqs(binfo, hdr, &vec_start, &vec_cnt);
 478
 479         return create_feature_instance(binfo, hdr, 0, 0, vec_start, vec_cnt);
 480 }
 481
 482 static int parse_feature_private(struct build_feature_devs_info *binfo,
 483                                  struct feature_header *hdr)
 484 {
 485         struct feature_header header;
 486
 487         header.csr = readq(hdr);
 488
 489         switch (binfo->current_type) {
 490         case FME_ID:
 491                 return parse_feature_fme_private(binfo, hdr);
 492         case PORT_ID:
 493                 return parse_feature_port_private(binfo, hdr);
 494         default:
 495                 dev_err(binfo, "private feature %x belonging to AFU %d (unknown_type) is not supported yet.\n",
 496                         header.id, binfo->current_type);
 497         }
 498         return 0;
 499 }
 500
 501 static int parse_feature(struct build_feature_devs_info *binfo,
 502                          struct feature_header *hdr)
 503 {
 504         struct feature_header header;
 505         int ret = 0;
 506
 507         header.csr = readq(hdr);
 508
 509         switch (header.type) {
 510         case FEATURE_TYPE_AFU:
 511                 ret = parse_feature_afus(binfo, hdr);
 512                 break;
 513         case FEATURE_TYPE_PRIVATE:
 514                 ret = parse_feature_private(binfo, hdr);
 515                 break;
 516         case FEATURE_TYPE_FIU:
 517                 ret = parse_feature_fiu(binfo, hdr);
 518                 break;
 519         default:
 520                 dev_err(binfo, "Feature Type %x is not supported.\n",
 521                         hdr->type);
 522         };
 523
 524         return ret;
 525 }
 526
 527 static int
 528 parse_feature_list(struct build_feature_devs_info *binfo, u8 __iomem *start)
 529 {
 530         struct feature_header *hdr, header;
 531         u8 __iomem *end = (u8 __iomem *)binfo->ioend;
 532         int ret = 0;
 533
 534         for (; start < end; start += header.next_header_offset) {
 535                 if ((unsigned int)(end - start) < (unsigned int)sizeof(*hdr)) {
 536                         dev_err(binfo, "The region is too small to contain a feature.\n");
 537                         ret =  -EINVAL;
 538                         break;
 539                 }
 540
 541                 hdr = (struct feature_header *)start;
 542                 header.csr = readq(hdr);
 543
 544                 dev_debug(binfo, "%s: address=0x%p, val=0x%llx, header.id=0x%x, header.next_offset=0x%x, header.eol=0x%x, header.type=0x%x\n",
 545                         __func__, hdr, (unsigned long long)header.csr,
 546                         header.id, header.next_header_offset,
 547                         header.end_of_list, header.type);
 548
 549                 ret = parse_feature(binfo, hdr);
 550                 if (ret)
 551                         return ret;
 552
 553                 if (header.end_of_list || !header.next_header_offset)
 554                         break;
 555         }
 556
 557         return build_info_commit_dev(binfo);
 558 }
 559
 560 /* switch the memory mapping to BAR# @bar */
 561 static int parse_switch_to(struct build_feature_devs_info *binfo, int bar)
 562 {
 563         struct opae_adapter_data_pci *pci_data = binfo->pci_data;
 564
 565         if (!pci_data->region[bar].addr)
 566                 return -ENOMEM;
 567
 568         binfo->ioaddr = pci_data->region[bar].addr;
 569         binfo->ioend = (u8 __iomem *)binfo->ioaddr + pci_data->region[bar].len;
 570         binfo->phys_addr = pci_data->region[bar].phys_addr;
 571         binfo->current_bar = bar;
 572
 573         return 0;
 574 }
 575
 576 static int parse_ports_from_fme(struct build_feature_devs_info *binfo)
 577 {
 578         struct feature_fme_header *fme_hdr;
 579         struct feature_fme_port port;
 580         int i = 0, ret = 0;
 581
 582         if (!binfo->pfme_hdr) {
 583                 dev_info(binfo,  "VF is detected.\n");
 584                 return ret;
 585         }
 586
 587         fme_hdr = binfo->pfme_hdr;
 588
 589         do {
 590                 port.csr = readq(&fme_hdr->port[i]);
 591                 if (!port.port_implemented)
 592                         break;
 593
 594                 /* skip port which only could be accessed via VF */
 595                 if (port.afu_access_control == FME_AFU_ACCESS_VF)
 596                         continue;
 597
 598                 ret = parse_switch_to(binfo, port.port_bar);
 599                 if (ret)
 600                         break;
 601
 602                 ret = parse_feature_list(binfo,
 603                                          (u8 __iomem *)binfo->ioaddr +
 604                                           port.port_offset);
 605                 if (ret)
 606                         break;
 607         } while (++i < MAX_FPGA_PORT_NUM);
 608
 609         return ret;
 610 }
 611
 612 static struct build_feature_devs_info *
 613 build_info_alloc_and_init(struct ifpga_hw *hw)
 614 {
 615         struct build_feature_devs_info *binfo;
 616
 617         binfo = zmalloc(sizeof(*binfo));
 618         if (!binfo)
 619                 return binfo;
 620
 621         binfo->hw = hw;
 622         binfo->pci_data = hw->pci_data;
 623
 624         /* fpga feature list starts from BAR 0 */
 625         if (parse_switch_to(binfo, 0)) {
 626                 free(binfo);
 627                 return NULL;
 628         }
 629
 630         return binfo;
 631 }
 632
 633 static void build_info_free(struct build_feature_devs_info *binfo)
 634 {
 635         free(binfo);
 636 }
 637
 638 static void ifpga_print_device_feature_list(struct ifpga_hw *hw)
 639 {
 640         struct ifpga_fme_hw *fme = &hw->fme;
 641         struct ifpga_port_hw *port;
 642         struct feature *feature;
 643         int i;
 644
 645         dev_info(hw, "found fme_device, is in PF: %s\n",
 646                  is_ifpga_hw_pf(hw) ? "yes" : "no");
 647
 648         ifpga_for_each_fme_feature(fme, feature) {
 649                 if (feature->state != IFPGA_FEATURE_ATTACHED)
 650                         continue;
 651
 652                 dev_info(hw, "%12s:     %p - %p  - paddr: 0x%lx\n",
 653                          feature->name, feature->addr,
 654                          feature->addr + feature->size - 1,
 655                          (unsigned long)feature->phys_addr);
 656
 657         }
 658
 659         for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
 660                 port = &hw->port[i];
 661
 662                 if (port->state != IFPGA_PORT_ATTACHED)
 663                         continue;
 664
 665                 dev_info(hw, "port device: %d\n", port->port_id);
 666
 667                 ifpga_for_each_port_feature(port, feature) {
 668                         if (feature->state != IFPGA_FEATURE_ATTACHED)
 669                                 continue;
 670
 671                         dev_info(hw, "%12s:     %p - %p  - paddr:0x%lx\n",
 672                                  feature->name,
 673                                  feature->addr,
 674                                  feature->addr +
 675                                  feature->size - 1,
 676                                  (unsigned long)feature->phys_addr);
 677                 }
 678
 679         }
 680 }
 681
 682 int ifpga_bus_enumerate(struct ifpga_hw *hw)
 683 {
 684         struct build_feature_devs_info *binfo;
 685         int ret;
 686
 687         binfo = build_info_alloc_and_init(hw);
 688         if (!binfo)
 689                 return -ENOMEM;
 690
 691         ret = parse_feature_list(binfo, binfo->ioaddr);
 692         if (ret)
 693                 goto exit;
 694
 695         ret = parse_ports_from_fme(binfo);
 696         if (ret)
 697                 goto exit;
 698
 699         ifpga_print_device_feature_list(hw);
 700
 701 exit:
 702         build_info_free(binfo);
 703         return ret;
 704 }
 705
 706 int ifpga_bus_init(struct ifpga_hw *hw)
 707 {
 708         int i;
 709         struct ifpga_port_hw *port;
 710
 711         fme_hw_init(&hw->fme);
 712         for (i = 0; i < MAX_FPGA_PORT_NUM; i++) {
 713                 port = &hw->port[i];
 714                 port_hw_init(port);
 715         }
 716
 717         return 0;
 718 }