drivers/raw/ifpga/ifpga_rawdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6 #include <dirent.h>
   7 #include <sys/stat.h>
   8 #include <unistd.h>
   9 #include <sys/types.h>
  10 #include <fcntl.h>
  11 #include <sys/ioctl.h>
  12 #include <sys/epoll.h>
  13 #include <rte_log.h>
  14 #include <rte_bus.h>
  15 #include <rte_malloc.h>
  16 #include <rte_devargs.h>
  17 #include <rte_memcpy.h>
  18 #include <rte_pci.h>
  19 #include <rte_bus_pci.h>
  20 #include <rte_kvargs.h>
  21 #include <rte_alarm.h>
  22 #include <rte_interrupts.h>
  23 #include <rte_errno.h>
  24 #include <rte_per_lcore.h>
  25 #include <rte_memory.h>
  26 #include <rte_memzone.h>
  27 #include <rte_eal.h>
  28 #include <rte_common.h>
  29 #include <rte_bus_vdev.h>
  30 #include <rte_string_fns.h>
  31 #include <rte_pmd_i40e.h>
  32
  33 #include "base/opae_hw_api.h"
  34 #include "base/opae_ifpga_hw_api.h"
  35 #include "base/ifpga_api.h"
  36 #include "rte_rawdev.h"
  37 #include "rte_rawdev_pmd.h"
  38 #include "rte_bus_ifpga.h"
  39 #include "ifpga_common.h"
  40 #include "ifpga_logs.h"
  41 #include "ifpga_rawdev.h"
  42 #include "ipn3ke_rawdev_api.h"
  43
  44 #define PCI_VENDOR_ID_INTEL          0x8086
  45 /* PCI Device ID */
  46 #define PCIE_DEVICE_ID_PF_INT_5_X    0xBCBD
  47 #define PCIE_DEVICE_ID_PF_INT_6_X    0xBCC0
  48 #define PCIE_DEVICE_ID_PF_DSC_1_X    0x09C4
  49 #define PCIE_DEVICE_ID_PAC_N3000     0x0B30
  50 /* VF Device */
  51 #define PCIE_DEVICE_ID_VF_INT_5_X    0xBCBF
  52 #define PCIE_DEVICE_ID_VF_INT_6_X    0xBCC1
  53 #define PCIE_DEVICE_ID_VF_DSC_1_X    0x09C5
  54 #define PCIE_DEVICE_ID_VF_PAC_N3000  0x0B31
  55 #define RTE_MAX_RAW_DEVICE           10
  56
  57 static const struct rte_pci_id pci_ifpga_map[] = {
  58         { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_INT_5_X) },
  59         { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_VF_INT_5_X) },
  60         { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_INT_6_X) },
  61         { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_VF_INT_6_X) },
  62         { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PF_DSC_1_X) },
  63         { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_VF_DSC_1_X) },
  64         { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_PAC_N3000),},
  65         { RTE_PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCIE_DEVICE_ID_VF_PAC_N3000),},
  66         { .vendor_id = 0, /* sentinel */ },
  67 };
  68
  69 static struct ifpga_rawdev ifpga_rawdevices[IFPGA_RAWDEV_NUM];
  70
  71 static int ifpga_monitor_refcnt;
  72 static pthread_t ifpga_monitor_start_thread;
  73
  74 static struct ifpga_rawdev *
  75 ifpga_rawdev_allocate(struct rte_rawdev *rawdev);
  76 static int set_surprise_link_check_aer(
  77                 struct ifpga_rawdev *ifpga_rdev, int force_disable);
  78 static int ifpga_pci_find_next_ext_capability(unsigned int fd,
  79                                               int start, uint32_t cap);
  80 static int ifpga_pci_find_ext_capability(unsigned int fd, uint32_t cap);
  81
  82 struct ifpga_rawdev *
  83 ifpga_rawdev_get(const struct rte_rawdev *rawdev)
  84 {
  85         struct ifpga_rawdev *dev;
  86         unsigned int i;
  87
  88         if (rawdev == NULL)
  89                 return NULL;
  90
  91         for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
  92                 dev = &ifpga_rawdevices[i];
  93                 if (dev->rawdev == rawdev)
  94                         return dev;
  95         }
  96
  97         return NULL;
  98 }
  99
 100 static inline uint8_t
 101 ifpga_rawdev_find_free_device_index(void)
 102 {
 103         uint16_t dev_id;
 104
 105         for (dev_id = 0; dev_id < IFPGA_RAWDEV_NUM; dev_id++) {
 106                 if (ifpga_rawdevices[dev_id].rawdev == NULL)
 107                         return dev_id;
 108         }
 109
 110         return IFPGA_RAWDEV_NUM;
 111 }
 112 static struct ifpga_rawdev *
 113 ifpga_rawdev_allocate(struct rte_rawdev *rawdev)
 114 {
 115         struct ifpga_rawdev *dev;
 116         uint16_t dev_id;
 117         int i = 0;
 118
 119         dev = ifpga_rawdev_get(rawdev);
 120         if (dev != NULL) {
 121                 IFPGA_RAWDEV_PMD_ERR("Event device already allocated!");
 122                 return NULL;
 123         }
 124
 125         dev_id = ifpga_rawdev_find_free_device_index();
 126         if (dev_id == IFPGA_RAWDEV_NUM) {
 127                 IFPGA_RAWDEV_PMD_ERR("Reached maximum number of raw devices");
 128                 return NULL;
 129         }
 130
 131         dev = &ifpga_rawdevices[dev_id];
 132         dev->rawdev = rawdev;
 133         dev->dev_id = dev_id;
 134         for (i = 0; i < IFPGA_MAX_IRQ; i++)
 135                 dev->intr_handle[i] = NULL;
 136         dev->poll_enabled = 0;
 137
 138         return dev;
 139 }
 140
 141 static int
 142 ifpga_pci_find_next_ext_capability(unsigned int fd, int start, uint32_t cap)
 143 {
 144         uint32_t header;
 145         int ttl;
 146         int pos = RTE_PCI_CFG_SPACE_SIZE;
 147         int ret;
 148
 149         /* minimum 8 bytes per capability */
 150         ttl = (RTE_PCI_CFG_SPACE_EXP_SIZE - RTE_PCI_CFG_SPACE_SIZE) / 8;
 151
 152         if (start)
 153                 pos = start;
 154         ret = pread(fd, &header, sizeof(header), pos);
 155         if (ret == -1)
 156                 return -1;
 157
 158         /*
 159          * If we have no capabilities, this is indicated by cap ID,
 160          * cap version and next pointer all being 0.
 161          */
 162         if (header == 0)
 163                 return 0;
 164
 165         while (ttl-- > 0) {
 166                 if (RTE_PCI_EXT_CAP_ID(header) == cap && pos != start)
 167                         return pos;
 168
 169                 pos = RTE_PCI_EXT_CAP_NEXT(header);
 170                 if (pos < RTE_PCI_CFG_SPACE_SIZE)
 171                         break;
 172                 ret = pread(fd, &header, sizeof(header), pos);
 173                 if (ret == -1)
 174                         return -1;
 175         }
 176
 177         return 0;
 178 }
 179
 180 static int
 181 ifpga_pci_find_ext_capability(unsigned int fd, uint32_t cap)
 182 {
 183         return ifpga_pci_find_next_ext_capability(fd, 0, cap);
 184 }
 185
 186 static int ifpga_get_dev_vendor_id(const char *bdf,
 187         uint32_t *dev_id, uint32_t *vendor_id)
 188 {
 189         int fd;
 190         char path[1024];
 191         int ret;
 192         uint32_t header;
 193
 194         strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
 195         strlcat(path, bdf, sizeof(path));
 196         strlcat(path, "/config", sizeof(path));
 197         fd = open(path, O_RDWR);
 198         if (fd < 0)
 199                 return -1;
 200         ret = pread(fd, &header, sizeof(header), 0);
 201         if (ret == -1) {
 202                 close(fd);
 203                 return -1;
 204         }
 205         (*vendor_id) = header & 0xffff;
 206         (*dev_id) = (header >> 16) & 0xffff;
 207         close(fd);
 208
 209         return 0;
 210 }
 211
 212 static int ifpga_rawdev_fill_info(struct ifpga_rawdev *ifpga_dev)
 213 {
 214         struct opae_adapter *adapter = NULL;
 215         char path[1024] = "/sys/bus/pci/devices/";
 216         char link[1024], link1[1024];
 217         char dir[1024] = "/sys/devices/";
 218         char *c;
 219         int ret;
 220         char sub_brg_bdf[4][16] = {{0}};
 221         int point;
 222         DIR *dp = NULL;
 223         struct dirent *entry;
 224         int i, j;
 225
 226         unsigned int dom, bus, dev;
 227         int func;
 228         uint32_t dev_id, vendor_id;
 229
 230         adapter = ifpga_dev ? ifpga_rawdev_get_priv(ifpga_dev->rawdev) : NULL;
 231         if (!adapter)
 232                 return -ENODEV;
 233
 234         strlcat(path, adapter->name, sizeof(path));
 235         memset(link, 0, sizeof(link));
 236         memset(link1, 0, sizeof(link1));
 237         ret = readlink(path, link, (sizeof(link)-1));
 238         if ((ret < 0) || ((unsigned int)ret > (sizeof(link)-1)))
 239                 return -1;
 240         link[ret] = 0;   /* terminate string with null character */
 241         strlcpy(link1, link, sizeof(link1));
 242         memset(ifpga_dev->parent_bdf, 0, 16);
 243         point = strlen(link);
 244         if (point < 39)
 245                 return -1;
 246         point -= 39;
 247         link[point] = 0;
 248         if (point < 12)
 249                 return -1;
 250         point -= 12;
 251         rte_memcpy(ifpga_dev->parent_bdf, &link[point], 12);
 252
 253         point = strlen(link1);
 254         if (point < 26)
 255                 return -1;
 256         point -= 26;
 257         link1[point] = 0;
 258         if (point < 12)
 259                 return -1;
 260         point -= 12;
 261         c = strchr(link1, 'p');
 262         if (!c)
 263                 return -1;
 264         strlcat(dir, c, sizeof(dir));
 265
 266         /* scan folder */
 267         dp = opendir(dir);
 268         if (dp == NULL)
 269                 return -1;
 270         i = 0;
 271         while ((entry = readdir(dp)) != NULL) {
 272                 if (i >= 4)
 273                         break;
 274                 if (entry->d_name[0] == '.')
 275                         continue;
 276                 if (strlen(entry->d_name) > 12)
 277                         continue;
 278                 if (sscanf(entry->d_name, "%x:%x:%x.%d",
 279                         &dom, &bus, &dev, &func) < 4)
 280                         continue;
 281                 else {
 282                         strlcpy(sub_brg_bdf[i],
 283                                 entry->d_name,
 284                                 sizeof(sub_brg_bdf[i]));
 285                         i++;
 286                 }
 287         }
 288         closedir(dp);
 289
 290         /* get fpga and fvl */
 291         j = 0;
 292         for (i = 0; i < 4; i++) {
 293                 strlcpy(link, dir, sizeof(link));
 294                 strlcat(link, "/", sizeof(link));
 295                 strlcat(link, sub_brg_bdf[i], sizeof(link));
 296                 dp = opendir(link);
 297                 if (dp == NULL)
 298                         return -1;
 299                 while ((entry = readdir(dp)) != NULL) {
 300                         if (j >= 8)
 301                                 break;
 302                         if (entry->d_name[0] == '.')
 303                                 continue;
 304
 305                         if (strlen(entry->d_name) > 12)
 306                                 continue;
 307                         if (sscanf(entry->d_name, "%x:%x:%x.%d",
 308                                 &dom, &bus, &dev, &func) < 4)
 309                                 continue;
 310                         else {
 311                                 if (ifpga_get_dev_vendor_id(entry->d_name,
 312                                         &dev_id, &vendor_id))
 313                                         continue;
 314                                 if (vendor_id == 0x8086 &&
 315                                         (dev_id == 0x0CF8 ||
 316                                         dev_id == 0x0D58 ||
 317                                         dev_id == 0x1580)) {
 318                                         strlcpy(ifpga_dev->fvl_bdf[j],
 319                                                 entry->d_name,
 320                                                 sizeof(ifpga_dev->fvl_bdf[j]));
 321                                         j++;
 322                                 }
 323                         }
 324                 }
 325                 closedir(dp);
 326         }
 327
 328         return 0;
 329 }
 330
 331 #define HIGH_FATAL(_sens, value)\
 332         (((_sens)->flags & OPAE_SENSOR_HIGH_FATAL_VALID) &&\
 333          (value > (_sens)->high_fatal))
 334
 335 #define HIGH_WARN(_sens, value)\
 336         (((_sens)->flags & OPAE_SENSOR_HIGH_WARN_VALID) &&\
 337          (value > (_sens)->high_warn))
 338
 339 #define LOW_FATAL(_sens, value)\
 340         (((_sens)->flags & OPAE_SENSOR_LOW_FATAL_VALID) &&\
 341          (value > (_sens)->low_fatal))
 342
 343 #define LOW_WARN(_sens, value)\
 344         (((_sens)->flags & OPAE_SENSOR_LOW_WARN_VALID) &&\
 345          (value > (_sens)->low_warn))
 346
 347 #define AUX_VOLTAGE_WARN 11400
 348
 349 static int
 350 ifpga_monitor_sensor(struct rte_rawdev *raw_dev,
 351                bool *gsd_start)
 352 {
 353         struct opae_adapter *adapter;
 354         struct opae_manager *mgr;
 355         struct opae_sensor_info *sensor;
 356         unsigned int value;
 357         int ret;
 358
 359         adapter = ifpga_rawdev_get_priv(raw_dev);
 360         if (!adapter)
 361                 return -ENODEV;
 362
 363         mgr = opae_adapter_get_mgr(adapter);
 364         if (!mgr)
 365                 return -ENODEV;
 366
 367         opae_mgr_for_each_sensor(mgr, sensor) {
 368                 if (!(sensor->flags & OPAE_SENSOR_VALID))
 369                         goto fail;
 370
 371                 ret = opae_mgr_get_sensor_value(mgr, sensor, &value);
 372                 if (ret)
 373                         goto fail;
 374
 375                 if (value == 0xdeadbeef) {
 376                         IFPGA_RAWDEV_PMD_ERR("dev_id %d sensor %s value %x\n",
 377                                         raw_dev->dev_id, sensor->name, value);
 378                         continue;
 379                 }
 380
 381                 /* monitor temperature sensors */
 382                 if (!strcmp(sensor->name, "Board Temperature") ||
 383                                 !strcmp(sensor->name, "FPGA Die Temperature")) {
 384                         IFPGA_RAWDEV_PMD_DEBUG("read sensor %s %d %d %d\n",
 385                                         sensor->name, value, sensor->high_warn,
 386                                         sensor->high_fatal);
 387
 388                         if (HIGH_WARN(sensor, value) ||
 389                                 LOW_WARN(sensor, value)) {
 390                                 IFPGA_RAWDEV_PMD_INFO("%s reach threshold %d\n",
 391                                         sensor->name, value);
 392                                 *gsd_start = true;
 393                                 break;
 394                         }
 395                 }
 396
 397                 /* monitor 12V AUX sensor */
 398                 if (!strcmp(sensor->name, "12V AUX Voltage")) {
 399                         if (value < AUX_VOLTAGE_WARN) {
 400                                 IFPGA_RAWDEV_PMD_INFO(
 401                                         "%s reach threshold %d mV\n",
 402                                         sensor->name, value);
 403                                 *gsd_start = true;
 404                                 break;
 405                         }
 406                 }
 407         }
 408
 409         return 0;
 410 fail:
 411         return -EFAULT;
 412 }
 413
 414 static int set_surprise_link_check_aer(
 415         struct ifpga_rawdev *ifpga_rdev, int force_disable)
 416 {
 417         struct rte_rawdev *rdev;
 418         int fd = -1;
 419         char path[1024];
 420         int pos;
 421         int ret;
 422         uint32_t data;
 423         bool enable = 0;
 424         uint32_t aer_new0, aer_new1;
 425
 426         if (!ifpga_rdev || !ifpga_rdev->rawdev) {
 427                 printf("\n device does not exist\n");
 428                 return -EFAULT;
 429         }
 430
 431         rdev = ifpga_rdev->rawdev;
 432         if (ifpga_rdev->aer_enable)
 433                 return -EFAULT;
 434         if (ifpga_monitor_sensor(rdev, &enable))
 435                 return -EFAULT;
 436         if (enable || force_disable) {
 437                 IFPGA_RAWDEV_PMD_ERR("Set AER, pls graceful shutdown\n");
 438                 ifpga_rdev->aer_enable = 1;
 439                 /* get bridge fd */
 440                 strlcpy(path, "/sys/bus/pci/devices/", sizeof(path));
 441                 strlcat(path, ifpga_rdev->parent_bdf, sizeof(path));
 442                 strlcat(path, "/config", sizeof(path));
 443                 fd = open(path, O_RDWR);
 444                 if (fd < 0)
 445                         goto end;
 446                 pos = ifpga_pci_find_ext_capability(fd, RTE_PCI_EXT_CAP_ID_ERR);
 447                 if (!pos)
 448                         goto end;
 449                 /* save previous ECAP_AER+0x08 */
 450                 ret = pread(fd, &data, sizeof(data), pos+0x08);
 451                 if (ret == -1)
 452                         goto end;
 453                 ifpga_rdev->aer_old[0] = data;
 454                 /* save previous ECAP_AER+0x14 */
 455                 ret = pread(fd, &data, sizeof(data), pos+0x14);
 456                 if (ret == -1)
 457                         goto end;
 458                 ifpga_rdev->aer_old[1] = data;
 459
 460                 /* set ECAP_AER+0x08 to 0xFFFFFFFF */
 461                 data = 0xffffffff;
 462                 ret = pwrite(fd, &data, 4, pos+0x08);
 463                 if (ret == -1)
 464                         goto end;
 465                 /* set ECAP_AER+0x14 to 0xFFFFFFFF */
 466                 ret = pwrite(fd, &data, 4, pos+0x14);
 467                 if (ret == -1)
 468                         goto end;
 469
 470                 /* read current ECAP_AER+0x08 */
 471                 ret = pread(fd, &data, sizeof(data), pos+0x08);
 472                 if (ret == -1)
 473                         goto end;
 474                 aer_new0 = data;
 475                 /* read current ECAP_AER+0x14 */
 476                 ret = pread(fd, &data, sizeof(data), pos+0x14);
 477                 if (ret == -1)
 478                         goto end;
 479                 aer_new1 = data;
 480
 481                 if (fd != -1)
 482                         close(fd);
 483
 484                 printf(">>>>>>Set AER %x,%x %x,%x\n",
 485                         ifpga_rdev->aer_old[0], ifpga_rdev->aer_old[1],
 486                         aer_new0, aer_new1);
 487
 488                 return 1;
 489                 }
 490
 491 end:
 492         if (fd != -1)
 493                 close(fd);
 494         return -EFAULT;
 495 }
 496
 497 static void *
 498 ifpga_rawdev_gsd_handle(__rte_unused void *param)
 499 {
 500         struct ifpga_rawdev *ifpga_rdev;
 501         int i;
 502         int gsd_enable, ret;
 503 #define MS 1000
 504
 505         while (__atomic_load_n(&ifpga_monitor_refcnt, __ATOMIC_RELAXED)) {
 506                 gsd_enable = 0;
 507                 for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
 508                         ifpga_rdev = &ifpga_rawdevices[i];
 509                         if (ifpga_rdev->poll_enabled) {
 510                                 ret = set_surprise_link_check_aer(ifpga_rdev,
 511                                         gsd_enable);
 512                                 if (ret == 1 && !gsd_enable) {
 513                                         gsd_enable = 1;
 514                                         i = -1;
 515                                 }
 516                         }
 517                 }
 518
 519                 if (gsd_enable)
 520                         printf(">>>>>>Pls Shutdown APP\n");
 521
 522                 rte_delay_us(100 * MS);
 523         }
 524
 525         return NULL;
 526 }
 527
 528 static int
 529 ifpga_monitor_start_func(struct ifpga_rawdev *dev)
 530 {
 531         int ret;
 532
 533         if (!dev)
 534                 return -ENODEV;
 535
 536         ret = ifpga_rawdev_fill_info(dev);
 537         if (ret)
 538                 return ret;
 539
 540         dev->poll_enabled = 1;
 541
 542         if (!__atomic_fetch_add(&ifpga_monitor_refcnt, 1, __ATOMIC_RELAXED)) {
 543                 ret = rte_ctrl_thread_create(&ifpga_monitor_start_thread,
 544                                              "ifpga-monitor", NULL,
 545                                              ifpga_rawdev_gsd_handle, NULL);
 546                 if (ret != 0) {
 547                         ifpga_monitor_start_thread = 0;
 548                         IFPGA_RAWDEV_PMD_ERR(
 549                                 "Fail to create ifpga monitor thread");
 550                         return -1;
 551                 }
 552         }
 553
 554         return 0;
 555 }
 556
 557 static int
 558 ifpga_monitor_stop_func(struct ifpga_rawdev *dev)
 559 {
 560         int ret;
 561
 562         if (!dev || !dev->poll_enabled)
 563                 return 0;
 564
 565         dev->poll_enabled = 0;
 566
 567         if (!__atomic_sub_fetch(&ifpga_monitor_refcnt, 1, __ATOMIC_RELAXED) &&
 568                 ifpga_monitor_start_thread) {
 569                 ret = pthread_cancel(ifpga_monitor_start_thread);
 570                 if (ret)
 571                         IFPGA_RAWDEV_PMD_ERR("Can't cancel the thread");
 572
 573                 ret = pthread_join(ifpga_monitor_start_thread, NULL);
 574                 if (ret)
 575                         IFPGA_RAWDEV_PMD_ERR("Can't join the thread");
 576
 577                 return ret;
 578         }
 579
 580         return 0;
 581 }
 582
 583 static int
 584 ifpga_fill_afu_dev(struct opae_accelerator *acc,
 585                 struct rte_afu_device *afu_dev)
 586 {
 587         struct rte_mem_resource *res = afu_dev->mem_resource;
 588         struct opae_acc_region_info region_info;
 589         struct opae_acc_info info;
 590         unsigned long i;
 591         int ret;
 592
 593         ret = opae_acc_get_info(acc, &info);
 594         if (ret)
 595                 return ret;
 596
 597         if (info.num_regions > PCI_MAX_RESOURCE)
 598                 return -EFAULT;
 599
 600         afu_dev->num_region = info.num_regions;
 601
 602         for (i = 0; i < info.num_regions; i++) {
 603                 region_info.index = i;
 604                 ret = opae_acc_get_region_info(acc, &region_info);
 605                 if (ret)
 606                         return ret;
 607
 608                 if ((region_info.flags & ACC_REGION_MMIO) &&
 609                     (region_info.flags & ACC_REGION_READ) &&
 610                     (region_info.flags & ACC_REGION_WRITE)) {
 611                         res[i].phys_addr = region_info.phys_addr;
 612                         res[i].len = region_info.len;
 613                         res[i].addr = region_info.addr;
 614                 } else
 615                         return -EFAULT;
 616         }
 617
 618         return 0;
 619 }
 620
 621 static int
 622 ifpga_rawdev_info_get(struct rte_rawdev *dev,
 623                       rte_rawdev_obj_t dev_info,
 624                       size_t dev_info_size)
 625 {
 626         struct opae_adapter *adapter;
 627         struct opae_accelerator *acc;
 628         struct rte_afu_device *afu_dev;
 629         struct opae_manager *mgr = NULL;
 630         struct opae_eth_group_region_info opae_lside_eth_info;
 631         struct opae_eth_group_region_info opae_nside_eth_info;
 632         int lside_bar_idx, nside_bar_idx;
 633
 634         IFPGA_RAWDEV_PMD_FUNC_TRACE();
 635
 636         if (!dev_info || dev_info_size != sizeof(*afu_dev)) {
 637                 IFPGA_RAWDEV_PMD_ERR("Invalid request");
 638                 return -EINVAL;
 639         }
 640
 641         adapter = ifpga_rawdev_get_priv(dev);
 642         if (!adapter)
 643                 return -ENOENT;
 644
 645         afu_dev = dev_info;
 646         afu_dev->rawdev = dev;
 647
 648         /* find opae_accelerator and fill info into afu_device */
 649         opae_adapter_for_each_acc(adapter, acc) {
 650                 if (acc->index != afu_dev->id.port)
 651                         continue;
 652
 653                 if (ifpga_fill_afu_dev(acc, afu_dev)) {
 654                         IFPGA_RAWDEV_PMD_ERR("cannot get info\n");
 655                         return -ENOENT;
 656                 }
 657         }
 658
 659         /* get opae_manager to rawdev */
 660         mgr = opae_adapter_get_mgr(adapter);
 661         if (mgr) {
 662                 /* get LineSide BAR Index */
 663                 if (opae_manager_get_eth_group_region_info(mgr, 0,
 664                         &opae_lside_eth_info)) {
 665                         return -ENOENT;
 666                 }
 667                 lside_bar_idx = opae_lside_eth_info.mem_idx;
 668
 669                 /* get NICSide BAR Index */
 670                 if (opae_manager_get_eth_group_region_info(mgr, 1,
 671                         &opae_nside_eth_info)) {
 672                         return -ENOENT;
 673                 }
 674                 nside_bar_idx = opae_nside_eth_info.mem_idx;
 675
 676                 if (lside_bar_idx >= PCI_MAX_RESOURCE ||
 677                         nside_bar_idx >= PCI_MAX_RESOURCE ||
 678                         lside_bar_idx == nside_bar_idx)
 679                         return -ENOENT;
 680
 681                 /* fill LineSide BAR Index */
 682                 afu_dev->mem_resource[lside_bar_idx].phys_addr =
 683                         opae_lside_eth_info.phys_addr;
 684                 afu_dev->mem_resource[lside_bar_idx].len =
 685                         opae_lside_eth_info.len;
 686                 afu_dev->mem_resource[lside_bar_idx].addr =
 687                         opae_lside_eth_info.addr;
 688
 689                 /* fill NICSide BAR Index */
 690                 afu_dev->mem_resource[nside_bar_idx].phys_addr =
 691                         opae_nside_eth_info.phys_addr;
 692                 afu_dev->mem_resource[nside_bar_idx].len =
 693                         opae_nside_eth_info.len;
 694                 afu_dev->mem_resource[nside_bar_idx].addr =
 695                         opae_nside_eth_info.addr;
 696         }
 697         return 0;
 698 }
 699
 700 static int
 701 ifpga_rawdev_configure(const struct rte_rawdev *dev,
 702                 rte_rawdev_obj_t config,
 703                 size_t config_size __rte_unused)
 704 {
 705         IFPGA_RAWDEV_PMD_FUNC_TRACE();
 706
 707         RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
 708
 709         return config ? 0 : 1;
 710 }
 711
 712 static int
 713 ifpga_rawdev_start(struct rte_rawdev *dev)
 714 {
 715         int ret = 0;
 716         struct opae_adapter *adapter;
 717
 718         IFPGA_RAWDEV_PMD_FUNC_TRACE();
 719
 720         RTE_FUNC_PTR_OR_ERR_RET(dev, -EINVAL);
 721
 722         adapter = ifpga_rawdev_get_priv(dev);
 723         if (!adapter)
 724                 return -ENODEV;
 725
 726         return ret;
 727 }
 728
 729 static void
 730 ifpga_rawdev_stop(struct rte_rawdev *dev)
 731 {
 732         dev->started = 0;
 733 }
 734
 735 static int
 736 ifpga_rawdev_close(struct rte_rawdev *dev)
 737 {
 738         struct opae_adapter *adapter;
 739
 740         if (dev) {
 741                 ifpga_monitor_stop_func(ifpga_rawdev_get(dev));
 742                 adapter = ifpga_rawdev_get_priv(dev);
 743                 if (adapter) {
 744                         opae_adapter_destroy(adapter);
 745                         opae_adapter_data_free(adapter->data);
 746                 }
 747         }
 748
 749         return dev ? 0:1;
 750 }
 751
 752 static int
 753 ifpga_rawdev_reset(struct rte_rawdev *dev)
 754 {
 755         return dev ? 0:1;
 756 }
 757
 758 static int
 759 fpga_pr(struct rte_rawdev *raw_dev, u32 port_id, const char *buffer, u32 size,
 760                         u64 *status)
 761 {
 762
 763         struct opae_adapter *adapter;
 764         struct opae_manager *mgr;
 765         struct opae_accelerator *acc;
 766         struct opae_bridge *br;
 767         int ret;
 768
 769         adapter = ifpga_rawdev_get_priv(raw_dev);
 770         if (!adapter)
 771                 return -ENODEV;
 772
 773         mgr = opae_adapter_get_mgr(adapter);
 774         if (!mgr)
 775                 return -ENODEV;
 776
 777         acc = opae_adapter_get_acc(adapter, port_id);
 778         if (!acc)
 779                 return -ENODEV;
 780
 781         br = opae_acc_get_br(acc);
 782         if (!br)
 783                 return -ENODEV;
 784
 785         ret = opae_manager_flash(mgr, port_id, buffer, size, status);
 786         if (ret) {
 787                 IFPGA_RAWDEV_PMD_ERR("%s pr error %d\n", __func__, ret);
 788                 return ret;
 789         }
 790
 791         ret = opae_bridge_reset(br);
 792         if (ret) {
 793                 IFPGA_RAWDEV_PMD_ERR("%s reset port:%d error %d\n",
 794                                 __func__, port_id, ret);
 795                 return ret;
 796         }
 797
 798         return ret;
 799 }
 800
 801 static int
 802 rte_fpga_do_pr(struct rte_rawdev *rawdev, int port_id,
 803                 const char *file_name)
 804 {
 805         struct stat file_stat;
 806         int file_fd;
 807         int ret = 0;
 808         ssize_t buffer_size;
 809         void *buffer, *buf_to_free;
 810         u64 pr_error;
 811
 812         if (!file_name)
 813                 return -EINVAL;
 814
 815         file_fd = open(file_name, O_RDONLY);
 816         if (file_fd < 0) {
 817                 IFPGA_RAWDEV_PMD_ERR("%s: open file error: %s\n",
 818                                 __func__, file_name);
 819                 IFPGA_RAWDEV_PMD_ERR("Message : %s\n", strerror(errno));
 820                 return -EINVAL;
 821         }
 822         ret = stat(file_name, &file_stat);
 823         if (ret) {
 824                 IFPGA_RAWDEV_PMD_ERR("stat on bitstream file failed: %s\n",
 825                                 file_name);
 826                 ret = -EINVAL;
 827                 goto close_fd;
 828         }
 829         buffer_size = file_stat.st_size;
 830         if (buffer_size <= 0) {
 831                 ret = -EINVAL;
 832                 goto close_fd;
 833         }
 834
 835         IFPGA_RAWDEV_PMD_INFO("bitstream file size: %zu\n", buffer_size);
 836         buffer = rte_malloc(NULL, buffer_size, 0);
 837         if (!buffer) {
 838                 ret = -ENOMEM;
 839                 goto close_fd;
 840         }
 841         buf_to_free = buffer;
 842
 843         /*read the raw data*/
 844         if (buffer_size != read(file_fd, (void *)buffer, buffer_size)) {
 845                 ret = -EINVAL;
 846                 goto free_buffer;
 847         }
 848
 849         /*do PR now*/
 850         ret = fpga_pr(rawdev, port_id, buffer, buffer_size, &pr_error);
 851         IFPGA_RAWDEV_PMD_INFO("downloading to device port %d....%s.\n", port_id,
 852                 ret ? "failed" : "success");
 853         if (ret) {
 854                 ret = -EINVAL;
 855                 goto free_buffer;
 856         }
 857
 858 free_buffer:
 859         rte_free(buf_to_free);
 860 close_fd:
 861         close(file_fd);
 862         file_fd = 0;
 863         return ret;
 864 }
 865
 866 static int
 867 ifpga_rawdev_pr(struct rte_rawdev *dev,
 868         rte_rawdev_obj_t pr_conf)
 869 {
 870         struct opae_adapter *adapter;
 871         struct opae_manager *mgr;
 872         struct opae_board_info *info;
 873         struct rte_afu_pr_conf *afu_pr_conf;
 874         int ret;
 875         struct uuid uuid;
 876         struct opae_accelerator *acc;
 877
 878         IFPGA_RAWDEV_PMD_FUNC_TRACE();
 879
 880         adapter = ifpga_rawdev_get_priv(dev);
 881         if (!adapter)
 882                 return -ENODEV;
 883
 884         if (!pr_conf)
 885                 return -EINVAL;
 886
 887         afu_pr_conf = pr_conf;
 888
 889         if (afu_pr_conf->pr_enable) {
 890                 ret = rte_fpga_do_pr(dev,
 891                                 afu_pr_conf->afu_id.port,
 892                                 afu_pr_conf->bs_path);
 893                 if (ret) {
 894                         IFPGA_RAWDEV_PMD_ERR("do pr error %d\n", ret);
 895                         return ret;
 896                 }
 897         }
 898
 899         mgr = opae_adapter_get_mgr(adapter);
 900         if (!mgr) {
 901                 IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
 902                 return -1;
 903         }
 904
 905         if (ifpga_mgr_ops.get_board_info(mgr, &info)) {
 906                 IFPGA_RAWDEV_PMD_ERR("ifpga manager get_board_info fail!");
 907                 return -1;
 908         }
 909
 910         if (info->lightweight) {
 911                 /* set uuid to all 0, when fpga is lightweight image */
 912                 memset(&afu_pr_conf->afu_id.uuid.uuid_low, 0, sizeof(u64));
 913                 memset(&afu_pr_conf->afu_id.uuid.uuid_high, 0, sizeof(u64));
 914         } else {
 915                 acc = opae_adapter_get_acc(adapter, afu_pr_conf->afu_id.port);
 916                 if (!acc)
 917                         return -ENODEV;
 918
 919                 ret = opae_acc_get_uuid(acc, &uuid);
 920                 if (ret)
 921                         return ret;
 922
 923                 rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_low, uuid.b,
 924                         sizeof(u64));
 925                 rte_memcpy(&afu_pr_conf->afu_id.uuid.uuid_high, uuid.b + 8,
 926                         sizeof(u64));
 927
 928                 IFPGA_RAWDEV_PMD_INFO("%s: uuid_l=0x%lx, uuid_h=0x%lx\n",
 929                         __func__,
 930                         (unsigned long)afu_pr_conf->afu_id.uuid.uuid_low,
 931                         (unsigned long)afu_pr_conf->afu_id.uuid.uuid_high);
 932                 }
 933         return 0;
 934 }
 935
 936 static int
 937 ifpga_rawdev_get_attr(struct rte_rawdev *dev,
 938         const char *attr_name, uint64_t *attr_value)
 939 {
 940         struct opae_adapter *adapter;
 941         struct opae_manager *mgr;
 942         struct opae_retimer_info opae_rtm_info;
 943         struct opae_retimer_status opae_rtm_status;
 944         struct opae_eth_group_info opae_eth_grp_info;
 945         struct opae_eth_group_region_info opae_eth_grp_reg_info;
 946         int eth_group_num = 0;
 947         uint64_t port_link_bitmap = 0, port_link_bit;
 948         uint32_t i, j, p, q;
 949
 950 #define MAX_PORT_PER_RETIMER    4
 951
 952         IFPGA_RAWDEV_PMD_FUNC_TRACE();
 953
 954         if (!dev || !attr_name || !attr_value) {
 955                 IFPGA_RAWDEV_PMD_ERR("Invalid arguments for getting attributes");
 956                 return -1;
 957         }
 958
 959         adapter = ifpga_rawdev_get_priv(dev);
 960         if (!adapter) {
 961                 IFPGA_RAWDEV_PMD_ERR("Adapter of dev %s is NULL", dev->name);
 962                 return -1;
 963         }
 964
 965         mgr = opae_adapter_get_mgr(adapter);
 966         if (!mgr) {
 967                 IFPGA_RAWDEV_PMD_ERR("opae_manager of opae_adapter is NULL");
 968                 return -1;
 969         }
 970
 971         /* currently, eth_group_num is always 2 */
 972         eth_group_num = opae_manager_get_eth_group_nums(mgr);
 973         if (eth_group_num < 0)
 974                 return -1;
 975
 976         if (!strcmp(attr_name, "LineSideBaseMAC")) {
 977                 /* Currently FPGA not implement, so just set all zeros*/
 978                 *attr_value = (uint64_t)0;
 979                 return 0;
 980         }
 981         if (!strcmp(attr_name, "LineSideMACType")) {
 982                 /* eth_group 0 on FPGA connect to LineSide */
 983                 if (opae_manager_get_eth_group_info(mgr, 0,
 984                         &opae_eth_grp_info))
 985                         return -1;
 986                 switch (opae_eth_grp_info.speed) {
 987                 case ETH_SPEED_10G:
 988                         *attr_value =
 989                         (uint64_t)(IFPGA_RAWDEV_RETIMER_MAC_TYPE_10GE_XFI);
 990                         break;
 991                 case ETH_SPEED_25G:
 992                         *attr_value =
 993                         (uint64_t)(IFPGA_RAWDEV_RETIMER_MAC_TYPE_25GE_25GAUI);
 994                         break;
 995                 default:
 996                         *attr_value =
 997                         (uint64_t)(IFPGA_RAWDEV_RETIMER_MAC_TYPE_UNKNOWN);
 998                         break;
 999                 }
1000                 return 0;
1001         }
1002         if (!strcmp(attr_name, "LineSideLinkSpeed")) {
1003                 if (opae_manager_get_retimer_status(mgr, &opae_rtm_status))
1004                         return -1;
1005                 switch (opae_rtm_status.speed) {
1006                 case MXD_1GB:
1007                         *attr_value =
1008                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_UNKNOWN);
1009                         break;
1010                 case MXD_2_5GB:
1011                         *attr_value =
1012                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_UNKNOWN);
1013                         break;
1014                 case MXD_5GB:
1015                         *attr_value =
1016                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_UNKNOWN);
1017                         break;
1018                 case MXD_10GB:
1019                         *attr_value =
1020                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_10GB);
1021                         break;
1022                 case MXD_25GB:
1023                         *attr_value =
1024                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_25GB);
1025                         break;
1026                 case MXD_40GB:
1027                         *attr_value =
1028                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_40GB);
1029                         break;
1030                 case MXD_100GB:
1031                         *attr_value =
1032                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_UNKNOWN);
1033                         break;
1034                 case MXD_SPEED_UNKNOWN:
1035                         *attr_value =
1036                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_UNKNOWN);
1037                         break;
1038                 default:
1039                         *attr_value =
1040                                 (uint64_t)(IFPGA_RAWDEV_LINK_SPEED_UNKNOWN);
1041                         break;
1042                 }
1043                 return 0;
1044         }
1045         if (!strcmp(attr_name, "LineSideLinkRetimerNum")) {
1046                 if (opae_manager_get_retimer_info(mgr, &opae_rtm_info))
1047                         return -1;
1048                 *attr_value = (uint64_t)(opae_rtm_info.nums_retimer);
1049                 return 0;
1050         }
1051         if (!strcmp(attr_name, "LineSideLinkPortNum")) {
1052                 if (opae_manager_get_retimer_info(mgr, &opae_rtm_info))
1053                         return -1;
1054                 uint64_t tmp = (uint64_t)opae_rtm_info.ports_per_retimer *
1055                                         (uint64_t)opae_rtm_info.nums_retimer;
1056                 *attr_value = tmp;
1057                 return 0;
1058         }
1059         if (!strcmp(attr_name, "LineSideLinkStatus")) {
1060                 if (opae_manager_get_retimer_info(mgr, &opae_rtm_info))
1061                         return -1;
1062                 if (opae_manager_get_retimer_status(mgr, &opae_rtm_status))
1063                         return -1;
1064                 (*attr_value) = 0;
1065                 q = 0;
1066                 port_link_bitmap = (uint64_t)(opae_rtm_status.line_link_bitmap);
1067                 for (i = 0; i < opae_rtm_info.nums_retimer; i++) {
1068                         p = i * MAX_PORT_PER_RETIMER;
1069                         for (j = 0; j < opae_rtm_info.ports_per_retimer; j++) {
1070                                 port_link_bit = 0;
1071                                 IFPGA_BIT_SET(port_link_bit, (p+j));
1072                                 port_link_bit &= port_link_bitmap;
1073                                 if (port_link_bit)
1074                                         IFPGA_BIT_SET((*attr_value), q);
1075                                 q++;
1076                         }
1077                 }
1078                 return 0;
1079         }
1080         if (!strcmp(attr_name, "LineSideBARIndex")) {
1081                 /* eth_group 0 on FPGA connect to LineSide */
1082                 if (opae_manager_get_eth_group_region_info(mgr, 0,
1083                         &opae_eth_grp_reg_info))
1084                         return -1;
1085                 *attr_value = (uint64_t)opae_eth_grp_reg_info.mem_idx;
1086                 return 0;
1087         }
1088         if (!strcmp(attr_name, "NICSideMACType")) {
1089                 /* eth_group 1 on FPGA connect to NicSide */
1090                 if (opae_manager_get_eth_group_info(mgr, 1,
1091                         &opae_eth_grp_info))
1092                         return -1;
1093                 *attr_value = (uint64_t)(opae_eth_grp_info.speed);
1094                 return 0;
1095         }
1096         if (!strcmp(attr_name, "NICSideLinkSpeed")) {
1097                 /* eth_group 1 on FPGA connect to NicSide */
1098                 if (opae_manager_get_eth_group_info(mgr, 1,
1099                         &opae_eth_grp_info))
1100                         return -1;
1101                 *attr_value = (uint64_t)(opae_eth_grp_info.speed);
1102                 return 0;
1103         }
1104         if (!strcmp(attr_name, "NICSideLinkPortNum")) {
1105                 if (opae_manager_get_retimer_info(mgr, &opae_rtm_info))
1106                         return -1;
1107                 uint64_t tmp = (uint64_t)opae_rtm_info.nums_fvl *
1108                                         (uint64_t)opae_rtm_info.ports_per_fvl;
1109                 *attr_value = tmp;
1110                 return 0;
1111         }
1112         if (!strcmp(attr_name, "NICSideLinkStatus"))
1113                 return 0;
1114         if (!strcmp(attr_name, "NICSideBARIndex")) {
1115                 /* eth_group 1 on FPGA connect to NicSide */
1116                 if (opae_manager_get_eth_group_region_info(mgr, 1,
1117                         &opae_eth_grp_reg_info))
1118                         return -1;
1119                 *attr_value = (uint64_t)opae_eth_grp_reg_info.mem_idx;
1120                 return 0;
1121         }
1122
1123         IFPGA_RAWDEV_PMD_ERR("%s not support", attr_name);
1124         return -1;
1125 }
1126
1127 static const struct rte_rawdev_ops ifpga_rawdev_ops = {
1128         .dev_info_get = ifpga_rawdev_info_get,
1129         .dev_configure = ifpga_rawdev_configure,
1130         .dev_start = ifpga_rawdev_start,
1131         .dev_stop = ifpga_rawdev_stop,
1132         .dev_close = ifpga_rawdev_close,
1133         .dev_reset = ifpga_rawdev_reset,
1134
1135         .queue_def_conf = NULL,
1136         .queue_setup = NULL,
1137         .queue_release = NULL,
1138
1139         .attr_get = ifpga_rawdev_get_attr,
1140         .attr_set = NULL,
1141
1142         .enqueue_bufs = NULL,
1143         .dequeue_bufs = NULL,
1144
1145         .dump = NULL,
1146
1147         .xstats_get = NULL,
1148         .xstats_get_names = NULL,
1149         .xstats_get_by_name = NULL,
1150         .xstats_reset = NULL,
1151
1152         .firmware_status_get = NULL,
1153         .firmware_version_get = NULL,
1154         .firmware_load = ifpga_rawdev_pr,
1155         .firmware_unload = NULL,
1156
1157         .dev_selftest = NULL,
1158 };
1159
1160 static int
1161 ifpga_get_fme_error_prop(struct opae_manager *mgr,
1162                 u64 prop_id, u64 *val)
1163 {
1164         struct feature_prop prop;
1165
1166         prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
1167         prop.prop_id = prop_id;
1168
1169         if (opae_manager_ifpga_get_prop(mgr, &prop))
1170                 return -EINVAL;
1171
1172         *val = prop.data;
1173
1174         return 0;
1175 }
1176
1177 static int
1178 ifpga_set_fme_error_prop(struct opae_manager *mgr,
1179                 u64 prop_id, u64 val)
1180 {
1181         struct feature_prop prop;
1182
1183         prop.feature_id = IFPGA_FME_FEATURE_ID_GLOBAL_ERR;
1184         prop.prop_id = prop_id;
1185
1186         prop.data = val;
1187
1188         if (opae_manager_ifpga_set_prop(mgr, &prop))
1189                 return -EINVAL;
1190
1191         return 0;
1192 }
1193
1194 static int
1195 fme_err_read_seu_emr(struct opae_manager *mgr)
1196 {
1197         u64 val;
1198         int ret;
1199
1200         ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_LOW, &val);
1201         if (ret)
1202                 return -EINVAL;
1203
1204         IFPGA_RAWDEV_PMD_INFO("seu emr low: 0x%" PRIx64 "\n", val);
1205
1206         ret = ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_SEU_EMR_HIGH, &val);
1207         if (ret)
1208                 return -EINVAL;
1209
1210         IFPGA_RAWDEV_PMD_INFO("seu emr high: 0x%" PRIx64 "\n", val);
1211
1212         return 0;
1213 }
1214
1215 static int fme_clear_warning_intr(struct opae_manager *mgr)
1216 {
1217         u64 val;
1218
1219         if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_INJECT_ERRORS, 0))
1220                 return -EINVAL;
1221
1222         if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
1223                 return -EINVAL;
1224         if ((val & 0x40) != 0)
1225                 IFPGA_RAWDEV_PMD_INFO("clean not done\n");
1226
1227         return 0;
1228 }
1229
1230 static int fme_clean_fme_error(struct opae_manager *mgr)
1231 {
1232         u64 val;
1233
1234         if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
1235                 return -EINVAL;
1236
1237         IFPGA_RAWDEV_PMD_DEBUG("before clean 0x%" PRIx64 "\n", val);
1238
1239         ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_CLEAR, val);
1240
1241         if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
1242                 return -EINVAL;
1243
1244         IFPGA_RAWDEV_PMD_DEBUG("after clean 0x%" PRIx64 "\n", val);
1245
1246         return 0;
1247 }
1248
1249 static int
1250 fme_err_handle_error0(struct opae_manager *mgr)
1251 {
1252         struct feature_fme_error0 fme_error0;
1253         u64 val;
1254
1255         if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, &val))
1256                 return -EINVAL;
1257
1258         if (fme_clean_fme_error(mgr))
1259                 return -EINVAL;
1260
1261         fme_error0.csr = val;
1262
1263         if (fme_error0.fabric_err)
1264                 IFPGA_RAWDEV_PMD_ERR("Fabric error\n");
1265         else if (fme_error0.fabfifo_overflow)
1266                 IFPGA_RAWDEV_PMD_ERR("Fabric fifo under/overflow error\n");
1267         else if (fme_error0.afu_acc_mode_err)
1268                 IFPGA_RAWDEV_PMD_ERR("AFU PF/VF access mismatch detected\n");
1269         else if (fme_error0.pcie0cdc_parity_err)
1270                 IFPGA_RAWDEV_PMD_ERR("PCIe0 CDC Parity Error\n");
1271         else if (fme_error0.cvlcdc_parity_err)
1272                 IFPGA_RAWDEV_PMD_ERR("CVL CDC Parity Error\n");
1273         else if (fme_error0.fpgaseuerr)
1274                 fme_err_read_seu_emr(mgr);
1275
1276         /* clean the errors */
1277         if (ifpga_set_fme_error_prop(mgr, FME_ERR_PROP_ERRORS, val))
1278                 return -EINVAL;
1279
1280         return 0;
1281 }
1282
1283 static int
1284 fme_err_handle_catfatal_error(struct opae_manager *mgr)
1285 {
1286         struct feature_fme_ras_catfaterror fme_catfatal;
1287         u64 val;
1288
1289         if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_CATFATAL_ERRORS, &val))
1290                 return -EINVAL;
1291
1292         fme_catfatal.csr = val;
1293
1294         if (fme_catfatal.cci_fatal_err)
1295                 IFPGA_RAWDEV_PMD_ERR("CCI error detected\n");
1296         else if (fme_catfatal.fabric_fatal_err)
1297                 IFPGA_RAWDEV_PMD_ERR("Fabric fatal error detected\n");
1298         else if (fme_catfatal.pcie_poison_err)
1299                 IFPGA_RAWDEV_PMD_ERR("Poison error from PCIe ports\n");
1300         else if (fme_catfatal.inject_fata_err)
1301                 IFPGA_RAWDEV_PMD_ERR("Injected Fatal Error\n");
1302         else if (fme_catfatal.crc_catast_err)
1303                 IFPGA_RAWDEV_PMD_ERR("a catastrophic EDCRC error\n");
1304         else if (fme_catfatal.injected_catast_err)
1305                 IFPGA_RAWDEV_PMD_ERR("Injected Catastrophic Error\n");
1306         else if (fme_catfatal.bmc_seu_catast_err)
1307                 fme_err_read_seu_emr(mgr);
1308
1309         return 0;
1310 }
1311
1312 static int
1313 fme_err_handle_nonfaterror(struct opae_manager *mgr)
1314 {
1315         struct feature_fme_ras_nonfaterror nonfaterr;
1316         u64 val;
1317
1318         if (ifpga_get_fme_error_prop(mgr, FME_ERR_PROP_NONFATAL_ERRORS, &val))
1319                 return -EINVAL;
1320
1321         nonfaterr.csr = val;
1322
1323         if (nonfaterr.temp_thresh_ap1)
1324                 IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP1\n");
1325         else if (nonfaterr.temp_thresh_ap2)
1326                 IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP2\n");
1327         else if (nonfaterr.pcie_error)
1328                 IFPGA_RAWDEV_PMD_INFO("an error has occurred in pcie\n");
1329         else if (nonfaterr.portfatal_error)
1330                 IFPGA_RAWDEV_PMD_INFO("fatal error occurred in AFU port.\n");
1331         else if (nonfaterr.proc_hot)
1332                 IFPGA_RAWDEV_PMD_INFO("a ProcHot event\n");
1333         else if (nonfaterr.afu_acc_mode_err)
1334                 IFPGA_RAWDEV_PMD_INFO("an AFU PF/VF access mismatch\n");
1335         else if (nonfaterr.injected_nonfata_err) {
1336                 IFPGA_RAWDEV_PMD_INFO("Injected Warning Error\n");
1337                 fme_clear_warning_intr(mgr);
1338         } else if (nonfaterr.temp_thresh_AP6)
1339                 IFPGA_RAWDEV_PMD_INFO("Temperature threshold triggered AP6\n");
1340         else if (nonfaterr.power_thresh_AP1)
1341                 IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP1\n");
1342         else if (nonfaterr.power_thresh_AP2)
1343                 IFPGA_RAWDEV_PMD_INFO("Power threshold triggered AP2\n");
1344         else if (nonfaterr.mbp_err)
1345                 IFPGA_RAWDEV_PMD_INFO("an MBP event\n");
1346
1347         return 0;
1348 }
1349
1350 static void
1351 fme_interrupt_handler(void *param)
1352 {
1353         struct opae_manager *mgr = (struct opae_manager *)param;
1354
1355         IFPGA_RAWDEV_PMD_INFO("%s interrupt occurred\n", __func__);
1356
1357         fme_err_handle_error0(mgr);
1358         fme_err_handle_nonfaterror(mgr);
1359         fme_err_handle_catfatal_error(mgr);
1360 }
1361
1362 int
1363 ifpga_unregister_msix_irq(struct ifpga_rawdev *dev, enum ifpga_irq_type type,
1364                 int vec_start, rte_intr_callback_fn handler, void *arg)
1365 {
1366         struct rte_intr_handle **intr_handle;
1367         int rc = 0;
1368         int i = vec_start + 1;
1369
1370         if (!dev)
1371                 return -ENODEV;
1372
1373         if (type == IFPGA_FME_IRQ)
1374                 intr_handle = (struct rte_intr_handle **)&dev->intr_handle[0];
1375         else if (type == IFPGA_AFU_IRQ)
1376                 intr_handle = (struct rte_intr_handle **)&dev->intr_handle[i];
1377         else
1378                 return -EINVAL;
1379
1380         if ((*intr_handle) == NULL) {
1381                 IFPGA_RAWDEV_PMD_ERR("%s interrupt %d not registered\n",
1382                         type == IFPGA_FME_IRQ ? "FME" : "AFU",
1383                         type == IFPGA_FME_IRQ ? 0 : vec_start);
1384                 return -ENOENT;
1385         }
1386
1387         rte_intr_efd_disable(*intr_handle);
1388
1389         rc = rte_intr_callback_unregister(*intr_handle, handler, arg);
1390         if (rc < 0) {
1391                 IFPGA_RAWDEV_PMD_ERR("Failed to unregister %s interrupt %d\n",
1392                         type == IFPGA_FME_IRQ ? "FME" : "AFU",
1393                         type == IFPGA_FME_IRQ ? 0 : vec_start);
1394         } else {
1395                 rte_intr_instance_free(*intr_handle);
1396                 *intr_handle = NULL;
1397         }
1398
1399         return rc;
1400 }
1401
1402 int
1403 ifpga_register_msix_irq(struct ifpga_rawdev *dev, int port_id,
1404                 enum ifpga_irq_type type, int vec_start, int count,
1405                 rte_intr_callback_fn handler, const char *name,
1406                 void *arg)
1407 {
1408         int ret;
1409         struct rte_intr_handle **intr_handle;
1410         struct opae_adapter *adapter;
1411         struct opae_manager *mgr;
1412         struct opae_accelerator *acc;
1413         int *intr_efds = NULL, nb_intr, i;
1414
1415         if (!dev || !dev->rawdev)
1416                 return -ENODEV;
1417
1418         adapter = ifpga_rawdev_get_priv(dev->rawdev);
1419         if (!adapter)
1420                 return -ENODEV;
1421
1422         mgr = opae_adapter_get_mgr(adapter);
1423         if (!mgr)
1424                 return -ENODEV;
1425
1426         if (type == IFPGA_FME_IRQ) {
1427                 intr_handle = (struct rte_intr_handle **)&dev->intr_handle[0];
1428                 count = 1;
1429         } else if (type == IFPGA_AFU_IRQ) {
1430                 i = vec_start + 1;
1431                 intr_handle = (struct rte_intr_handle **)&dev->intr_handle[i];
1432         } else {
1433                 return -EINVAL;
1434         }
1435
1436         if (*intr_handle)
1437                 return -EBUSY;
1438
1439         *intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE);
1440         if (!(*intr_handle))
1441                 return -ENOMEM;
1442
1443         if (rte_intr_type_set(*intr_handle, RTE_INTR_HANDLE_VFIO_MSIX))
1444                 return -rte_errno;
1445
1446         ret = rte_intr_efd_enable(*intr_handle, count);
1447         if (ret)
1448                 return -ENODEV;
1449
1450         if (rte_intr_fd_set(*intr_handle,
1451                         rte_intr_efds_index_get(*intr_handle, 0)))
1452                 return -rte_errno;
1453
1454         IFPGA_RAWDEV_PMD_DEBUG("register %s irq, vfio_fd=%d, fd=%d\n",
1455                         name, rte_intr_dev_fd_get(*intr_handle),
1456                         rte_intr_fd_get(*intr_handle));
1457
1458         if (type == IFPGA_FME_IRQ) {
1459                 struct fpga_fme_err_irq_set err_irq_set;
1460                 err_irq_set.evtfd = rte_intr_efds_index_get(*intr_handle,
1461                                                                    0);
1462
1463                 ret = opae_manager_ifpga_set_err_irq(mgr, &err_irq_set);
1464                 if (ret)
1465                         return -EINVAL;
1466         } else if (type == IFPGA_AFU_IRQ) {
1467                 acc = opae_adapter_get_acc(adapter, port_id);
1468                 if (!acc)
1469                         return -EINVAL;
1470
1471                 nb_intr = rte_intr_nb_intr_get(*intr_handle);
1472
1473                 intr_efds = calloc(nb_intr, sizeof(int));
1474                 if (!intr_efds)
1475                         return -ENOMEM;
1476
1477                 for (i = 0; i < nb_intr; i++)
1478                         intr_efds[i] = rte_intr_efds_index_get(*intr_handle, i);
1479
1480                 ret = opae_acc_set_irq(acc, vec_start, count, intr_efds);
1481                 if (ret) {
1482                         free(intr_efds);
1483                         return -EINVAL;
1484                 }
1485         }
1486
1487         /* register interrupt handler using DPDK API */
1488         ret = rte_intr_callback_register(*intr_handle,
1489                         handler, (void *)arg);
1490         if (ret) {
1491                 free(intr_efds);
1492                 return -EINVAL;
1493         }
1494
1495         IFPGA_RAWDEV_PMD_INFO("success register %s interrupt\n", name);
1496
1497         free(intr_efds);
1498         return 0;
1499 }
1500
1501 static int
1502 ifpga_rawdev_create(struct rte_pci_device *pci_dev,
1503                         int socket_id)
1504 {
1505         int ret = 0;
1506         struct rte_rawdev *rawdev = NULL;
1507         struct ifpga_rawdev *dev = NULL;
1508         struct opae_adapter *adapter = NULL;
1509         struct opae_manager *mgr = NULL;
1510         struct opae_adapter_data_pci *data = NULL;
1511         char name[RTE_RAWDEV_NAME_MAX_LEN];
1512         int i;
1513
1514         if (!pci_dev) {
1515                 IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
1516                 ret = -EINVAL;
1517                 goto cleanup;
1518         }
1519
1520         memset(name, 0, sizeof(name));
1521         snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, IFPGA_RAWDEV_NAME_FMT,
1522                 pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
1523
1524         IFPGA_RAWDEV_PMD_INFO("Init %s on NUMA node %d", name, rte_socket_id());
1525
1526         /* Allocate device structure */
1527         rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct opae_adapter),
1528                                          socket_id);
1529         if (rawdev == NULL) {
1530                 IFPGA_RAWDEV_PMD_ERR("Unable to allocate rawdevice");
1531                 ret = -EINVAL;
1532                 goto cleanup;
1533         }
1534
1535         ipn3ke_bridge_func.get_ifpga_rawdev = ifpga_rawdev_get;
1536         ipn3ke_bridge_func.set_i40e_sw_dev = rte_pmd_i40e_set_switch_dev;
1537
1538         dev = ifpga_rawdev_allocate(rawdev);
1539         if (dev == NULL) {
1540                 IFPGA_RAWDEV_PMD_ERR("Unable to allocate ifpga_rawdevice");
1541                 ret = -EINVAL;
1542                 goto cleanup;
1543         }
1544         dev->aer_enable = 0;
1545
1546         /* alloc OPAE_FPGA_PCI data to register to OPAE hardware level API */
1547         data = opae_adapter_data_alloc(OPAE_FPGA_PCI);
1548         if (!data) {
1549                 ret = -ENOMEM;
1550                 goto cleanup;
1551         }
1552
1553         /* init opae_adapter_data_pci for device specific information */
1554         for (i = 0; i < PCI_MAX_RESOURCE; i++) {
1555                 data->region[i].phys_addr = pci_dev->mem_resource[i].phys_addr;
1556                 data->region[i].len = pci_dev->mem_resource[i].len;
1557                 data->region[i].addr = pci_dev->mem_resource[i].addr;
1558         }
1559         data->device_id = pci_dev->id.device_id;
1560         data->vendor_id = pci_dev->id.vendor_id;
1561         data->bus = pci_dev->addr.bus;
1562         data->devid = pci_dev->addr.devid;
1563         data->function = pci_dev->addr.function;
1564         data->vfio_dev_fd = rte_intr_dev_fd_get(pci_dev->intr_handle);
1565
1566         adapter = rawdev->dev_private;
1567         /* create a opae_adapter based on above device data */
1568         ret = opae_adapter_init(adapter, pci_dev->device.name, data);
1569         if (ret) {
1570                 ret = -ENOMEM;
1571                 goto free_adapter_data;
1572         }
1573
1574         rawdev->dev_ops = &ifpga_rawdev_ops;
1575         rawdev->device = &pci_dev->device;
1576         rawdev->driver_name = pci_dev->driver->driver.name;
1577
1578         /* must enumerate the adapter before use it */
1579         ret = opae_adapter_enumerate(adapter);
1580         if (ret)
1581                 goto free_adapter_data;
1582
1583         /* get opae_manager to rawdev */
1584         mgr = opae_adapter_get_mgr(adapter);
1585         if (mgr) {
1586                 /* PF function */
1587                 IFPGA_RAWDEV_PMD_INFO("this is a PF function");
1588         }
1589
1590         ret = ifpga_register_msix_irq(dev, 0, IFPGA_FME_IRQ, 0, 0,
1591                         fme_interrupt_handler, "fme_irq", mgr);
1592         if (ret)
1593                 goto free_adapter_data;
1594
1595         ret = ifpga_monitor_start_func(dev);
1596         if (ret)
1597                 goto free_adapter_data;
1598
1599         return ret;
1600
1601 free_adapter_data:
1602         if (data)
1603                 opae_adapter_data_free(data);
1604 cleanup:
1605         if (rawdev)
1606                 rte_rawdev_pmd_release(rawdev);
1607
1608         return ret;
1609 }
1610
1611 static int
1612 ifpga_rawdev_destroy(struct rte_pci_device *pci_dev)
1613 {
1614         int ret;
1615         struct rte_rawdev *rawdev;
1616         char name[RTE_RAWDEV_NAME_MAX_LEN];
1617         struct opae_adapter *adapter;
1618         struct opae_manager *mgr;
1619         struct ifpga_rawdev *dev;
1620
1621         if (!pci_dev) {
1622                 IFPGA_RAWDEV_PMD_ERR("Invalid pci_dev of the device!");
1623                 ret = -EINVAL;
1624                 return ret;
1625         }
1626
1627         memset(name, 0, sizeof(name));
1628         snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, IFPGA_RAWDEV_NAME_FMT,
1629                 pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function);
1630
1631         IFPGA_RAWDEV_PMD_INFO("Closing %s on NUMA node %d",
1632                 name, rte_socket_id());
1633
1634         rawdev = rte_rawdev_pmd_get_named_dev(name);
1635         if (!rawdev) {
1636                 IFPGA_RAWDEV_PMD_ERR("Invalid device name (%s)", name);
1637                 return -EINVAL;
1638         }
1639         dev = ifpga_rawdev_get(rawdev);
1640         if (dev)
1641                 dev->rawdev = NULL;
1642
1643         adapter = ifpga_rawdev_get_priv(rawdev);
1644         if (!adapter)
1645                 return -ENODEV;
1646
1647         mgr = opae_adapter_get_mgr(adapter);
1648         if (!mgr)
1649                 return -ENODEV;
1650
1651         if (ifpga_unregister_msix_irq(dev, IFPGA_FME_IRQ, 0,
1652                                 fme_interrupt_handler, mgr) < 0)
1653                 return -EINVAL;
1654
1655         /* rte_rawdev_close is called by pmd_release */
1656         ret = rte_rawdev_pmd_release(rawdev);
1657         if (ret)
1658                 IFPGA_RAWDEV_PMD_DEBUG("Device cleanup failed");
1659
1660         return ret;
1661 }
1662
1663 static int
1664 ifpga_rawdev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1665         struct rte_pci_device *pci_dev)
1666 {
1667         IFPGA_RAWDEV_PMD_FUNC_TRACE();
1668         return ifpga_rawdev_create(pci_dev, rte_socket_id());
1669 }
1670
1671 static int
1672 ifpga_rawdev_pci_remove(struct rte_pci_device *pci_dev)
1673 {
1674         IFPGA_RAWDEV_PMD_INFO("remove pci_dev %s", pci_dev->device.name);
1675         return ifpga_rawdev_destroy(pci_dev);
1676 }
1677
1678 static struct rte_pci_driver rte_ifpga_rawdev_pmd = {
1679         .id_table  = pci_ifpga_map,
1680         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1681         .probe     = ifpga_rawdev_pci_probe,
1682         .remove    = ifpga_rawdev_pci_remove,
1683 };
1684
1685 RTE_PMD_REGISTER_PCI(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
1686 RTE_PMD_REGISTER_PCI_TABLE(ifpga_rawdev_pci_driver, rte_ifpga_rawdev_pmd);
1687 RTE_PMD_REGISTER_KMOD_DEP(ifpga_rawdev_pci_driver, "* igb_uio | uio_pci_generic | vfio-pci");
1688 RTE_LOG_REGISTER_DEFAULT(ifpga_rawdev_logtype, NOTICE);
1689
1690 static const char * const valid_args[] = {
1691 #define IFPGA_ARG_NAME         "ifpga"
1692         IFPGA_ARG_NAME,
1693 #define IFPGA_ARG_PORT         "port"
1694         IFPGA_ARG_PORT,
1695 #define IFPGA_AFU_BTS          "afu_bts"
1696         IFPGA_AFU_BTS,
1697         NULL
1698 };
1699
1700 static int ifpga_rawdev_get_string_arg(const char *key __rte_unused,
1701         const char *value, void *extra_args)
1702 {
1703         int size;
1704         if (!value || !extra_args)
1705                 return -EINVAL;
1706
1707         size = strlen(value) + 1;
1708         *(char **)extra_args = rte_malloc(NULL, size, RTE_CACHE_LINE_SIZE);
1709         if (!*(char **)extra_args)
1710                 return -ENOMEM;
1711
1712         strlcpy(*(char **)extra_args, value, size);
1713
1714         return 0;
1715 }
1716 static int
1717 ifpga_cfg_probe(struct rte_vdev_device *dev)
1718 {
1719         struct rte_devargs *devargs;
1720         struct rte_kvargs *kvlist = NULL;
1721         struct rte_rawdev *rawdev = NULL;
1722         struct ifpga_rawdev *ifpga_dev;
1723         int port;
1724         char *name = NULL;
1725         char dev_name[RTE_RAWDEV_NAME_MAX_LEN];
1726         int ret = -1;
1727
1728         devargs = dev->device.devargs;
1729
1730         kvlist = rte_kvargs_parse(devargs->args, valid_args);
1731         if (!kvlist) {
1732                 IFPGA_RAWDEV_PMD_LOG(ERR, "error when parsing param");
1733                 goto end;
1734         }
1735
1736         if (rte_kvargs_count(kvlist, IFPGA_ARG_NAME) == 1) {
1737                 if (rte_kvargs_process(kvlist, IFPGA_ARG_NAME,
1738                                        &ifpga_rawdev_get_string_arg,
1739                                        &name) < 0) {
1740                         IFPGA_RAWDEV_PMD_ERR("error to parse %s",
1741                                      IFPGA_ARG_NAME);
1742                         goto end;
1743                 }
1744         } else {
1745                 IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga bus",
1746                           IFPGA_ARG_NAME);
1747                 goto end;
1748         }
1749
1750         if (rte_kvargs_count(kvlist, IFPGA_ARG_PORT) == 1) {
1751                 if (rte_kvargs_process(kvlist,
1752                         IFPGA_ARG_PORT,
1753                         &rte_ifpga_get_integer32_arg,
1754                         &port) < 0) {
1755                         IFPGA_RAWDEV_PMD_ERR("error to parse %s",
1756                                 IFPGA_ARG_PORT);
1757                         goto end;
1758                 }
1759         } else {
1760                 IFPGA_RAWDEV_PMD_ERR("arg %s is mandatory for ifpga bus",
1761                           IFPGA_ARG_PORT);
1762                 goto end;
1763         }
1764
1765         memset(dev_name, 0, sizeof(dev_name));
1766         snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "IFPGA:%s", name);
1767         rawdev = rte_rawdev_pmd_get_named_dev(dev_name);
1768         if (!rawdev)
1769                 goto end;
1770         ifpga_dev = ifpga_rawdev_get(rawdev);
1771         if (!ifpga_dev)
1772                 goto end;
1773
1774         memset(dev_name, 0, sizeof(dev_name));
1775         snprintf(dev_name, RTE_RAWDEV_NAME_MAX_LEN, "%d|%s",
1776         port, name);
1777
1778         ret = rte_eal_hotplug_add(RTE_STR(IFPGA_BUS_NAME),
1779                         dev_name, devargs->args);
1780 end:
1781         rte_kvargs_free(kvlist);
1782         free(name);
1783
1784         return ret;
1785 }
1786
1787 static int
1788 ifpga_cfg_remove(struct rte_vdev_device *vdev)
1789 {
1790         IFPGA_RAWDEV_PMD_INFO("Remove ifpga_cfg %p",
1791                 vdev);
1792
1793         return 0;
1794 }
1795
1796 static struct rte_vdev_driver ifpga_cfg_driver = {
1797         .probe = ifpga_cfg_probe,
1798         .remove = ifpga_cfg_remove,
1799 };
1800
1801 RTE_PMD_REGISTER_VDEV(ifpga_rawdev_cfg, ifpga_cfg_driver);
1802 RTE_PMD_REGISTER_ALIAS(ifpga_rawdev_cfg, ifpga_cfg);
1803 RTE_PMD_REGISTER_PARAM_STRING(ifpga_rawdev_cfg,
1804         "ifpga=<string> "
1805         "port=<int> "
1806         "afu_bts=<path>");
1807
1808 struct rte_pci_bus *ifpga_get_pci_bus(void)
1809 {
1810         return rte_ifpga_rawdev_pmd.bus;
1811 }
1812
1813 int ifpga_rawdev_partial_reconfigure(struct rte_rawdev *dev, int port,
1814         const char *file)
1815 {
1816         if (!dev) {
1817                 IFPGA_RAWDEV_PMD_ERR("Input parameter is invalid");
1818                 return -EINVAL;
1819         }
1820
1821         return rte_fpga_do_pr(dev, port, file);
1822 }
1823
1824 void ifpga_rawdev_cleanup(void)
1825 {
1826         struct ifpga_rawdev *dev;
1827         unsigned int i;
1828
1829         for (i = 0; i < IFPGA_RAWDEV_NUM; i++) {
1830                 dev = &ifpga_rawdevices[i];
1831                 if (dev->rawdev) {
1832                         rte_rawdev_pmd_release(dev->rawdev);
1833                         dev->rawdev = NULL;
1834                 }
1835         }
1836 }