drivers/net/avp/avp_ethdev.c

   1 /*
   2  *   BSD LICENSE
   3  *
   4  * Copyright (c) 2013-2017, Wind River Systems, Inc.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions are met:
   8  *
   9  * 1) Redistributions of source code must retain the above copyright notice,
  10  * this list of conditions and the following disclaimer.
  11  *
  12  * 2) Redistributions in binary form must reproduce the above copyright notice,
  13  * this list of conditions and the following disclaimer in the documentation
  14  * and/or other materials provided with the distribution.
  15  *
  16  * 3) Neither the name of Wind River Systems nor the names of its contributors
  17  * may be used to endorse or promote products derived from this software
  18  * without specific prior written permission.
  19  *
  20  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
  24  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30  * POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <stdint.h>
  34 #include <string.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <unistd.h>
  38
  39 #include <rte_ethdev.h>
  40 #include <rte_memcpy.h>
  41 #include <rte_string_fns.h>
  42 #include <rte_memzone.h>
  43 #include <rte_malloc.h>
  44 #include <rte_atomic.h>
  45 #include <rte_branch_prediction.h>
  46 #include <rte_pci.h>
  47 #include <rte_ether.h>
  48 #include <rte_common.h>
  49 #include <rte_cycles.h>
  50 #include <rte_byteorder.h>
  51 #include <rte_dev.h>
  52 #include <rte_memory.h>
  53 #include <rte_eal.h>
  54 #include <rte_io.h>
  55
  56 #include "rte_avp_common.h"
  57 #include "rte_avp_fifo.h"
  58
  59 #include "avp_logs.h"
  60
  61
  62
  63 #define AVP_DEV_TO_PCI(eth_dev) RTE_DEV_TO_PCI((eth_dev)->device)
  64
  65
  66 #define AVP_MAX_MAC_ADDRS 1
  67 #define AVP_MIN_RX_BUFSIZE ETHER_MIN_LEN
  68
  69
  70 /*
  71  * Defines the number of microseconds to wait before checking the response
  72  * queue for completion.
  73  */
  74 #define AVP_REQUEST_DELAY_USECS (5000)
  75
  76 /*
  77  * Defines the number times to check the response queue for completion before
  78  * declaring a timeout.
  79  */
  80 #define AVP_MAX_REQUEST_RETRY (100)
  81
  82 /* Defines the current PCI driver version number */
  83 #define AVP_DPDK_DRIVER_VERSION RTE_AVP_CURRENT_GUEST_VERSION
  84
  85 /*
  86  * The set of PCI devices this driver supports
  87  */
  88 static const struct rte_pci_id pci_id_avp_map[] = {
  89         { .vendor_id = RTE_AVP_PCI_VENDOR_ID,
  90           .device_id = RTE_AVP_PCI_DEVICE_ID,
  91           .subsystem_vendor_id = RTE_AVP_PCI_SUB_VENDOR_ID,
  92           .subsystem_device_id = RTE_AVP_PCI_SUB_DEVICE_ID,
  93           .class_id = RTE_CLASS_ANY_ID,
  94         },
  95
  96         { .vendor_id = 0, /* sentinel */
  97         },
  98 };
  99
 100
 101 /**@{ AVP device flags */
 102 #define AVP_F_PROMISC (1 << 1)
 103 #define AVP_F_CONFIGURED (1 << 2)
 104 #define AVP_F_LINKUP (1 << 3)
 105 /**@} */
 106
 107 /* Ethernet device validation marker */
 108 #define AVP_ETHDEV_MAGIC 0x92972862
 109
 110 /*
 111  * Defines the AVP device attributes which are attached to an RTE ethernet
 112  * device
 113  */
 114 struct avp_dev {
 115         uint32_t magic; /**< Memory validation marker */
 116         uint64_t device_id; /**< Unique system identifier */
 117         struct ether_addr ethaddr; /**< Host specified MAC address */
 118         struct rte_eth_dev_data *dev_data;
 119         /**< Back pointer to ethernet device data */
 120         volatile uint32_t flags; /**< Device operational flags */
 121         uint8_t port_id; /**< Ethernet port identifier */
 122         struct rte_mempool *pool; /**< pkt mbuf mempool */
 123         unsigned int guest_mbuf_size; /**< local pool mbuf size */
 124         unsigned int host_mbuf_size; /**< host mbuf size */
 125         unsigned int max_rx_pkt_len; /**< maximum receive unit */
 126         uint32_t host_features; /**< Supported feature bitmap */
 127         uint32_t features; /**< Enabled feature bitmap */
 128         unsigned int num_tx_queues; /**< Negotiated number of transmit queues */
 129         unsigned int max_tx_queues; /**< Maximum number of transmit queues */
 130         unsigned int num_rx_queues; /**< Negotiated number of receive queues */
 131         unsigned int max_rx_queues; /**< Maximum number of receive queues */
 132
 133         struct rte_avp_fifo *tx_q[RTE_AVP_MAX_QUEUES]; /**< TX queue */
 134         struct rte_avp_fifo *rx_q[RTE_AVP_MAX_QUEUES]; /**< RX queue */
 135         struct rte_avp_fifo *alloc_q[RTE_AVP_MAX_QUEUES];
 136         /**< Allocated mbufs queue */
 137         struct rte_avp_fifo *free_q[RTE_AVP_MAX_QUEUES];
 138         /**< To be freed mbufs queue */
 139
 140         /* For request & response */
 141         struct rte_avp_fifo *req_q; /**< Request queue */
 142         struct rte_avp_fifo *resp_q; /**< Response queue */
 143         void *host_sync_addr; /**< (host) Req/Resp Mem address */
 144         void *sync_addr; /**< Req/Resp Mem address */
 145         void *host_mbuf_addr; /**< (host) MBUF pool start address */
 146         void *mbuf_addr; /**< MBUF pool start address */
 147 } __rte_cache_aligned;
 148
 149 /* RTE ethernet private data */
 150 struct avp_adapter {
 151         struct avp_dev avp;
 152 } __rte_cache_aligned;
 153
 154
 155 /* 32-bit MMIO register write */
 156 #define AVP_WRITE32(_value, _addr) rte_write32_relaxed((_value), (_addr))
 157
 158 /* 32-bit MMIO register read */
 159 #define AVP_READ32(_addr) rte_read32_relaxed((_addr))
 160
 161 /* Macro to cast the ethernet device private data to a AVP object */
 162 #define AVP_DEV_PRIVATE_TO_HW(adapter) \
 163         (&((struct avp_adapter *)adapter)->avp)
 164
 165 /*
 166  * Defines the structure of a AVP device queue for the purpose of handling the
 167  * receive and transmit burst callback functions
 168  */
 169 struct avp_queue {
 170         struct rte_eth_dev_data *dev_data;
 171         /**< Backpointer to ethernet device data */
 172         struct avp_dev *avp; /**< Backpointer to AVP device */
 173         uint16_t queue_id;
 174         /**< Queue identifier used for indexing current queue */
 175         uint16_t queue_base;
 176         /**< Base queue identifier for queue servicing */
 177         uint16_t queue_limit;
 178         /**< Maximum queue identifier for queue servicing */
 179
 180         uint64_t packets;
 181         uint64_t bytes;
 182         uint64_t errors;
 183 };
 184
 185 /* translate from host physical address to guest virtual address */
 186 static void *
 187 avp_dev_translate_address(struct rte_eth_dev *eth_dev,
 188                           phys_addr_t host_phys_addr)
 189 {
 190         struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev);
 191         struct rte_mem_resource *resource;
 192         struct rte_avp_memmap_info *info;
 193         struct rte_avp_memmap *map;
 194         off_t offset;
 195         void *addr;
 196         unsigned int i;
 197
 198         addr = pci_dev->mem_resource[RTE_AVP_PCI_MEMORY_BAR].addr;
 199         resource = &pci_dev->mem_resource[RTE_AVP_PCI_MEMMAP_BAR];
 200         info = (struct rte_avp_memmap_info *)resource->addr;
 201
 202         offset = 0;
 203         for (i = 0; i < info->nb_maps; i++) {
 204                 /* search all segments looking for a matching address */
 205                 map = &info->maps[i];
 206
 207                 if ((host_phys_addr >= map->phys_addr) &&
 208                         (host_phys_addr < (map->phys_addr + map->length))) {
 209                         /* address is within this segment */
 210                         offset += (host_phys_addr - map->phys_addr);
 211                         addr = RTE_PTR_ADD(addr, offset);
 212
 213                         PMD_DRV_LOG(DEBUG, "Translating host physical 0x%" PRIx64 " to guest virtual 0x%p\n",
 214                                     host_phys_addr, addr);
 215
 216                         return addr;
 217                 }
 218                 offset += map->length;
 219         }
 220
 221         return NULL;
 222 }
 223
 224 /* verify that the incoming device version is compatible with our version */
 225 static int
 226 avp_dev_version_check(uint32_t version)
 227 {
 228         uint32_t driver = RTE_AVP_STRIP_MINOR_VERSION(AVP_DPDK_DRIVER_VERSION);
 229         uint32_t device = RTE_AVP_STRIP_MINOR_VERSION(version);
 230
 231         if (device <= driver) {
 232                 /* the host driver version is less than or equal to ours */
 233                 return 0;
 234         }
 235
 236         return 1;
 237 }
 238
 239 /* verify that memory regions have expected version and validation markers */
 240 static int
 241 avp_dev_check_regions(struct rte_eth_dev *eth_dev)
 242 {
 243         struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev);
 244         struct rte_avp_memmap_info *memmap;
 245         struct rte_avp_device_info *info;
 246         struct rte_mem_resource *resource;
 247         unsigned int i;
 248
 249         /* Dump resource info for debug */
 250         for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 251                 resource = &pci_dev->mem_resource[i];
 252                 if ((resource->phys_addr == 0) || (resource->len == 0))
 253                         continue;
 254
 255                 PMD_DRV_LOG(DEBUG, "resource[%u]: phys=0x%" PRIx64 " len=%" PRIu64 " addr=%p\n",
 256                             i, resource->phys_addr,
 257                             resource->len, resource->addr);
 258
 259                 switch (i) {
 260                 case RTE_AVP_PCI_MEMMAP_BAR:
 261                         memmap = (struct rte_avp_memmap_info *)resource->addr;
 262                         if ((memmap->magic != RTE_AVP_MEMMAP_MAGIC) ||
 263                             (memmap->version != RTE_AVP_MEMMAP_VERSION)) {
 264                                 PMD_DRV_LOG(ERR, "Invalid memmap magic 0x%08x and version %u\n",
 265                                             memmap->magic, memmap->version);
 266                                 return -EINVAL;
 267                         }
 268                         break;
 269
 270                 case RTE_AVP_PCI_DEVICE_BAR:
 271                         info = (struct rte_avp_device_info *)resource->addr;
 272                         if ((info->magic != RTE_AVP_DEVICE_MAGIC) ||
 273                             avp_dev_version_check(info->version)) {
 274                                 PMD_DRV_LOG(ERR, "Invalid device info magic 0x%08x or version 0x%08x > 0x%08x\n",
 275                                             info->magic, info->version,
 276                                             AVP_DPDK_DRIVER_VERSION);
 277                                 return -EINVAL;
 278                         }
 279                         break;
 280
 281                 case RTE_AVP_PCI_MEMORY_BAR:
 282                 case RTE_AVP_PCI_MMIO_BAR:
 283                         if (resource->addr == NULL) {
 284                                 PMD_DRV_LOG(ERR, "Missing address space for BAR%u\n",
 285                                             i);
 286                                 return -EINVAL;
 287                         }
 288                         break;
 289
 290                 case RTE_AVP_PCI_MSIX_BAR:
 291                 default:
 292                         /* no validation required */
 293                         break;
 294                 }
 295         }
 296
 297         return 0;
 298 }
 299
 300 /*
 301  * create a AVP device using the supplied device info by first translating it
 302  * to guest address space(s).
 303  */
 304 static int
 305 avp_dev_create(struct rte_pci_device *pci_dev,
 306                struct rte_eth_dev *eth_dev)
 307 {
 308         struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 309         struct rte_avp_device_info *host_info;
 310         struct rte_mem_resource *resource;
 311         unsigned int i;
 312
 313         resource = &pci_dev->mem_resource[RTE_AVP_PCI_DEVICE_BAR];
 314         if (resource->addr == NULL) {
 315                 PMD_DRV_LOG(ERR, "BAR%u is not mapped\n",
 316                             RTE_AVP_PCI_DEVICE_BAR);
 317                 return -EFAULT;
 318         }
 319         host_info = (struct rte_avp_device_info *)resource->addr;
 320
 321         if ((host_info->magic != RTE_AVP_DEVICE_MAGIC) ||
 322                 avp_dev_version_check(host_info->version)) {
 323                 PMD_DRV_LOG(ERR, "Invalid AVP PCI device, magic 0x%08x version 0x%08x > 0x%08x\n",
 324                             host_info->magic, host_info->version,
 325                             AVP_DPDK_DRIVER_VERSION);
 326                 return -EINVAL;
 327         }
 328
 329         PMD_DRV_LOG(DEBUG, "AVP host device is v%u.%u.%u\n",
 330                     RTE_AVP_GET_RELEASE_VERSION(host_info->version),
 331                     RTE_AVP_GET_MAJOR_VERSION(host_info->version),
 332                     RTE_AVP_GET_MINOR_VERSION(host_info->version));
 333
 334         PMD_DRV_LOG(DEBUG, "AVP host supports %u to %u TX queue(s)\n",
 335                     host_info->min_tx_queues, host_info->max_tx_queues);
 336         PMD_DRV_LOG(DEBUG, "AVP host supports %u to %u RX queue(s)\n",
 337                     host_info->min_rx_queues, host_info->max_rx_queues);
 338         PMD_DRV_LOG(DEBUG, "AVP host supports features 0x%08x\n",
 339                     host_info->features);
 340
 341         if (avp->magic != AVP_ETHDEV_MAGIC) {
 342                 /*
 343                  * First time initialization (i.e., not during a VM
 344                  * migration)
 345                  */
 346                 memset(avp, 0, sizeof(*avp));
 347                 avp->magic = AVP_ETHDEV_MAGIC;
 348                 avp->dev_data = eth_dev->data;
 349                 avp->port_id = eth_dev->data->port_id;
 350                 avp->host_mbuf_size = host_info->mbuf_size;
 351                 avp->host_features = host_info->features;
 352                 memcpy(&avp->ethaddr.addr_bytes[0],
 353                        host_info->ethaddr, ETHER_ADDR_LEN);
 354                 /* adjust max values to not exceed our max */
 355                 avp->max_tx_queues =
 356                         RTE_MIN(host_info->max_tx_queues, RTE_AVP_MAX_QUEUES);
 357                 avp->max_rx_queues =
 358                         RTE_MIN(host_info->max_rx_queues, RTE_AVP_MAX_QUEUES);
 359         } else {
 360                 /* Re-attaching during migration */
 361
 362                 /* TODO... requires validation of host values */
 363                 if ((host_info->features & avp->features) != avp->features) {
 364                         PMD_DRV_LOG(ERR, "AVP host features mismatched; 0x%08x, host=0x%08x\n",
 365                                     avp->features, host_info->features);
 366                         /* this should not be possible; continue for now */
 367                 }
 368         }
 369
 370         /* the device id is allowed to change over migrations */
 371         avp->device_id = host_info->device_id;
 372
 373         /* translate incoming host addresses to guest address space */
 374         PMD_DRV_LOG(DEBUG, "AVP first host tx queue at 0x%" PRIx64 "\n",
 375                     host_info->tx_phys);
 376         PMD_DRV_LOG(DEBUG, "AVP first host alloc queue at 0x%" PRIx64 "\n",
 377                     host_info->alloc_phys);
 378         for (i = 0; i < avp->max_tx_queues; i++) {
 379                 avp->tx_q[i] = avp_dev_translate_address(eth_dev,
 380                         host_info->tx_phys + (i * host_info->tx_size));
 381
 382                 avp->alloc_q[i] = avp_dev_translate_address(eth_dev,
 383                         host_info->alloc_phys + (i * host_info->alloc_size));
 384         }
 385
 386         PMD_DRV_LOG(DEBUG, "AVP first host rx queue at 0x%" PRIx64 "\n",
 387                     host_info->rx_phys);
 388         PMD_DRV_LOG(DEBUG, "AVP first host free queue at 0x%" PRIx64 "\n",
 389                     host_info->free_phys);
 390         for (i = 0; i < avp->max_rx_queues; i++) {
 391                 avp->rx_q[i] = avp_dev_translate_address(eth_dev,
 392                         host_info->rx_phys + (i * host_info->rx_size));
 393                 avp->free_q[i] = avp_dev_translate_address(eth_dev,
 394                         host_info->free_phys + (i * host_info->free_size));
 395         }
 396
 397         PMD_DRV_LOG(DEBUG, "AVP host request queue at 0x%" PRIx64 "\n",
 398                     host_info->req_phys);
 399         PMD_DRV_LOG(DEBUG, "AVP host response queue at 0x%" PRIx64 "\n",
 400                     host_info->resp_phys);
 401         PMD_DRV_LOG(DEBUG, "AVP host sync address at 0x%" PRIx64 "\n",
 402                     host_info->sync_phys);
 403         PMD_DRV_LOG(DEBUG, "AVP host mbuf address at 0x%" PRIx64 "\n",
 404                     host_info->mbuf_phys);
 405         avp->req_q = avp_dev_translate_address(eth_dev, host_info->req_phys);
 406         avp->resp_q = avp_dev_translate_address(eth_dev, host_info->resp_phys);
 407         avp->sync_addr =
 408                 avp_dev_translate_address(eth_dev, host_info->sync_phys);
 409         avp->mbuf_addr =
 410                 avp_dev_translate_address(eth_dev, host_info->mbuf_phys);
 411
 412         /*
 413          * store the host mbuf virtual address so that we can calculate
 414          * relative offsets for each mbuf as they are processed
 415          */
 416         avp->host_mbuf_addr = host_info->mbuf_va;
 417         avp->host_sync_addr = host_info->sync_va;
 418
 419         /*
 420          * store the maximum packet length that is supported by the host.
 421          */
 422         avp->max_rx_pkt_len = host_info->max_rx_pkt_len;
 423         PMD_DRV_LOG(DEBUG, "AVP host max receive packet length is %u\n",
 424                                 host_info->max_rx_pkt_len);
 425
 426         return 0;
 427 }
 428
 429 /*
 430  * This function is based on probe() function in avp_pci.c
 431  * It returns 0 on success.
 432  */
 433 static int
 434 eth_avp_dev_init(struct rte_eth_dev *eth_dev)
 435 {
 436         struct avp_dev *avp =
 437                 AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
 438         struct rte_pci_device *pci_dev;
 439         int ret;
 440
 441         pci_dev = AVP_DEV_TO_PCI(eth_dev);
 442
 443         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 444                 /*
 445                  * no setup required on secondary processes.  All data is saved
 446                  * in dev_private by the primary process. All resource should
 447                  * be mapped to the same virtual address so all pointers should
 448                  * be valid.
 449                  */
 450                 return 0;
 451         }
 452
 453         rte_eth_copy_pci_info(eth_dev, pci_dev);
 454
 455         eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
 456
 457         /* Check BAR resources */
 458         ret = avp_dev_check_regions(eth_dev);
 459         if (ret < 0) {
 460                 PMD_DRV_LOG(ERR, "Failed to validate BAR resources, ret=%d\n",
 461                             ret);
 462                 return ret;
 463         }
 464
 465         /* Handle each subtype */
 466         ret = avp_dev_create(pci_dev, eth_dev);
 467         if (ret < 0) {
 468                 PMD_DRV_LOG(ERR, "Failed to create device, ret=%d\n", ret);
 469                 return ret;
 470         }
 471
 472         /* Allocate memory for storing MAC addresses */
 473         eth_dev->data->mac_addrs = rte_zmalloc("avp_ethdev", ETHER_ADDR_LEN, 0);
 474         if (eth_dev->data->mac_addrs == NULL) {
 475                 PMD_DRV_LOG(ERR, "Failed to allocate %d bytes needed to store MAC addresses\n",
 476                             ETHER_ADDR_LEN);
 477                 return -ENOMEM;
 478         }
 479
 480         /* Get a mac from device config */
 481         ether_addr_copy(&avp->ethaddr, &eth_dev->data->mac_addrs[0]);
 482
 483         return 0;
 484 }
 485
 486 static int
 487 eth_avp_dev_uninit(struct rte_eth_dev *eth_dev)
 488 {
 489         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 490                 return -EPERM;
 491
 492         if (eth_dev->data == NULL)
 493                 return 0;
 494
 495         if (eth_dev->data->mac_addrs != NULL) {
 496                 rte_free(eth_dev->data->mac_addrs);
 497                 eth_dev->data->mac_addrs = NULL;
 498         }
 499
 500         return 0;
 501 }
 502
 503
 504 static struct eth_driver rte_avp_pmd = {
 505         {
 506                 .id_table = pci_id_avp_map,
 507                 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 508                 .probe = rte_eth_dev_pci_probe,
 509                 .remove = rte_eth_dev_pci_remove,
 510         },
 511         .eth_dev_init = eth_avp_dev_init,
 512         .eth_dev_uninit = eth_avp_dev_uninit,
 513         .dev_private_size = sizeof(struct avp_adapter),
 514 };
 515
 516 RTE_PMD_REGISTER_PCI(net_avp, rte_avp_pmd.pci_drv);
 517 RTE_PMD_REGISTER_PCI_TABLE(net_avp, pci_id_avp_map);