4 * Copyright (c) 2013-2017, Wind River Systems, Inc.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
9 * 1) Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
12 * 2) Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
16 * 3) Neither the name of Wind River Systems nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
24 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
39 #include <rte_ethdev.h>
40 #include <rte_memcpy.h>
41 #include <rte_string_fns.h>
42 #include <rte_memzone.h>
43 #include <rte_malloc.h>
44 #include <rte_atomic.h>
45 #include <rte_branch_prediction.h>
47 #include <rte_ether.h>
48 #include <rte_common.h>
49 #include <rte_cycles.h>
50 #include <rte_byteorder.h>
52 #include <rte_memory.h>
56 #include "rte_avp_common.h"
57 #include "rte_avp_fifo.h"
63 static int avp_dev_configure(struct rte_eth_dev *dev);
64 static int avp_dev_start(struct rte_eth_dev *dev);
65 static void avp_dev_stop(struct rte_eth_dev *dev);
66 static void avp_dev_close(struct rte_eth_dev *dev);
67 static void avp_dev_info_get(struct rte_eth_dev *dev,
68 struct rte_eth_dev_info *dev_info);
69 static void avp_vlan_offload_set(struct rte_eth_dev *dev, int mask);
70 static int avp_dev_link_update(struct rte_eth_dev *dev,
71 __rte_unused int wait_to_complete);
72 static void avp_dev_promiscuous_enable(struct rte_eth_dev *dev);
73 static void avp_dev_promiscuous_disable(struct rte_eth_dev *dev);
75 static int avp_dev_rx_queue_setup(struct rte_eth_dev *dev,
78 unsigned int socket_id,
79 const struct rte_eth_rxconf *rx_conf,
80 struct rte_mempool *pool);
82 static int avp_dev_tx_queue_setup(struct rte_eth_dev *dev,
85 unsigned int socket_id,
86 const struct rte_eth_txconf *tx_conf);
88 static uint16_t avp_recv_scattered_pkts(void *rx_queue,
89 struct rte_mbuf **rx_pkts,
92 static uint16_t avp_recv_pkts(void *rx_queue,
93 struct rte_mbuf **rx_pkts,
96 static uint16_t avp_xmit_scattered_pkts(void *tx_queue,
97 struct rte_mbuf **tx_pkts,
100 static uint16_t avp_xmit_pkts(void *tx_queue,
101 struct rte_mbuf **tx_pkts,
104 static void avp_dev_rx_queue_release(void *rxq);
105 static void avp_dev_tx_queue_release(void *txq);
107 static void avp_dev_stats_get(struct rte_eth_dev *dev,
108 struct rte_eth_stats *stats);
109 static void avp_dev_stats_reset(struct rte_eth_dev *dev);
112 #define AVP_DEV_TO_PCI(eth_dev) RTE_DEV_TO_PCI((eth_dev)->device)
115 #define AVP_MAX_RX_BURST 64
116 #define AVP_MAX_TX_BURST 64
117 #define AVP_MAX_MAC_ADDRS 1
118 #define AVP_MIN_RX_BUFSIZE ETHER_MIN_LEN
122 * Defines the number of microseconds to wait before checking the response
123 * queue for completion.
125 #define AVP_REQUEST_DELAY_USECS (5000)
128 * Defines the number times to check the response queue for completion before
129 * declaring a timeout.
131 #define AVP_MAX_REQUEST_RETRY (100)
133 /* Defines the current PCI driver version number */
134 #define AVP_DPDK_DRIVER_VERSION RTE_AVP_CURRENT_GUEST_VERSION
137 * The set of PCI devices this driver supports
139 static const struct rte_pci_id pci_id_avp_map[] = {
140 { .vendor_id = RTE_AVP_PCI_VENDOR_ID,
141 .device_id = RTE_AVP_PCI_DEVICE_ID,
142 .subsystem_vendor_id = RTE_AVP_PCI_SUB_VENDOR_ID,
143 .subsystem_device_id = RTE_AVP_PCI_SUB_DEVICE_ID,
144 .class_id = RTE_CLASS_ANY_ID,
147 { .vendor_id = 0, /* sentinel */
152 * dev_ops for avp, bare necessities for basic operation
154 static const struct eth_dev_ops avp_eth_dev_ops = {
155 .dev_configure = avp_dev_configure,
156 .dev_start = avp_dev_start,
157 .dev_stop = avp_dev_stop,
158 .dev_close = avp_dev_close,
159 .dev_infos_get = avp_dev_info_get,
160 .vlan_offload_set = avp_vlan_offload_set,
161 .stats_get = avp_dev_stats_get,
162 .stats_reset = avp_dev_stats_reset,
163 .link_update = avp_dev_link_update,
164 .promiscuous_enable = avp_dev_promiscuous_enable,
165 .promiscuous_disable = avp_dev_promiscuous_disable,
166 .rx_queue_setup = avp_dev_rx_queue_setup,
167 .rx_queue_release = avp_dev_rx_queue_release,
168 .tx_queue_setup = avp_dev_tx_queue_setup,
169 .tx_queue_release = avp_dev_tx_queue_release,
172 /**@{ AVP device flags */
173 #define AVP_F_PROMISC (1 << 1)
174 #define AVP_F_CONFIGURED (1 << 2)
175 #define AVP_F_LINKUP (1 << 3)
178 /* Ethernet device validation marker */
179 #define AVP_ETHDEV_MAGIC 0x92972862
182 * Defines the AVP device attributes which are attached to an RTE ethernet
186 uint32_t magic; /**< Memory validation marker */
187 uint64_t device_id; /**< Unique system identifier */
188 struct ether_addr ethaddr; /**< Host specified MAC address */
189 struct rte_eth_dev_data *dev_data;
190 /**< Back pointer to ethernet device data */
191 volatile uint32_t flags; /**< Device operational flags */
192 uint8_t port_id; /**< Ethernet port identifier */
193 struct rte_mempool *pool; /**< pkt mbuf mempool */
194 unsigned int guest_mbuf_size; /**< local pool mbuf size */
195 unsigned int host_mbuf_size; /**< host mbuf size */
196 unsigned int max_rx_pkt_len; /**< maximum receive unit */
197 uint32_t host_features; /**< Supported feature bitmap */
198 uint32_t features; /**< Enabled feature bitmap */
199 unsigned int num_tx_queues; /**< Negotiated number of transmit queues */
200 unsigned int max_tx_queues; /**< Maximum number of transmit queues */
201 unsigned int num_rx_queues; /**< Negotiated number of receive queues */
202 unsigned int max_rx_queues; /**< Maximum number of receive queues */
204 struct rte_avp_fifo *tx_q[RTE_AVP_MAX_QUEUES]; /**< TX queue */
205 struct rte_avp_fifo *rx_q[RTE_AVP_MAX_QUEUES]; /**< RX queue */
206 struct rte_avp_fifo *alloc_q[RTE_AVP_MAX_QUEUES];
207 /**< Allocated mbufs queue */
208 struct rte_avp_fifo *free_q[RTE_AVP_MAX_QUEUES];
209 /**< To be freed mbufs queue */
211 /* For request & response */
212 struct rte_avp_fifo *req_q; /**< Request queue */
213 struct rte_avp_fifo *resp_q; /**< Response queue */
214 void *host_sync_addr; /**< (host) Req/Resp Mem address */
215 void *sync_addr; /**< Req/Resp Mem address */
216 void *host_mbuf_addr; /**< (host) MBUF pool start address */
217 void *mbuf_addr; /**< MBUF pool start address */
218 } __rte_cache_aligned;
220 /* RTE ethernet private data */
223 } __rte_cache_aligned;
226 /* 32-bit MMIO register write */
227 #define AVP_WRITE32(_value, _addr) rte_write32_relaxed((_value), (_addr))
229 /* 32-bit MMIO register read */
230 #define AVP_READ32(_addr) rte_read32_relaxed((_addr))
232 /* Macro to cast the ethernet device private data to a AVP object */
233 #define AVP_DEV_PRIVATE_TO_HW(adapter) \
234 (&((struct avp_adapter *)adapter)->avp)
237 * Defines the structure of a AVP device queue for the purpose of handling the
238 * receive and transmit burst callback functions
241 struct rte_eth_dev_data *dev_data;
242 /**< Backpointer to ethernet device data */
243 struct avp_dev *avp; /**< Backpointer to AVP device */
245 /**< Queue identifier used for indexing current queue */
247 /**< Base queue identifier for queue servicing */
248 uint16_t queue_limit;
249 /**< Maximum queue identifier for queue servicing */
256 /* send a request and wait for a response
258 * @warning must be called while holding the avp->lock spinlock.
261 avp_dev_process_request(struct avp_dev *avp, struct rte_avp_request *request)
263 unsigned int retry = AVP_MAX_REQUEST_RETRY;
264 void *resp_addr = NULL;
268 PMD_DRV_LOG(DEBUG, "Sending request %u to host\n", request->req_id);
270 request->result = -ENOTSUP;
272 /* Discard any stale responses before starting a new request */
273 while (avp_fifo_get(avp->resp_q, (void **)&resp_addr, 1))
274 PMD_DRV_LOG(DEBUG, "Discarding stale response\n");
276 rte_memcpy(avp->sync_addr, request, sizeof(*request));
277 count = avp_fifo_put(avp->req_q, &avp->host_sync_addr, 1);
279 PMD_DRV_LOG(ERR, "Cannot send request %u to host\n",
286 /* wait for a response */
287 usleep(AVP_REQUEST_DELAY_USECS);
289 count = avp_fifo_count(avp->resp_q);
291 /* response received */
295 if ((count < 1) && (retry == 0)) {
296 PMD_DRV_LOG(ERR, "Timeout while waiting for a response for %u\n",
303 /* retrieve the response */
304 count = avp_fifo_get(avp->resp_q, (void **)&resp_addr, 1);
305 if ((count != 1) || (resp_addr != avp->host_sync_addr)) {
306 PMD_DRV_LOG(ERR, "Invalid response from host, count=%u resp=%p host_sync_addr=%p\n",
307 count, resp_addr, avp->host_sync_addr);
312 /* copy to user buffer */
313 rte_memcpy(request, avp->sync_addr, sizeof(*request));
316 PMD_DRV_LOG(DEBUG, "Result %d received for request %u\n",
317 request->result, request->req_id);
324 avp_dev_ctrl_set_link_state(struct rte_eth_dev *eth_dev, unsigned int state)
326 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
327 struct rte_avp_request request;
330 /* setup a link state change request */
331 memset(&request, 0, sizeof(request));
332 request.req_id = RTE_AVP_REQ_CFG_NETWORK_IF;
333 request.if_up = state;
335 ret = avp_dev_process_request(avp, &request);
337 return ret == 0 ? request.result : ret;
341 avp_dev_ctrl_set_config(struct rte_eth_dev *eth_dev,
342 struct rte_avp_device_config *config)
344 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
345 struct rte_avp_request request;
348 /* setup a configure request */
349 memset(&request, 0, sizeof(request));
350 request.req_id = RTE_AVP_REQ_CFG_DEVICE;
351 memcpy(&request.config, config, sizeof(request.config));
353 ret = avp_dev_process_request(avp, &request);
355 return ret == 0 ? request.result : ret;
359 avp_dev_ctrl_shutdown(struct rte_eth_dev *eth_dev)
361 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
362 struct rte_avp_request request;
365 /* setup a shutdown request */
366 memset(&request, 0, sizeof(request));
367 request.req_id = RTE_AVP_REQ_SHUTDOWN_DEVICE;
369 ret = avp_dev_process_request(avp, &request);
371 return ret == 0 ? request.result : ret;
374 /* translate from host mbuf virtual address to guest virtual address */
376 avp_dev_translate_buffer(struct avp_dev *avp, void *host_mbuf_address)
378 return RTE_PTR_ADD(RTE_PTR_SUB(host_mbuf_address,
379 (uintptr_t)avp->host_mbuf_addr),
380 (uintptr_t)avp->mbuf_addr);
383 /* translate from host physical address to guest virtual address */
385 avp_dev_translate_address(struct rte_eth_dev *eth_dev,
386 phys_addr_t host_phys_addr)
388 struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev);
389 struct rte_mem_resource *resource;
390 struct rte_avp_memmap_info *info;
391 struct rte_avp_memmap *map;
396 addr = pci_dev->mem_resource[RTE_AVP_PCI_MEMORY_BAR].addr;
397 resource = &pci_dev->mem_resource[RTE_AVP_PCI_MEMMAP_BAR];
398 info = (struct rte_avp_memmap_info *)resource->addr;
401 for (i = 0; i < info->nb_maps; i++) {
402 /* search all segments looking for a matching address */
403 map = &info->maps[i];
405 if ((host_phys_addr >= map->phys_addr) &&
406 (host_phys_addr < (map->phys_addr + map->length))) {
407 /* address is within this segment */
408 offset += (host_phys_addr - map->phys_addr);
409 addr = RTE_PTR_ADD(addr, offset);
411 PMD_DRV_LOG(DEBUG, "Translating host physical 0x%" PRIx64 " to guest virtual 0x%p\n",
412 host_phys_addr, addr);
416 offset += map->length;
422 /* verify that the incoming device version is compatible with our version */
424 avp_dev_version_check(uint32_t version)
426 uint32_t driver = RTE_AVP_STRIP_MINOR_VERSION(AVP_DPDK_DRIVER_VERSION);
427 uint32_t device = RTE_AVP_STRIP_MINOR_VERSION(version);
429 if (device <= driver) {
430 /* the host driver version is less than or equal to ours */
437 /* verify that memory regions have expected version and validation markers */
439 avp_dev_check_regions(struct rte_eth_dev *eth_dev)
441 struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev);
442 struct rte_avp_memmap_info *memmap;
443 struct rte_avp_device_info *info;
444 struct rte_mem_resource *resource;
447 /* Dump resource info for debug */
448 for (i = 0; i < PCI_MAX_RESOURCE; i++) {
449 resource = &pci_dev->mem_resource[i];
450 if ((resource->phys_addr == 0) || (resource->len == 0))
453 PMD_DRV_LOG(DEBUG, "resource[%u]: phys=0x%" PRIx64 " len=%" PRIu64 " addr=%p\n",
454 i, resource->phys_addr,
455 resource->len, resource->addr);
458 case RTE_AVP_PCI_MEMMAP_BAR:
459 memmap = (struct rte_avp_memmap_info *)resource->addr;
460 if ((memmap->magic != RTE_AVP_MEMMAP_MAGIC) ||
461 (memmap->version != RTE_AVP_MEMMAP_VERSION)) {
462 PMD_DRV_LOG(ERR, "Invalid memmap magic 0x%08x and version %u\n",
463 memmap->magic, memmap->version);
468 case RTE_AVP_PCI_DEVICE_BAR:
469 info = (struct rte_avp_device_info *)resource->addr;
470 if ((info->magic != RTE_AVP_DEVICE_MAGIC) ||
471 avp_dev_version_check(info->version)) {
472 PMD_DRV_LOG(ERR, "Invalid device info magic 0x%08x or version 0x%08x > 0x%08x\n",
473 info->magic, info->version,
474 AVP_DPDK_DRIVER_VERSION);
479 case RTE_AVP_PCI_MEMORY_BAR:
480 case RTE_AVP_PCI_MMIO_BAR:
481 if (resource->addr == NULL) {
482 PMD_DRV_LOG(ERR, "Missing address space for BAR%u\n",
488 case RTE_AVP_PCI_MSIX_BAR:
490 /* no validation required */
499 _avp_set_rx_queue_mappings(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
501 struct avp_dev *avp =
502 AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
503 struct avp_queue *rxq;
504 uint16_t queue_count;
507 rxq = (struct avp_queue *)eth_dev->data->rx_queues[rx_queue_id];
510 * Must map all AVP fifos as evenly as possible between the configured
511 * device queues. Each device queue will service a subset of the AVP
512 * fifos. If there is an odd number of device queues the first set of
513 * device queues will get the extra AVP fifos.
515 queue_count = avp->num_rx_queues / eth_dev->data->nb_rx_queues;
516 remainder = avp->num_rx_queues % eth_dev->data->nb_rx_queues;
517 if (rx_queue_id < remainder) {
518 /* these queues must service one extra FIFO */
519 rxq->queue_base = rx_queue_id * (queue_count + 1);
520 rxq->queue_limit = rxq->queue_base + (queue_count + 1) - 1;
522 /* these queues service the regular number of FIFO */
523 rxq->queue_base = ((remainder * (queue_count + 1)) +
524 ((rx_queue_id - remainder) * queue_count));
525 rxq->queue_limit = rxq->queue_base + queue_count - 1;
528 PMD_DRV_LOG(DEBUG, "rxq %u at %p base %u limit %u\n",
529 rx_queue_id, rxq, rxq->queue_base, rxq->queue_limit);
531 rxq->queue_id = rxq->queue_base;
535 _avp_set_queue_counts(struct rte_eth_dev *eth_dev)
537 struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev);
538 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
539 struct rte_avp_device_info *host_info;
542 addr = pci_dev->mem_resource[RTE_AVP_PCI_DEVICE_BAR].addr;
543 host_info = (struct rte_avp_device_info *)addr;
546 * the transmit direction is not negotiated beyond respecting the max
547 * number of queues because the host can handle arbitrary guest tx
548 * queues (host rx queues).
550 avp->num_tx_queues = eth_dev->data->nb_tx_queues;
553 * the receive direction is more restrictive. The host requires a
554 * minimum number of guest rx queues (host tx queues) therefore
555 * negotiate a value that is at least as large as the host minimum
556 * requirement. If the host and guest values are not identical then a
557 * mapping will be established in the receive_queue_setup function.
559 avp->num_rx_queues = RTE_MAX(host_info->min_rx_queues,
560 eth_dev->data->nb_rx_queues);
562 PMD_DRV_LOG(DEBUG, "Requesting %u Tx and %u Rx queues from host\n",
563 avp->num_tx_queues, avp->num_rx_queues);
567 * create a AVP device using the supplied device info by first translating it
568 * to guest address space(s).
571 avp_dev_create(struct rte_pci_device *pci_dev,
572 struct rte_eth_dev *eth_dev)
574 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
575 struct rte_avp_device_info *host_info;
576 struct rte_mem_resource *resource;
579 resource = &pci_dev->mem_resource[RTE_AVP_PCI_DEVICE_BAR];
580 if (resource->addr == NULL) {
581 PMD_DRV_LOG(ERR, "BAR%u is not mapped\n",
582 RTE_AVP_PCI_DEVICE_BAR);
585 host_info = (struct rte_avp_device_info *)resource->addr;
587 if ((host_info->magic != RTE_AVP_DEVICE_MAGIC) ||
588 avp_dev_version_check(host_info->version)) {
589 PMD_DRV_LOG(ERR, "Invalid AVP PCI device, magic 0x%08x version 0x%08x > 0x%08x\n",
590 host_info->magic, host_info->version,
591 AVP_DPDK_DRIVER_VERSION);
595 PMD_DRV_LOG(DEBUG, "AVP host device is v%u.%u.%u\n",
596 RTE_AVP_GET_RELEASE_VERSION(host_info->version),
597 RTE_AVP_GET_MAJOR_VERSION(host_info->version),
598 RTE_AVP_GET_MINOR_VERSION(host_info->version));
600 PMD_DRV_LOG(DEBUG, "AVP host supports %u to %u TX queue(s)\n",
601 host_info->min_tx_queues, host_info->max_tx_queues);
602 PMD_DRV_LOG(DEBUG, "AVP host supports %u to %u RX queue(s)\n",
603 host_info->min_rx_queues, host_info->max_rx_queues);
604 PMD_DRV_LOG(DEBUG, "AVP host supports features 0x%08x\n",
605 host_info->features);
607 if (avp->magic != AVP_ETHDEV_MAGIC) {
609 * First time initialization (i.e., not during a VM
612 memset(avp, 0, sizeof(*avp));
613 avp->magic = AVP_ETHDEV_MAGIC;
614 avp->dev_data = eth_dev->data;
615 avp->port_id = eth_dev->data->port_id;
616 avp->host_mbuf_size = host_info->mbuf_size;
617 avp->host_features = host_info->features;
618 memcpy(&avp->ethaddr.addr_bytes[0],
619 host_info->ethaddr, ETHER_ADDR_LEN);
620 /* adjust max values to not exceed our max */
622 RTE_MIN(host_info->max_tx_queues, RTE_AVP_MAX_QUEUES);
624 RTE_MIN(host_info->max_rx_queues, RTE_AVP_MAX_QUEUES);
626 /* Re-attaching during migration */
628 /* TODO... requires validation of host values */
629 if ((host_info->features & avp->features) != avp->features) {
630 PMD_DRV_LOG(ERR, "AVP host features mismatched; 0x%08x, host=0x%08x\n",
631 avp->features, host_info->features);
632 /* this should not be possible; continue for now */
636 /* the device id is allowed to change over migrations */
637 avp->device_id = host_info->device_id;
639 /* translate incoming host addresses to guest address space */
640 PMD_DRV_LOG(DEBUG, "AVP first host tx queue at 0x%" PRIx64 "\n",
642 PMD_DRV_LOG(DEBUG, "AVP first host alloc queue at 0x%" PRIx64 "\n",
643 host_info->alloc_phys);
644 for (i = 0; i < avp->max_tx_queues; i++) {
645 avp->tx_q[i] = avp_dev_translate_address(eth_dev,
646 host_info->tx_phys + (i * host_info->tx_size));
648 avp->alloc_q[i] = avp_dev_translate_address(eth_dev,
649 host_info->alloc_phys + (i * host_info->alloc_size));
652 PMD_DRV_LOG(DEBUG, "AVP first host rx queue at 0x%" PRIx64 "\n",
654 PMD_DRV_LOG(DEBUG, "AVP first host free queue at 0x%" PRIx64 "\n",
655 host_info->free_phys);
656 for (i = 0; i < avp->max_rx_queues; i++) {
657 avp->rx_q[i] = avp_dev_translate_address(eth_dev,
658 host_info->rx_phys + (i * host_info->rx_size));
659 avp->free_q[i] = avp_dev_translate_address(eth_dev,
660 host_info->free_phys + (i * host_info->free_size));
663 PMD_DRV_LOG(DEBUG, "AVP host request queue at 0x%" PRIx64 "\n",
664 host_info->req_phys);
665 PMD_DRV_LOG(DEBUG, "AVP host response queue at 0x%" PRIx64 "\n",
666 host_info->resp_phys);
667 PMD_DRV_LOG(DEBUG, "AVP host sync address at 0x%" PRIx64 "\n",
668 host_info->sync_phys);
669 PMD_DRV_LOG(DEBUG, "AVP host mbuf address at 0x%" PRIx64 "\n",
670 host_info->mbuf_phys);
671 avp->req_q = avp_dev_translate_address(eth_dev, host_info->req_phys);
672 avp->resp_q = avp_dev_translate_address(eth_dev, host_info->resp_phys);
674 avp_dev_translate_address(eth_dev, host_info->sync_phys);
676 avp_dev_translate_address(eth_dev, host_info->mbuf_phys);
679 * store the host mbuf virtual address so that we can calculate
680 * relative offsets for each mbuf as they are processed
682 avp->host_mbuf_addr = host_info->mbuf_va;
683 avp->host_sync_addr = host_info->sync_va;
686 * store the maximum packet length that is supported by the host.
688 avp->max_rx_pkt_len = host_info->max_rx_pkt_len;
689 PMD_DRV_LOG(DEBUG, "AVP host max receive packet length is %u\n",
690 host_info->max_rx_pkt_len);
696 * This function is based on probe() function in avp_pci.c
697 * It returns 0 on success.
700 eth_avp_dev_init(struct rte_eth_dev *eth_dev)
702 struct avp_dev *avp =
703 AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
704 struct rte_pci_device *pci_dev;
707 pci_dev = AVP_DEV_TO_PCI(eth_dev);
708 eth_dev->dev_ops = &avp_eth_dev_ops;
709 eth_dev->rx_pkt_burst = &avp_recv_pkts;
710 eth_dev->tx_pkt_burst = &avp_xmit_pkts;
712 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
714 * no setup required on secondary processes. All data is saved
715 * in dev_private by the primary process. All resource should
716 * be mapped to the same virtual address so all pointers should
719 if (eth_dev->data->scattered_rx) {
720 PMD_DRV_LOG(NOTICE, "AVP device configured for chained mbufs\n");
721 eth_dev->rx_pkt_burst = avp_recv_scattered_pkts;
722 eth_dev->tx_pkt_burst = avp_xmit_scattered_pkts;
727 rte_eth_copy_pci_info(eth_dev, pci_dev);
729 eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
731 /* Check BAR resources */
732 ret = avp_dev_check_regions(eth_dev);
734 PMD_DRV_LOG(ERR, "Failed to validate BAR resources, ret=%d\n",
739 /* Handle each subtype */
740 ret = avp_dev_create(pci_dev, eth_dev);
742 PMD_DRV_LOG(ERR, "Failed to create device, ret=%d\n", ret);
746 /* Allocate memory for storing MAC addresses */
747 eth_dev->data->mac_addrs = rte_zmalloc("avp_ethdev", ETHER_ADDR_LEN, 0);
748 if (eth_dev->data->mac_addrs == NULL) {
749 PMD_DRV_LOG(ERR, "Failed to allocate %d bytes needed to store MAC addresses\n",
754 /* Get a mac from device config */
755 ether_addr_copy(&avp->ethaddr, ð_dev->data->mac_addrs[0]);
761 eth_avp_dev_uninit(struct rte_eth_dev *eth_dev)
763 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
766 if (eth_dev->data == NULL)
769 if (eth_dev->data->mac_addrs != NULL) {
770 rte_free(eth_dev->data->mac_addrs);
771 eth_dev->data->mac_addrs = NULL;
778 static struct eth_driver rte_avp_pmd = {
780 .id_table = pci_id_avp_map,
781 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
782 .probe = rte_eth_dev_pci_probe,
783 .remove = rte_eth_dev_pci_remove,
785 .eth_dev_init = eth_avp_dev_init,
786 .eth_dev_uninit = eth_avp_dev_uninit,
787 .dev_private_size = sizeof(struct avp_adapter),
791 avp_dev_enable_scattered(struct rte_eth_dev *eth_dev,
794 unsigned int max_rx_pkt_len;
796 max_rx_pkt_len = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
798 if ((max_rx_pkt_len > avp->guest_mbuf_size) ||
799 (max_rx_pkt_len > avp->host_mbuf_size)) {
801 * If the guest MTU is greater than either the host or guest
802 * buffers then chained mbufs have to be enabled in the TX
803 * direction. It is assumed that the application will not need
804 * to send packets larger than their max_rx_pkt_len (MRU).
809 if ((avp->max_rx_pkt_len > avp->guest_mbuf_size) ||
810 (avp->max_rx_pkt_len > avp->host_mbuf_size)) {
812 * If the host MRU is greater than its own mbuf size or the
813 * guest mbuf size then chained mbufs have to be enabled in the
823 avp_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
824 uint16_t rx_queue_id,
826 unsigned int socket_id,
827 const struct rte_eth_rxconf *rx_conf,
828 struct rte_mempool *pool)
830 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
831 struct rte_pktmbuf_pool_private *mbp_priv;
832 struct avp_queue *rxq;
834 if (rx_queue_id >= eth_dev->data->nb_rx_queues) {
835 PMD_DRV_LOG(ERR, "RX queue id is out of range: rx_queue_id=%u, nb_rx_queues=%u\n",
836 rx_queue_id, eth_dev->data->nb_rx_queues);
840 /* Save mbuf pool pointer */
843 /* Save the local mbuf size */
844 mbp_priv = rte_mempool_get_priv(pool);
845 avp->guest_mbuf_size = (uint16_t)(mbp_priv->mbuf_data_room_size);
846 avp->guest_mbuf_size -= RTE_PKTMBUF_HEADROOM;
848 if (avp_dev_enable_scattered(eth_dev, avp)) {
849 if (!eth_dev->data->scattered_rx) {
850 PMD_DRV_LOG(NOTICE, "AVP device configured for chained mbufs\n");
851 eth_dev->data->scattered_rx = 1;
852 eth_dev->rx_pkt_burst = avp_recv_scattered_pkts;
853 eth_dev->tx_pkt_burst = avp_xmit_scattered_pkts;
857 PMD_DRV_LOG(DEBUG, "AVP max_rx_pkt_len=(%u,%u) mbuf_size=(%u,%u)\n",
859 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len,
861 avp->guest_mbuf_size);
863 /* allocate a queue object */
864 rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct avp_queue),
865 RTE_CACHE_LINE_SIZE, socket_id);
867 PMD_DRV_LOG(ERR, "Failed to allocate new Rx queue object\n");
871 /* save back pointers to AVP and Ethernet devices */
873 rxq->dev_data = eth_dev->data;
874 eth_dev->data->rx_queues[rx_queue_id] = (void *)rxq;
876 /* setup the queue receive mapping for the current queue. */
877 _avp_set_rx_queue_mappings(eth_dev, rx_queue_id);
879 PMD_DRV_LOG(DEBUG, "Rx queue %u setup at %p\n", rx_queue_id, rxq);
887 avp_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
888 uint16_t tx_queue_id,
890 unsigned int socket_id,
891 const struct rte_eth_txconf *tx_conf)
893 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
894 struct avp_queue *txq;
896 if (tx_queue_id >= eth_dev->data->nb_tx_queues) {
897 PMD_DRV_LOG(ERR, "TX queue id is out of range: tx_queue_id=%u, nb_tx_queues=%u\n",
898 tx_queue_id, eth_dev->data->nb_tx_queues);
902 /* allocate a queue object */
903 txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct avp_queue),
904 RTE_CACHE_LINE_SIZE, socket_id);
906 PMD_DRV_LOG(ERR, "Failed to allocate new Tx queue object\n");
910 /* only the configured set of transmit queues are used */
911 txq->queue_id = tx_queue_id;
912 txq->queue_base = tx_queue_id;
913 txq->queue_limit = tx_queue_id;
915 /* save back pointers to AVP and Ethernet devices */
917 txq->dev_data = eth_dev->data;
918 eth_dev->data->tx_queues[tx_queue_id] = (void *)txq;
920 PMD_DRV_LOG(DEBUG, "Tx queue %u setup at %p\n", tx_queue_id, txq);
928 _avp_cmp_ether_addr(struct ether_addr *a, struct ether_addr *b)
930 uint16_t *_a = (uint16_t *)&a->addr_bytes[0];
931 uint16_t *_b = (uint16_t *)&b->addr_bytes[0];
932 return (_a[0] ^ _b[0]) | (_a[1] ^ _b[1]) | (_a[2] ^ _b[2]);
936 _avp_mac_filter(struct avp_dev *avp, struct rte_mbuf *m)
938 struct ether_hdr *eth = rte_pktmbuf_mtod(m, struct ether_hdr *);
940 if (likely(_avp_cmp_ether_addr(&avp->ethaddr, ð->d_addr) == 0)) {
941 /* allow all packets destined to our address */
945 if (likely(is_broadcast_ether_addr(ð->d_addr))) {
946 /* allow all broadcast packets */
950 if (likely(is_multicast_ether_addr(ð->d_addr))) {
951 /* allow all multicast packets */
955 if (avp->flags & AVP_F_PROMISC) {
956 /* allow all packets when in promiscuous mode */
963 #ifdef RTE_LIBRTE_AVP_DEBUG_BUFFERS
965 __avp_dev_buffer_sanity_check(struct avp_dev *avp, struct rte_avp_desc *buf)
967 struct rte_avp_desc *first_buf;
968 struct rte_avp_desc *pkt_buf;
969 unsigned int pkt_len;
970 unsigned int nb_segs;
974 first_buf = avp_dev_translate_buffer(avp, buf);
978 nb_segs = first_buf->nb_segs;
980 /* Adjust pointers for guest addressing */
981 pkt_buf = avp_dev_translate_buffer(avp, buf);
983 rte_panic("bad buffer: segment %u has an invalid address %p\n",
985 pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data);
986 if (pkt_data == NULL)
987 rte_panic("bad buffer: segment %u has a NULL data pointer\n",
989 if (pkt_buf->data_len == 0)
990 rte_panic("bad buffer: segment %u has 0 data length\n",
992 pkt_len += pkt_buf->data_len;
996 } while (nb_segs && (buf = pkt_buf->next) != NULL);
999 rte_panic("bad buffer: expected %u segments found %u\n",
1000 first_buf->nb_segs, (first_buf->nb_segs - nb_segs));
1001 if (pkt_len != first_buf->pkt_len)
1002 rte_panic("bad buffer: expected length %u found %u\n",
1003 first_buf->pkt_len, pkt_len);
1006 #define avp_dev_buffer_sanity_check(a, b) \
1007 __avp_dev_buffer_sanity_check((a), (b))
1009 #else /* RTE_LIBRTE_AVP_DEBUG_BUFFERS */
1011 #define avp_dev_buffer_sanity_check(a, b) do {} while (0)
1016 * Copy a host buffer chain to a set of mbufs. This function assumes that
1017 * there exactly the required number of mbufs to copy all source bytes.
1019 static inline struct rte_mbuf *
1020 avp_dev_copy_from_buffers(struct avp_dev *avp,
1021 struct rte_avp_desc *buf,
1022 struct rte_mbuf **mbufs,
1025 struct rte_mbuf *m_previous = NULL;
1026 struct rte_avp_desc *pkt_buf;
1027 unsigned int total_length = 0;
1028 unsigned int copy_length;
1029 unsigned int src_offset;
1036 avp_dev_buffer_sanity_check(avp, buf);
1038 /* setup the first source buffer */
1039 pkt_buf = avp_dev_translate_buffer(avp, buf);
1040 pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data);
1041 total_length = pkt_buf->pkt_len;
1044 if (pkt_buf->ol_flags & RTE_AVP_RX_VLAN_PKT) {
1045 ol_flags = PKT_RX_VLAN_PKT;
1046 vlan_tci = pkt_buf->vlan_tci;
1052 for (i = 0; (i < count) && (buf != NULL); i++) {
1053 /* fill each destination buffer */
1056 if (m_previous != NULL)
1057 m_previous->next = m;
1063 * Copy as many source buffers as will fit in the
1064 * destination buffer.
1066 copy_length = RTE_MIN((avp->guest_mbuf_size -
1067 rte_pktmbuf_data_len(m)),
1068 (pkt_buf->data_len -
1070 rte_memcpy(RTE_PTR_ADD(rte_pktmbuf_mtod(m, void *),
1071 rte_pktmbuf_data_len(m)),
1072 RTE_PTR_ADD(pkt_data, src_offset),
1074 rte_pktmbuf_data_len(m) += copy_length;
1075 src_offset += copy_length;
1077 if (likely(src_offset == pkt_buf->data_len)) {
1078 /* need a new source buffer */
1079 buf = pkt_buf->next;
1081 pkt_buf = avp_dev_translate_buffer(
1083 pkt_data = avp_dev_translate_buffer(
1084 avp, pkt_buf->data);
1089 if (unlikely(rte_pktmbuf_data_len(m) ==
1090 avp->guest_mbuf_size)) {
1091 /* need a new destination mbuf */
1095 } while (buf != NULL);
1099 m->ol_flags = ol_flags;
1101 rte_pktmbuf_pkt_len(m) = total_length;
1102 m->vlan_tci = vlan_tci;
1104 __rte_mbuf_sanity_check(m, 1);
1110 avp_recv_scattered_pkts(void *rx_queue,
1111 struct rte_mbuf **rx_pkts,
1114 struct avp_queue *rxq = (struct avp_queue *)rx_queue;
1115 struct rte_avp_desc *avp_bufs[AVP_MAX_RX_BURST];
1116 struct rte_mbuf *mbufs[RTE_AVP_MAX_MBUF_SEGMENTS];
1117 struct avp_dev *avp = rxq->avp;
1118 struct rte_avp_desc *pkt_buf;
1119 struct rte_avp_fifo *free_q;
1120 struct rte_avp_fifo *rx_q;
1121 struct rte_avp_desc *buf;
1122 unsigned int count, avail, n;
1123 unsigned int guest_mbuf_size;
1125 unsigned int required;
1126 unsigned int buf_len;
1127 unsigned int port_id;
1130 guest_mbuf_size = avp->guest_mbuf_size;
1131 port_id = avp->port_id;
1132 rx_q = avp->rx_q[rxq->queue_id];
1133 free_q = avp->free_q[rxq->queue_id];
1135 /* setup next queue to service */
1136 rxq->queue_id = (rxq->queue_id < rxq->queue_limit) ?
1137 (rxq->queue_id + 1) : rxq->queue_base;
1139 /* determine how many slots are available in the free queue */
1140 count = avp_fifo_free_count(free_q);
1142 /* determine how many packets are available in the rx queue */
1143 avail = avp_fifo_count(rx_q);
1145 /* determine how many packets can be received */
1146 count = RTE_MIN(count, avail);
1147 count = RTE_MIN(count, nb_pkts);
1148 count = RTE_MIN(count, (unsigned int)AVP_MAX_RX_BURST);
1150 if (unlikely(count == 0)) {
1151 /* no free buffers, or no buffers on the rx queue */
1155 /* retrieve pending packets */
1156 n = avp_fifo_get(rx_q, (void **)&avp_bufs, count);
1157 PMD_RX_LOG(DEBUG, "Receiving %u packets from Rx queue at %p\n",
1161 for (i = 0; i < n; i++) {
1162 /* prefetch next entry while processing current one */
1164 pkt_buf = avp_dev_translate_buffer(avp,
1166 rte_prefetch0(pkt_buf);
1170 /* Peek into the first buffer to determine the total length */
1171 pkt_buf = avp_dev_translate_buffer(avp, buf);
1172 buf_len = pkt_buf->pkt_len;
1174 /* Allocate enough mbufs to receive the entire packet */
1175 required = (buf_len + guest_mbuf_size - 1) / guest_mbuf_size;
1176 if (rte_pktmbuf_alloc_bulk(avp->pool, mbufs, required)) {
1177 rxq->dev_data->rx_mbuf_alloc_failed++;
1181 /* Copy the data from the buffers to our mbufs */
1182 m = avp_dev_copy_from_buffers(avp, buf, mbufs, required);
1187 if (_avp_mac_filter(avp, m) != 0) {
1188 /* silently discard packets not destined to our MAC */
1189 rte_pktmbuf_free(m);
1193 /* return new mbuf to caller */
1194 rx_pkts[count++] = m;
1195 rxq->bytes += buf_len;
1198 rxq->packets += count;
1200 /* return the buffers to the free queue */
1201 avp_fifo_put(free_q, (void **)&avp_bufs[0], n);
1208 avp_recv_pkts(void *rx_queue,
1209 struct rte_mbuf **rx_pkts,
1212 struct avp_queue *rxq = (struct avp_queue *)rx_queue;
1213 struct rte_avp_desc *avp_bufs[AVP_MAX_RX_BURST];
1214 struct avp_dev *avp = rxq->avp;
1215 struct rte_avp_desc *pkt_buf;
1216 struct rte_avp_fifo *free_q;
1217 struct rte_avp_fifo *rx_q;
1218 unsigned int count, avail, n;
1219 unsigned int pkt_len;
1224 rx_q = avp->rx_q[rxq->queue_id];
1225 free_q = avp->free_q[rxq->queue_id];
1227 /* setup next queue to service */
1228 rxq->queue_id = (rxq->queue_id < rxq->queue_limit) ?
1229 (rxq->queue_id + 1) : rxq->queue_base;
1231 /* determine how many slots are available in the free queue */
1232 count = avp_fifo_free_count(free_q);
1234 /* determine how many packets are available in the rx queue */
1235 avail = avp_fifo_count(rx_q);
1237 /* determine how many packets can be received */
1238 count = RTE_MIN(count, avail);
1239 count = RTE_MIN(count, nb_pkts);
1240 count = RTE_MIN(count, (unsigned int)AVP_MAX_RX_BURST);
1242 if (unlikely(count == 0)) {
1243 /* no free buffers, or no buffers on the rx queue */
1247 /* retrieve pending packets */
1248 n = avp_fifo_get(rx_q, (void **)&avp_bufs, count);
1249 PMD_RX_LOG(DEBUG, "Receiving %u packets from Rx queue at %p\n",
1253 for (i = 0; i < n; i++) {
1254 /* prefetch next entry while processing current one */
1256 pkt_buf = avp_dev_translate_buffer(avp,
1258 rte_prefetch0(pkt_buf);
1261 /* Adjust host pointers for guest addressing */
1262 pkt_buf = avp_dev_translate_buffer(avp, avp_bufs[i]);
1263 pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data);
1264 pkt_len = pkt_buf->pkt_len;
1266 if (unlikely((pkt_len > avp->guest_mbuf_size) ||
1267 (pkt_buf->nb_segs > 1))) {
1269 * application should be using the scattered receive
1276 /* process each packet to be transmitted */
1277 m = rte_pktmbuf_alloc(avp->pool);
1278 if (unlikely(m == NULL)) {
1279 rxq->dev_data->rx_mbuf_alloc_failed++;
1283 /* copy data out of the host buffer to our buffer */
1284 m->data_off = RTE_PKTMBUF_HEADROOM;
1285 rte_memcpy(rte_pktmbuf_mtod(m, void *), pkt_data, pkt_len);
1287 /* initialize the local mbuf */
1288 rte_pktmbuf_data_len(m) = pkt_len;
1289 rte_pktmbuf_pkt_len(m) = pkt_len;
1290 m->port = avp->port_id;
1292 if (pkt_buf->ol_flags & RTE_AVP_RX_VLAN_PKT) {
1293 m->ol_flags = PKT_RX_VLAN_PKT;
1294 m->vlan_tci = pkt_buf->vlan_tci;
1297 if (_avp_mac_filter(avp, m) != 0) {
1298 /* silently discard packets not destined to our MAC */
1299 rte_pktmbuf_free(m);
1303 /* return new mbuf to caller */
1304 rx_pkts[count++] = m;
1305 rxq->bytes += pkt_len;
1308 rxq->packets += count;
1310 /* return the buffers to the free queue */
1311 avp_fifo_put(free_q, (void **)&avp_bufs[0], n);
1317 * Copy a chained mbuf to a set of host buffers. This function assumes that
1318 * there are sufficient destination buffers to contain the entire source
1321 static inline uint16_t
1322 avp_dev_copy_to_buffers(struct avp_dev *avp,
1323 struct rte_mbuf *mbuf,
1324 struct rte_avp_desc **buffers,
1327 struct rte_avp_desc *previous_buf = NULL;
1328 struct rte_avp_desc *first_buf = NULL;
1329 struct rte_avp_desc *pkt_buf;
1330 struct rte_avp_desc *buf;
1331 size_t total_length;
1338 __rte_mbuf_sanity_check(mbuf, 1);
1342 total_length = rte_pktmbuf_pkt_len(m);
1343 for (i = 0; (i < count) && (m != NULL); i++) {
1344 /* fill each destination buffer */
1347 if (i < count - 1) {
1348 /* prefetch next entry while processing this one */
1349 pkt_buf = avp_dev_translate_buffer(avp, buffers[i + 1]);
1350 rte_prefetch0(pkt_buf);
1353 /* Adjust pointers for guest addressing */
1354 pkt_buf = avp_dev_translate_buffer(avp, buf);
1355 pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data);
1357 /* setup the buffer chain */
1358 if (previous_buf != NULL)
1359 previous_buf->next = buf;
1361 first_buf = pkt_buf;
1363 previous_buf = pkt_buf;
1367 * copy as many source mbuf segments as will fit in the
1368 * destination buffer.
1370 copy_length = RTE_MIN((avp->host_mbuf_size -
1372 (rte_pktmbuf_data_len(m) -
1374 rte_memcpy(RTE_PTR_ADD(pkt_data, pkt_buf->data_len),
1375 RTE_PTR_ADD(rte_pktmbuf_mtod(m, void *),
1378 pkt_buf->data_len += copy_length;
1379 src_offset += copy_length;
1381 if (likely(src_offset == rte_pktmbuf_data_len(m))) {
1382 /* need a new source buffer */
1387 if (unlikely(pkt_buf->data_len ==
1388 avp->host_mbuf_size)) {
1389 /* need a new destination buffer */
1393 } while (m != NULL);
1396 first_buf->nb_segs = count;
1397 first_buf->pkt_len = total_length;
1399 if (mbuf->ol_flags & PKT_TX_VLAN_PKT) {
1400 first_buf->ol_flags |= RTE_AVP_TX_VLAN_PKT;
1401 first_buf->vlan_tci = mbuf->vlan_tci;
1404 avp_dev_buffer_sanity_check(avp, buffers[0]);
1406 return total_length;
1411 avp_xmit_scattered_pkts(void *tx_queue,
1412 struct rte_mbuf **tx_pkts,
1415 struct rte_avp_desc *avp_bufs[(AVP_MAX_TX_BURST *
1416 RTE_AVP_MAX_MBUF_SEGMENTS)];
1417 struct avp_queue *txq = (struct avp_queue *)tx_queue;
1418 struct rte_avp_desc *tx_bufs[AVP_MAX_TX_BURST];
1419 struct avp_dev *avp = txq->avp;
1420 struct rte_avp_fifo *alloc_q;
1421 struct rte_avp_fifo *tx_q;
1422 unsigned int count, avail, n;
1423 unsigned int orig_nb_pkts;
1425 unsigned int required;
1426 unsigned int segments;
1427 unsigned int tx_bytes;
1430 orig_nb_pkts = nb_pkts;
1431 tx_q = avp->tx_q[txq->queue_id];
1432 alloc_q = avp->alloc_q[txq->queue_id];
1434 /* limit the number of transmitted packets to the max burst size */
1435 if (unlikely(nb_pkts > AVP_MAX_TX_BURST))
1436 nb_pkts = AVP_MAX_TX_BURST;
1438 /* determine how many buffers are available to copy into */
1439 avail = avp_fifo_count(alloc_q);
1440 if (unlikely(avail > (AVP_MAX_TX_BURST *
1441 RTE_AVP_MAX_MBUF_SEGMENTS)))
1442 avail = AVP_MAX_TX_BURST * RTE_AVP_MAX_MBUF_SEGMENTS;
1444 /* determine how many slots are available in the transmit queue */
1445 count = avp_fifo_free_count(tx_q);
1447 /* determine how many packets can be sent */
1448 nb_pkts = RTE_MIN(count, nb_pkts);
1450 /* determine how many packets will fit in the available buffers */
1453 for (i = 0; i < nb_pkts; i++) {
1455 if (likely(i < (unsigned int)nb_pkts - 1)) {
1456 /* prefetch next entry while processing this one */
1457 rte_prefetch0(tx_pkts[i + 1]);
1459 required = (rte_pktmbuf_pkt_len(m) + avp->host_mbuf_size - 1) /
1460 avp->host_mbuf_size;
1462 if (unlikely((required == 0) ||
1463 (required > RTE_AVP_MAX_MBUF_SEGMENTS)))
1465 else if (unlikely(required + segments > avail))
1467 segments += required;
1472 if (unlikely(nb_pkts == 0)) {
1473 /* no available buffers, or no space on the tx queue */
1474 txq->errors += orig_nb_pkts;
1478 PMD_TX_LOG(DEBUG, "Sending %u packets on Tx queue at %p\n",
1481 /* retrieve sufficient send buffers */
1482 n = avp_fifo_get(alloc_q, (void **)&avp_bufs, segments);
1483 if (unlikely(n != segments)) {
1484 PMD_TX_LOG(DEBUG, "Failed to allocate buffers "
1485 "n=%u, segments=%u, orig=%u\n",
1486 n, segments, orig_nb_pkts);
1487 txq->errors += orig_nb_pkts;
1493 for (i = 0; i < nb_pkts; i++) {
1494 /* process each packet to be transmitted */
1497 /* determine how many buffers are required for this packet */
1498 required = (rte_pktmbuf_pkt_len(m) + avp->host_mbuf_size - 1) /
1499 avp->host_mbuf_size;
1501 tx_bytes += avp_dev_copy_to_buffers(avp, m,
1502 &avp_bufs[count], required);
1503 tx_bufs[i] = avp_bufs[count];
1506 /* free the original mbuf */
1507 rte_pktmbuf_free(m);
1510 txq->packets += nb_pkts;
1511 txq->bytes += tx_bytes;
1513 #ifdef RTE_LIBRTE_AVP_DEBUG_BUFFERS
1514 for (i = 0; i < nb_pkts; i++)
1515 avp_dev_buffer_sanity_check(avp, tx_bufs[i]);
1518 /* send the packets */
1519 n = avp_fifo_put(tx_q, (void **)&tx_bufs[0], nb_pkts);
1520 if (unlikely(n != orig_nb_pkts))
1521 txq->errors += (orig_nb_pkts - n);
1528 avp_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1530 struct avp_queue *txq = (struct avp_queue *)tx_queue;
1531 struct rte_avp_desc *avp_bufs[AVP_MAX_TX_BURST];
1532 struct avp_dev *avp = txq->avp;
1533 struct rte_avp_desc *pkt_buf;
1534 struct rte_avp_fifo *alloc_q;
1535 struct rte_avp_fifo *tx_q;
1536 unsigned int count, avail, n;
1538 unsigned int pkt_len;
1539 unsigned int tx_bytes;
1543 tx_q = avp->tx_q[txq->queue_id];
1544 alloc_q = avp->alloc_q[txq->queue_id];
1546 /* limit the number of transmitted packets to the max burst size */
1547 if (unlikely(nb_pkts > AVP_MAX_TX_BURST))
1548 nb_pkts = AVP_MAX_TX_BURST;
1550 /* determine how many buffers are available to copy into */
1551 avail = avp_fifo_count(alloc_q);
1553 /* determine how many slots are available in the transmit queue */
1554 count = avp_fifo_free_count(tx_q);
1556 /* determine how many packets can be sent */
1557 count = RTE_MIN(count, avail);
1558 count = RTE_MIN(count, nb_pkts);
1560 if (unlikely(count == 0)) {
1561 /* no available buffers, or no space on the tx queue */
1562 txq->errors += nb_pkts;
1566 PMD_TX_LOG(DEBUG, "Sending %u packets on Tx queue at %p\n",
1569 /* retrieve sufficient send buffers */
1570 n = avp_fifo_get(alloc_q, (void **)&avp_bufs, count);
1571 if (unlikely(n != count)) {
1577 for (i = 0; i < count; i++) {
1578 /* prefetch next entry while processing the current one */
1579 if (i < count - 1) {
1580 pkt_buf = avp_dev_translate_buffer(avp,
1582 rte_prefetch0(pkt_buf);
1585 /* process each packet to be transmitted */
1588 /* Adjust pointers for guest addressing */
1589 pkt_buf = avp_dev_translate_buffer(avp, avp_bufs[i]);
1590 pkt_data = avp_dev_translate_buffer(avp, pkt_buf->data);
1591 pkt_len = rte_pktmbuf_pkt_len(m);
1593 if (unlikely((pkt_len > avp->guest_mbuf_size) ||
1594 (pkt_len > avp->host_mbuf_size))) {
1596 * application should be using the scattered transmit
1597 * function; send it truncated to avoid the performance
1598 * hit of having to manage returning the already
1599 * allocated buffer to the free list. This should not
1600 * happen since the application should have set the
1601 * max_rx_pkt_len based on its MTU and it should be
1602 * policing its own packet sizes.
1605 pkt_len = RTE_MIN(avp->guest_mbuf_size,
1606 avp->host_mbuf_size);
1609 /* copy data out of our mbuf and into the AVP buffer */
1610 rte_memcpy(pkt_data, rte_pktmbuf_mtod(m, void *), pkt_len);
1611 pkt_buf->pkt_len = pkt_len;
1612 pkt_buf->data_len = pkt_len;
1613 pkt_buf->nb_segs = 1;
1614 pkt_buf->next = NULL;
1616 if (m->ol_flags & PKT_TX_VLAN_PKT) {
1617 pkt_buf->ol_flags |= RTE_AVP_TX_VLAN_PKT;
1618 pkt_buf->vlan_tci = m->vlan_tci;
1621 tx_bytes += pkt_len;
1623 /* free the original mbuf */
1624 rte_pktmbuf_free(m);
1627 txq->packets += count;
1628 txq->bytes += tx_bytes;
1630 /* send the packets */
1631 n = avp_fifo_put(tx_q, (void **)&avp_bufs[0], count);
1637 avp_dev_rx_queue_release(void *rx_queue)
1639 struct avp_queue *rxq = (struct avp_queue *)rx_queue;
1640 struct avp_dev *avp = rxq->avp;
1641 struct rte_eth_dev_data *data = avp->dev_data;
1644 for (i = 0; i < avp->num_rx_queues; i++) {
1645 if (data->rx_queues[i] == rxq)
1646 data->rx_queues[i] = NULL;
1651 avp_dev_tx_queue_release(void *tx_queue)
1653 struct avp_queue *txq = (struct avp_queue *)tx_queue;
1654 struct avp_dev *avp = txq->avp;
1655 struct rte_eth_dev_data *data = avp->dev_data;
1658 for (i = 0; i < avp->num_tx_queues; i++) {
1659 if (data->tx_queues[i] == txq)
1660 data->tx_queues[i] = NULL;
1665 avp_dev_configure(struct rte_eth_dev *eth_dev)
1667 struct rte_pci_device *pci_dev = AVP_DEV_TO_PCI(eth_dev);
1668 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1669 struct rte_avp_device_info *host_info;
1670 struct rte_avp_device_config config;
1675 addr = pci_dev->mem_resource[RTE_AVP_PCI_DEVICE_BAR].addr;
1676 host_info = (struct rte_avp_device_info *)addr;
1678 /* Setup required number of queues */
1679 _avp_set_queue_counts(eth_dev);
1681 mask = (ETH_VLAN_STRIP_MASK |
1682 ETH_VLAN_FILTER_MASK |
1683 ETH_VLAN_EXTEND_MASK);
1684 avp_vlan_offload_set(eth_dev, mask);
1686 /* update device config */
1687 memset(&config, 0, sizeof(config));
1688 config.device_id = host_info->device_id;
1689 config.driver_type = RTE_AVP_DRIVER_TYPE_DPDK;
1690 config.driver_version = AVP_DPDK_DRIVER_VERSION;
1691 config.features = avp->features;
1692 config.num_tx_queues = avp->num_tx_queues;
1693 config.num_rx_queues = avp->num_rx_queues;
1695 ret = avp_dev_ctrl_set_config(eth_dev, &config);
1697 PMD_DRV_LOG(ERR, "Config request failed by host, ret=%d\n",
1702 avp->flags |= AVP_F_CONFIGURED;
1710 avp_dev_start(struct rte_eth_dev *eth_dev)
1712 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1715 /* disable features that we do not support */
1716 eth_dev->data->dev_conf.rxmode.hw_ip_checksum = 0;
1717 eth_dev->data->dev_conf.rxmode.hw_vlan_filter = 0;
1718 eth_dev->data->dev_conf.rxmode.hw_vlan_extend = 0;
1719 eth_dev->data->dev_conf.rxmode.hw_strip_crc = 0;
1721 /* update link state */
1722 ret = avp_dev_ctrl_set_link_state(eth_dev, 1);
1724 PMD_DRV_LOG(ERR, "Link state change failed by host, ret=%d\n",
1729 /* remember current link state */
1730 avp->flags |= AVP_F_LINKUP;
1739 avp_dev_stop(struct rte_eth_dev *eth_dev)
1741 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1744 avp->flags &= ~AVP_F_LINKUP;
1746 /* update link state */
1747 ret = avp_dev_ctrl_set_link_state(eth_dev, 0);
1749 PMD_DRV_LOG(ERR, "Link state change failed by host, ret=%d\n",
1755 avp_dev_close(struct rte_eth_dev *eth_dev)
1757 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1760 /* remember current link state */
1761 avp->flags &= ~AVP_F_LINKUP;
1762 avp->flags &= ~AVP_F_CONFIGURED;
1764 /* update device state */
1765 ret = avp_dev_ctrl_shutdown(eth_dev);
1767 PMD_DRV_LOG(ERR, "Device shutdown failed by host, ret=%d\n",
1774 avp_dev_link_update(struct rte_eth_dev *eth_dev,
1775 __rte_unused int wait_to_complete)
1777 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1778 struct rte_eth_link *link = ð_dev->data->dev_link;
1780 link->link_speed = ETH_SPEED_NUM_10G;
1781 link->link_duplex = ETH_LINK_FULL_DUPLEX;
1782 link->link_status = !!(avp->flags & AVP_F_LINKUP);
1788 avp_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
1790 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1792 if ((avp->flags & AVP_F_PROMISC) == 0) {
1793 avp->flags |= AVP_F_PROMISC;
1794 PMD_DRV_LOG(DEBUG, "Promiscuous mode enabled on %u\n",
1795 eth_dev->data->port_id);
1800 avp_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
1802 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1804 if ((avp->flags & AVP_F_PROMISC) != 0) {
1805 avp->flags &= ~AVP_F_PROMISC;
1806 PMD_DRV_LOG(DEBUG, "Promiscuous mode disabled on %u\n",
1807 eth_dev->data->port_id);
1812 avp_dev_info_get(struct rte_eth_dev *eth_dev,
1813 struct rte_eth_dev_info *dev_info)
1815 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1817 dev_info->driver_name = "rte_avp_pmd";
1818 dev_info->pci_dev = RTE_DEV_TO_PCI(eth_dev->device);
1819 dev_info->max_rx_queues = avp->max_rx_queues;
1820 dev_info->max_tx_queues = avp->max_tx_queues;
1821 dev_info->min_rx_bufsize = AVP_MIN_RX_BUFSIZE;
1822 dev_info->max_rx_pktlen = avp->max_rx_pkt_len;
1823 dev_info->max_mac_addrs = AVP_MAX_MAC_ADDRS;
1824 if (avp->host_features & RTE_AVP_FEATURE_VLAN_OFFLOAD) {
1825 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
1826 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
1831 avp_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask)
1833 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1835 if (mask & ETH_VLAN_STRIP_MASK) {
1836 if (avp->host_features & RTE_AVP_FEATURE_VLAN_OFFLOAD) {
1837 if (eth_dev->data->dev_conf.rxmode.hw_vlan_strip)
1838 avp->features |= RTE_AVP_FEATURE_VLAN_OFFLOAD;
1840 avp->features &= ~RTE_AVP_FEATURE_VLAN_OFFLOAD;
1842 PMD_DRV_LOG(ERR, "VLAN strip offload not supported\n");
1846 if (mask & ETH_VLAN_FILTER_MASK) {
1847 if (eth_dev->data->dev_conf.rxmode.hw_vlan_filter)
1848 PMD_DRV_LOG(ERR, "VLAN filter offload not supported\n");
1851 if (mask & ETH_VLAN_EXTEND_MASK) {
1852 if (eth_dev->data->dev_conf.rxmode.hw_vlan_extend)
1853 PMD_DRV_LOG(ERR, "VLAN extend offload not supported\n");
1858 avp_dev_stats_get(struct rte_eth_dev *eth_dev, struct rte_eth_stats *stats)
1860 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1863 for (i = 0; i < avp->num_rx_queues; i++) {
1864 struct avp_queue *rxq = avp->dev_data->rx_queues[i];
1867 stats->ipackets += rxq->packets;
1868 stats->ibytes += rxq->bytes;
1869 stats->ierrors += rxq->errors;
1871 stats->q_ipackets[i] += rxq->packets;
1872 stats->q_ibytes[i] += rxq->bytes;
1873 stats->q_errors[i] += rxq->errors;
1877 for (i = 0; i < avp->num_tx_queues; i++) {
1878 struct avp_queue *txq = avp->dev_data->tx_queues[i];
1881 stats->opackets += txq->packets;
1882 stats->obytes += txq->bytes;
1883 stats->oerrors += txq->errors;
1885 stats->q_opackets[i] += txq->packets;
1886 stats->q_obytes[i] += txq->bytes;
1887 stats->q_errors[i] += txq->errors;
1893 avp_dev_stats_reset(struct rte_eth_dev *eth_dev)
1895 struct avp_dev *avp = AVP_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1898 for (i = 0; i < avp->num_rx_queues; i++) {
1899 struct avp_queue *rxq = avp->dev_data->rx_queues[i];
1908 for (i = 0; i < avp->num_tx_queues; i++) {
1909 struct avp_queue *txq = avp->dev_data->tx_queues[i];
1919 RTE_PMD_REGISTER_PCI(net_avp, rte_avp_pmd.pci_drv);
1920 RTE_PMD_REGISTER_PCI_TABLE(net_avp, pci_id_avp_map);