X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmemif%2Frte_eth_memif.c;h=8dd1d0d63d001be8c6b7100408622b23d3dd1a48;hb=c62b6e667310a58e444dbaae1d08371a6d221333;hp=b9f05a624fda83b1f201c96cce9fcea0e0cfb9cb;hpb=09c7e63a71f9f2a31efd7e94458019ea4dc427b6;p=dpdk.git diff --git a/drivers/net/memif/rte_eth_memif.c b/drivers/net/memif/rte_eth_memif.c index b9f05a624f..8dd1d0d63d 100644 --- a/drivers/net/memif/rte_eth_memif.c +++ b/drivers/net/memif/rte_eth_memif.c @@ -23,6 +23,10 @@ #include #include #include +#include +#include +#include +#include #include "rte_eth_memif.h" #include "memif_socket.h" @@ -48,13 +52,137 @@ static const char * const valid_arguments[] = { NULL }; +#define MEMIF_MP_SEND_REGION "memif_mp_send_region" + + +static int memif_region_init_zc(const struct rte_memseg_list *msl, + const struct rte_memseg *ms, void *arg); + const char * memif_version(void) { return ("memif-" RTE_STR(MEMIF_VERSION_MAJOR) "." RTE_STR(MEMIF_VERSION_MINOR)); } -static void +/* Message header to synchronize regions */ +struct mp_region_msg { + char port_name[RTE_DEV_NAME_MAX_LEN]; + memif_region_index_t idx; + memif_region_size_t size; +}; + +static int +memif_mp_send_region(const struct rte_mp_msg *msg, const void *peer) +{ + struct rte_eth_dev *dev; + struct pmd_process_private *proc_private; + const struct mp_region_msg *msg_param = (const struct mp_region_msg *)msg->param; + struct rte_mp_msg reply; + struct mp_region_msg *reply_param = (struct mp_region_msg *)reply.param; + uint16_t port_id; + int ret; + + /* Get requested port */ + ret = rte_eth_dev_get_port_by_name(msg_param->port_name, &port_id); + if (ret) { + MIF_LOG(ERR, "Failed to get port id for %s", + msg_param->port_name); + return -1; + } + dev = &rte_eth_devices[port_id]; + proc_private = dev->process_private; + + memset(&reply, 0, sizeof(reply)); + strlcpy(reply.name, msg->name, sizeof(reply.name)); + reply_param->idx = msg_param->idx; + if (proc_private->regions[msg_param->idx] != NULL) { + reply_param->size = proc_private->regions[msg_param->idx]->region_size; + reply.fds[0] = proc_private->regions[msg_param->idx]->fd; + reply.num_fds = 1; + } + reply.len_param = sizeof(*reply_param); + if (rte_mp_reply(&reply, peer) < 0) { + MIF_LOG(ERR, "Failed to reply to an add region request"); + return -1; + } + + return 0; +} + +/* + * Request regions + * Called by secondary process, when ports link status goes up. + */ +static int +memif_mp_request_regions(struct rte_eth_dev *dev) +{ + int ret, i; + struct timespec timeout = {.tv_sec = 5, .tv_nsec = 0}; + struct rte_mp_msg msg, *reply; + struct rte_mp_reply replies; + struct mp_region_msg *msg_param = (struct mp_region_msg *)msg.param; + struct mp_region_msg *reply_param; + struct memif_region *r; + struct pmd_process_private *proc_private = dev->process_private; + struct pmd_internals *pmd = dev->data->dev_private; + /* in case of zero-copy slave, only request region 0 */ + uint16_t max_region_num = (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) ? + 1 : ETH_MEMIF_MAX_REGION_NUM; + + MIF_LOG(DEBUG, "Requesting memory regions"); + + for (i = 0; i < max_region_num; i++) { + /* Prepare the message */ + memset(&msg, 0, sizeof(msg)); + strlcpy(msg.name, MEMIF_MP_SEND_REGION, sizeof(msg.name)); + strlcpy(msg_param->port_name, dev->data->name, + sizeof(msg_param->port_name)); + msg_param->idx = i; + msg.len_param = sizeof(*msg_param); + + /* Send message */ + ret = rte_mp_request_sync(&msg, &replies, &timeout); + if (ret < 0 || replies.nb_received != 1) { + MIF_LOG(ERR, "Failed to send mp msg: %d", + rte_errno); + return -1; + } + + reply = &replies.msgs[0]; + reply_param = (struct mp_region_msg *)reply->param; + + if (reply_param->size > 0) { + r = rte_zmalloc("region", sizeof(struct memif_region), 0); + if (r == NULL) { + MIF_LOG(ERR, "Failed to alloc memif region."); + free(reply); + return -ENOMEM; + } + r->region_size = reply_param->size; + if (reply->num_fds < 1) { + MIF_LOG(ERR, "Missing file descriptor."); + free(reply); + return -1; + } + r->fd = reply->fds[0]; + r->addr = NULL; + + proc_private->regions[reply_param->idx] = r; + proc_private->regions_num++; + } + free(reply); + } + + if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) { + ret = rte_memseg_walk(memif_region_init_zc, (void *)proc_private); + if (ret < 0) + return ret; + } + + return memif_connect(dev); +} + +static int memif_dev_info(struct rte_eth_dev *dev __rte_unused, struct rte_eth_dev_info *dev_info) { dev_info->max_mac_addrs = 1; @@ -62,13 +190,16 @@ memif_dev_info(struct rte_eth_dev *dev __rte_unused, struct rte_eth_dev_info *de dev_info->max_rx_queues = ETH_MEMIF_MAX_NUM_Q_PAIRS; dev_info->max_tx_queues = ETH_MEMIF_MAX_NUM_Q_PAIRS; dev_info->min_rx_bufsize = 0; + + return 0; } static memif_ring_t * -memif_get_ring(struct pmd_internals *pmd, memif_ring_type_t type, uint16_t ring_num) +memif_get_ring(struct pmd_internals *pmd, struct pmd_process_private *proc_private, + memif_ring_type_t type, uint16_t ring_num) { /* rings only in region 0 */ - void *p = pmd->regions[0]->addr; + void *p = proc_private->regions[0]->addr; int ring_size = sizeof(memif_ring_t) + sizeof(memif_desc_t) * (1 << pmd->run.log2_ring_size); @@ -77,10 +208,51 @@ memif_get_ring(struct pmd_internals *pmd, memif_ring_type_t type, uint16_t ring_ return (memif_ring_t *)p; } +static memif_region_offset_t +memif_get_ring_offset(struct rte_eth_dev *dev, struct memif_queue *mq, + memif_ring_type_t type, uint16_t num) +{ + struct pmd_internals *pmd = dev->data->dev_private; + struct pmd_process_private *proc_private = dev->process_private; + + return ((uint8_t *)memif_get_ring(pmd, proc_private, type, num) - + (uint8_t *)proc_private->regions[mq->region]->addr); +} + +static memif_ring_t * +memif_get_ring_from_queue(struct pmd_process_private *proc_private, + struct memif_queue *mq) +{ + struct memif_region *r; + + r = proc_private->regions[mq->region]; + if (r == NULL) + return NULL; + + return (memif_ring_t *)((uint8_t *)r->addr + mq->ring_offset); +} + static void * -memif_get_buffer(struct pmd_internals *pmd, memif_desc_t *d) +memif_get_buffer(struct pmd_process_private *proc_private, memif_desc_t *d) { - return ((uint8_t *)pmd->regions[d->region]->addr + d->offset); + return ((uint8_t *)proc_private->regions[d->region]->addr + d->offset); +} + +/* Free mbufs received by master */ +static void +memif_free_stored_mbufs(struct pmd_process_private *proc_private, struct memif_queue *mq) +{ + uint16_t mask = (1 << mq->log2_ring_size) - 1; + memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq); + + /* FIXME: improve performance */ + while (mq->last_tail != ring->tail) { + RTE_MBUF_PREFETCH_TO_FREE(mq->buffers[(mq->last_tail + 1) & mask]); + /* Decrement refcnt and free mbuf. (current segment) */ + rte_mbuf_refcnt_update(mq->buffers[mq->last_tail & mask], -1); + rte_pktmbuf_free_seg(mq->buffers[mq->last_tail & mask]); + mq->last_tail++; + } } static int @@ -107,8 +279,10 @@ static uint16_t eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { struct memif_queue *mq = queue; - struct pmd_internals *pmd = mq->pmd; - memif_ring_t *ring = mq->ring; + struct pmd_internals *pmd = rte_eth_devices[mq->in_port].data->dev_private; + struct pmd_process_private *proc_private = + rte_eth_devices[mq->in_port].process_private; + memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq); uint16_t cur_slot, last_slot, n_slots, ring_size, mask, s0; uint16_t n_rx_pkts = 0; uint16_t mbuf_size = rte_pktmbuf_data_room_size(mq->mempool) - @@ -121,11 +295,18 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) ssize_t size __rte_unused; uint16_t head; int ret; + struct rte_eth_link link; if (unlikely((pmd->flags & ETH_MEMIF_FLAG_CONNECTED) == 0)) return 0; - if (unlikely(ring == NULL)) + if (unlikely(ring == NULL)) { + /* Secondary process will attempt to request regions. */ + ret = rte_eth_link_get(mq->in_port, &link); + if (ret < 0) + MIF_LOG(ERR, "Failed to get port %u link info: %s", + mq->in_port, rte_strerror(-ret)); return 0; + } /* consume interrupt */ if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0) @@ -134,8 +315,14 @@ eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) ring_size = 1 << mq->log2_ring_size; mask = ring_size - 1; - cur_slot = (type == MEMIF_RING_S2M) ? mq->last_head : mq->last_tail; - last_slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail; + if (type == MEMIF_RING_S2M) { + cur_slot = mq->last_head; + last_slot = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE); + } else { + cur_slot = mq->last_tail; + last_slot = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE); + } + if (cur_slot == last_slot) goto refill; n_slots = last_slot - cur_slot; @@ -169,8 +356,7 @@ next_slot: mbuf->port = mq->in_port; ret = memif_pktmbuf_chain(mbuf_head, mbuf_tail, mbuf); if (unlikely(ret < 0)) { - MIF_LOG(ERR, "%s: number-of-segments-overflow", - rte_vdev_device_name(pmd->vdev)); + MIF_LOG(ERR, "number-of-segments-overflow"); rte_pktmbuf_free(mbuf); goto no_free_bufs; } @@ -183,7 +369,8 @@ next_slot: rte_pktmbuf_pkt_len(mbuf_head) += cp_len; memcpy(rte_pktmbuf_mtod_offset(mbuf, void *, dst_off), - (uint8_t *)memif_get_buffer(pmd, d0) + src_off, cp_len); + (uint8_t *)memif_get_buffer(proc_private, d0) + src_off, + cp_len); src_off += cp_len; dst_off += cp_len; @@ -203,8 +390,7 @@ next_slot: no_free_bufs: if (type == MEMIF_RING_S2M) { - rte_mb(); - ring->tail = cur_slot; + __atomic_store_n(&ring->tail, cur_slot, __ATOMIC_RELEASE); mq->last_head = cur_slot; } else { mq->last_tail = cur_slot; @@ -212,7 +398,7 @@ no_free_bufs: refill: if (type == MEMIF_RING_M2S) { - head = ring->head; + head = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE); n_slots = ring_size - head + mq->last_tail; while (n_slots--) { @@ -220,20 +406,135 @@ refill: d0 = &ring->desc[s0]; d0->length = pmd->run.pkt_buffer_size; } - rte_mb(); - ring->head = head; + __atomic_store_n(&ring->head, head, __ATOMIC_RELEASE); } mq->n_pkts += n_rx_pkts; return n_rx_pkts; } +static uint16_t +eth_memif_rx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + struct memif_queue *mq = queue; + struct pmd_internals *pmd = rte_eth_devices[mq->in_port].data->dev_private; + struct pmd_process_private *proc_private = + rte_eth_devices[mq->in_port].process_private; + memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq); + uint16_t cur_slot, last_slot, n_slots, ring_size, mask, s0, head; + uint16_t n_rx_pkts = 0; + memif_desc_t *d0; + struct rte_mbuf *mbuf, *mbuf_tail; + struct rte_mbuf *mbuf_head = NULL; + int ret; + struct rte_eth_link link; + + if (unlikely((pmd->flags & ETH_MEMIF_FLAG_CONNECTED) == 0)) + return 0; + if (unlikely(ring == NULL)) { + /* Secondary process will attempt to request regions. */ + rte_eth_link_get(mq->in_port, &link); + return 0; + } + + /* consume interrupt */ + if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0) { + uint64_t b; + ssize_t size __rte_unused; + size = read(mq->intr_handle.fd, &b, sizeof(b)); + } + + ring_size = 1 << mq->log2_ring_size; + mask = ring_size - 1; + + cur_slot = mq->last_tail; + last_slot = ring->tail; + if (cur_slot == last_slot) + goto refill; + n_slots = last_slot - cur_slot; + + while (n_slots && n_rx_pkts < nb_pkts) { + s0 = cur_slot & mask; + + d0 = &ring->desc[s0]; + mbuf_head = mq->buffers[s0]; + mbuf = mbuf_head; + +next_slot: + /* prefetch next descriptor */ + if (n_rx_pkts + 1 < nb_pkts) + rte_prefetch0(&ring->desc[(cur_slot + 1) & mask]); + + mbuf->port = mq->in_port; + rte_pktmbuf_data_len(mbuf) = d0->length; + rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf); + + mq->n_bytes += rte_pktmbuf_data_len(mbuf); + + cur_slot++; + n_slots--; + if (d0->flags & MEMIF_DESC_FLAG_NEXT) { + s0 = cur_slot & mask; + d0 = &ring->desc[s0]; + mbuf_tail = mbuf; + mbuf = mq->buffers[s0]; + ret = memif_pktmbuf_chain(mbuf_head, mbuf_tail, mbuf); + if (unlikely(ret < 0)) { + MIF_LOG(ERR, "number-of-segments-overflow"); + goto refill; + } + goto next_slot; + } + + *bufs++ = mbuf_head; + n_rx_pkts++; + } + + mq->last_tail = cur_slot; + +/* Supply master with new buffers */ +refill: + head = ring->head; + n_slots = ring_size - head + mq->last_tail; + + if (n_slots < 32) + goto no_free_mbufs; + + ret = rte_pktmbuf_alloc_bulk(mq->mempool, &mq->buffers[head & mask], n_slots); + if (unlikely(ret < 0)) + goto no_free_mbufs; + + while (n_slots--) { + s0 = head++ & mask; + if (n_slots > 0) + rte_prefetch0(mq->buffers[head & mask]); + d0 = &ring->desc[s0]; + /* store buffer header */ + mbuf = mq->buffers[s0]; + /* populate descriptor */ + d0->length = rte_pktmbuf_data_room_size(mq->mempool) - + RTE_PKTMBUF_HEADROOM; + d0->region = 1; + d0->offset = rte_pktmbuf_mtod(mbuf, uint8_t *) - + (uint8_t *)proc_private->regions[d0->region]->addr; + } +no_free_mbufs: + rte_mb(); + ring->head = head; + + mq->n_pkts += n_rx_pkts; + + return n_rx_pkts; +} + static uint16_t eth_memif_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) { struct memif_queue *mq = queue; - struct pmd_internals *pmd = mq->pmd; - memif_ring_t *ring = mq->ring; + struct pmd_internals *pmd = rte_eth_devices[mq->in_port].data->dev_private; + struct pmd_process_private *proc_private = + rte_eth_devices[mq->in_port].process_private; + memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq); uint16_t slot, saved_slot, n_free, ring_size, mask, n_tx_pkts = 0; uint16_t src_len, src_off, dst_len, dst_off, cp_len; memif_ring_type_t type = mq->type; @@ -242,23 +543,34 @@ eth_memif_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) struct rte_mbuf *mbuf_head; uint64_t a; ssize_t size; + struct rte_eth_link link; if (unlikely((pmd->flags & ETH_MEMIF_FLAG_CONNECTED) == 0)) return 0; - if (unlikely(ring == NULL)) + if (unlikely(ring == NULL)) { + int ret; + + /* Secondary process will attempt to request regions. */ + ret = rte_eth_link_get(mq->in_port, &link); + if (ret < 0) + MIF_LOG(ERR, "Failed to get port %u link info: %s", + mq->in_port, rte_strerror(-ret)); return 0; + } ring_size = 1 << mq->log2_ring_size; mask = ring_size - 1; - n_free = ring->tail - mq->last_tail; + n_free = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE) - mq->last_tail; mq->last_tail += n_free; - slot = (type == MEMIF_RING_S2M) ? ring->head : ring->tail; - if (type == MEMIF_RING_S2M) - n_free = ring_size - ring->head + mq->last_tail; - else - n_free = ring->head - ring->tail; + if (type == MEMIF_RING_S2M) { + slot = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE); + n_free = ring_size - slot + mq->last_tail; + } else { + slot = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE); + n_free = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE) - slot; + } while (n_tx_pkts < nb_pkts && n_free) { mbuf_head = *bufs++; @@ -292,7 +604,7 @@ next_in_chain: } cp_len = RTE_MIN(dst_len, src_len); - memcpy((uint8_t *)memif_get_buffer(pmd, d0) + dst_off, + memcpy((uint8_t *)memif_get_buffer(proc_private, d0) + dst_off, rte_pktmbuf_mtod_offset(mbuf, void *, src_off), cp_len); @@ -316,40 +628,192 @@ next_in_chain: rte_pktmbuf_free(mbuf_head); } +no_free_slots: + if (type == MEMIF_RING_S2M) + __atomic_store_n(&ring->head, slot, __ATOMIC_RELEASE); + else + __atomic_store_n(&ring->tail, slot, __ATOMIC_RELEASE); + + if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0) { + a = 1; + size = write(mq->intr_handle.fd, &a, sizeof(a)); + if (unlikely(size < 0)) { + MIF_LOG(WARNING, + "Failed to send interrupt. %s", strerror(errno)); + } + } + + mq->n_pkts += n_tx_pkts; + return n_tx_pkts; +} + + +static int +memif_tx_one_zc(struct pmd_process_private *proc_private, struct memif_queue *mq, + memif_ring_t *ring, struct rte_mbuf *mbuf, const uint16_t mask, + uint16_t slot, uint16_t n_free) +{ + memif_desc_t *d0; + int used_slots = 1; + +next_in_chain: + /* store pointer to mbuf to free it later */ + mq->buffers[slot & mask] = mbuf; + /* Increment refcnt to make sure the buffer is not freed before master + * receives it. (current segment) + */ + rte_mbuf_refcnt_update(mbuf, 1); + /* populate descriptor */ + d0 = &ring->desc[slot & mask]; + d0->length = rte_pktmbuf_data_len(mbuf); + /* FIXME: get region index */ + d0->region = 1; + d0->offset = rte_pktmbuf_mtod(mbuf, uint8_t *) - + (uint8_t *)proc_private->regions[d0->region]->addr; + d0->flags = 0; + + /* check if buffer is chained */ + if (rte_pktmbuf_is_contiguous(mbuf) == 0) { + if (n_free < 2) + return 0; + /* mark buffer as chained */ + d0->flags |= MEMIF_DESC_FLAG_NEXT; + /* advance mbuf */ + mbuf = mbuf->next; + /* update counters */ + used_slots++; + slot++; + n_free--; + goto next_in_chain; + } + return used_slots; +} + +static uint16_t +eth_memif_tx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) +{ + struct memif_queue *mq = queue; + struct pmd_internals *pmd = rte_eth_devices[mq->in_port].data->dev_private; + struct pmd_process_private *proc_private = + rte_eth_devices[mq->in_port].process_private; + memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq); + uint16_t slot, n_free, ring_size, mask, n_tx_pkts = 0; + memif_ring_type_t type = mq->type; + struct rte_eth_link link; + + if (unlikely((pmd->flags & ETH_MEMIF_FLAG_CONNECTED) == 0)) + return 0; + if (unlikely(ring == NULL)) { + /* Secondary process will attempt to request regions. */ + rte_eth_link_get(mq->in_port, &link); + return 0; + } + + ring_size = 1 << mq->log2_ring_size; + mask = ring_size - 1; + + /* free mbufs received by master */ + memif_free_stored_mbufs(proc_private, mq); + + /* ring type always MEMIF_RING_S2M */ + slot = ring->head; + n_free = ring_size - ring->head + mq->last_tail; + + int used_slots; + + while (n_free && (n_tx_pkts < nb_pkts)) { + while ((n_free > 4) && ((nb_pkts - n_tx_pkts) > 4)) { + if ((nb_pkts - n_tx_pkts) > 8) { + rte_prefetch0(*bufs + 4); + rte_prefetch0(*bufs + 5); + rte_prefetch0(*bufs + 6); + rte_prefetch0(*bufs + 7); + } + used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++, + mask, slot, n_free); + if (unlikely(used_slots < 1)) + goto no_free_slots; + n_tx_pkts++; + slot += used_slots; + n_free -= used_slots; + + used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++, + mask, slot, n_free); + if (unlikely(used_slots < 1)) + goto no_free_slots; + n_tx_pkts++; + slot += used_slots; + n_free -= used_slots; + + used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++, + mask, slot, n_free); + if (unlikely(used_slots < 1)) + goto no_free_slots; + n_tx_pkts++; + slot += used_slots; + n_free -= used_slots; + + used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++, + mask, slot, n_free); + if (unlikely(used_slots < 1)) + goto no_free_slots; + n_tx_pkts++; + slot += used_slots; + n_free -= used_slots; + } + used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++, + mask, slot, n_free); + if (unlikely(used_slots < 1)) + goto no_free_slots; + n_tx_pkts++; + slot += used_slots; + n_free -= used_slots; + } + no_free_slots: rte_mb(); + /* update ring pointers */ if (type == MEMIF_RING_S2M) ring->head = slot; else ring->tail = slot; + /* Send interrupt, if enabled. */ if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0) { - a = 1; - size = write(mq->intr_handle.fd, &a, sizeof(a)); + uint64_t a = 1; + ssize_t size = write(mq->intr_handle.fd, &a, sizeof(a)); if (unlikely(size < 0)) { MIF_LOG(WARNING, - "%s: Failed to send interrupt. %s", - rte_vdev_device_name(pmd->vdev), strerror(errno)); + "Failed to send interrupt. %s", strerror(errno)); } } - mq->n_err += nb_pkts - n_tx_pkts; + /* increment queue counters */ mq->n_pkts += n_tx_pkts; + return n_tx_pkts; } void -memif_free_regions(struct pmd_internals *pmd) +memif_free_regions(struct rte_eth_dev *dev) { + struct pmd_process_private *proc_private = dev->process_private; + struct pmd_internals *pmd = dev->data->dev_private; int i; struct memif_region *r; /* regions are allocated contiguously, so it's - * enough to loop until 'pmd->regions_num' + * enough to loop until 'proc_private->regions_num' */ - for (i = 0; i < pmd->regions_num; i++) { - r = pmd->regions[i]; + for (i = 0; i < proc_private->regions_num; i++) { + r = proc_private->regions[i]; if (r != NULL) { + /* This is memzone */ + if (i > 0 && (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY)) { + r->addr = NULL; + if (r->fd > 0) + close(r->fd); + } if (r->addr != NULL) { munmap(r->addr, r->region_size); if (r->fd > 0) { @@ -358,28 +822,68 @@ memif_free_regions(struct pmd_internals *pmd) } } rte_free(r); - pmd->regions[i] = NULL; + proc_private->regions[i] = NULL; + } + } + proc_private->regions_num = 0; +} + +static int +memif_region_init_zc(const struct rte_memseg_list *msl, const struct rte_memseg *ms, + void *arg) +{ + struct pmd_process_private *proc_private = (struct pmd_process_private *)arg; + struct memif_region *r; + + if (proc_private->regions_num < 1) { + MIF_LOG(ERR, "Missing descriptor region"); + return -1; + } + + r = proc_private->regions[proc_private->regions_num - 1]; + + if (r->addr != msl->base_va) + r = proc_private->regions[++proc_private->regions_num - 1]; + + if (r == NULL) { + r = rte_zmalloc("region", sizeof(struct memif_region), 0); + if (r == NULL) { + MIF_LOG(ERR, "Failed to alloc memif region."); + return -ENOMEM; } + + r->addr = msl->base_va; + r->region_size = ms->len; + r->fd = rte_memseg_get_fd(ms); + if (r->fd < 0) + return -1; + r->pkt_buffer_offset = 0; + + proc_private->regions[proc_private->regions_num - 1] = r; + } else { + r->region_size += ms->len; } - pmd->regions_num = 0; + + return 0; } static int -memif_region_init_shm(struct pmd_internals *pmd, uint8_t has_buffers) +memif_region_init_shm(struct rte_eth_dev *dev, uint8_t has_buffers) { + struct pmd_internals *pmd = dev->data->dev_private; + struct pmd_process_private *proc_private = dev->process_private; char shm_name[ETH_MEMIF_SHM_NAME_SIZE]; int ret = 0; struct memif_region *r; - if (pmd->regions_num >= ETH_MEMIF_MAX_REGION_NUM) { - MIF_LOG(ERR, "%s: Too many regions.", rte_vdev_device_name(pmd->vdev)); + if (proc_private->regions_num >= ETH_MEMIF_MAX_REGION_NUM) { + MIF_LOG(ERR, "Too many regions."); return -1; } r = rte_zmalloc("region", sizeof(struct memif_region), 0); if (r == NULL) { - MIF_LOG(ERR, "%s: Failed to alloc memif region.", - rte_vdev_device_name(pmd->vdev)); + MIF_LOG(ERR, "Failed to alloc memif region."); return -ENOMEM; } @@ -398,45 +902,37 @@ memif_region_init_shm(struct pmd_internals *pmd, uint8_t has_buffers) memset(shm_name, 0, sizeof(char) * ETH_MEMIF_SHM_NAME_SIZE); snprintf(shm_name, ETH_MEMIF_SHM_NAME_SIZE, "memif_region_%d", - pmd->regions_num); + proc_private->regions_num); r->fd = memfd_create(shm_name, MFD_ALLOW_SEALING); if (r->fd < 0) { - MIF_LOG(ERR, "%s: Failed to create shm file: %s.", - rte_vdev_device_name(pmd->vdev), - strerror(errno)); + MIF_LOG(ERR, "Failed to create shm file: %s.", strerror(errno)); ret = -1; goto error; } ret = fcntl(r->fd, F_ADD_SEALS, F_SEAL_SHRINK); if (ret < 0) { - MIF_LOG(ERR, "%s: Failed to add seals to shm file: %s.", - rte_vdev_device_name(pmd->vdev), - strerror(errno)); + MIF_LOG(ERR, "Failed to add seals to shm file: %s.", strerror(errno)); goto error; } ret = ftruncate(r->fd, r->region_size); if (ret < 0) { - MIF_LOG(ERR, "%s: Failed to truncate shm file: %s.", - rte_vdev_device_name(pmd->vdev), - strerror(errno)); + MIF_LOG(ERR, "Failed to truncate shm file: %s.", strerror(errno)); goto error; } r->addr = mmap(NULL, r->region_size, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, 0); if (r->addr == MAP_FAILED) { - MIF_LOG(ERR, "%s: Failed to mmap shm region: %s.", - rte_vdev_device_name(pmd->vdev), - strerror(ret)); + MIF_LOG(ERR, "Failed to mmap shm region: %s.", strerror(ret)); ret = -1; goto error; } - pmd->regions[pmd->regions_num] = r; - pmd->regions_num++; + proc_private->regions[proc_private->regions_num] = r; + proc_private->regions_num++; return ret; @@ -449,14 +945,31 @@ error: } static int -memif_regions_init(struct pmd_internals *pmd) +memif_regions_init(struct rte_eth_dev *dev) { + struct pmd_internals *pmd = dev->data->dev_private; int ret; - /* create one buffer region */ - ret = memif_region_init_shm(pmd, /* has buffer */ 1); - if (ret < 0) - return ret; + /* + * Zero-copy exposes dpdk memory. + * Each memseg list will be represented by memif region. + * Zero-copy regions indexing: memseg list idx + 1, + * as we already have region 0 reserved for descriptors. + */ + if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) { + /* create region idx 0 containing descriptors */ + ret = memif_region_init_shm(dev, 0); + if (ret < 0) + return ret; + ret = rte_memseg_walk(memif_region_init_zc, (void *)dev->process_private); + if (ret < 0) + return ret; + } else { + /* create one memory region contaning rings and buffers */ + ret = memif_region_init_shm(dev, /* has buffers */ 1); + if (ret < 0) + return ret; + } return 0; } @@ -465,36 +978,47 @@ static void memif_init_rings(struct rte_eth_dev *dev) { struct pmd_internals *pmd = dev->data->dev_private; + struct pmd_process_private *proc_private = dev->process_private; memif_ring_t *ring; int i, j; uint16_t slot; for (i = 0; i < pmd->run.num_s2m_rings; i++) { - ring = memif_get_ring(pmd, MEMIF_RING_S2M, i); - ring->head = 0; - ring->tail = 0; + ring = memif_get_ring(pmd, proc_private, MEMIF_RING_S2M, i); + __atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED); + __atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED); ring->cookie = MEMIF_COOKIE; ring->flags = 0; + + if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) + continue; + for (j = 0; j < (1 << pmd->run.log2_ring_size); j++) { slot = i * (1 << pmd->run.log2_ring_size) + j; ring->desc[j].region = 0; - ring->desc[j].offset = pmd->regions[0]->pkt_buffer_offset + + ring->desc[j].offset = + proc_private->regions[0]->pkt_buffer_offset + (uint32_t)(slot * pmd->run.pkt_buffer_size); ring->desc[j].length = pmd->run.pkt_buffer_size; } } for (i = 0; i < pmd->run.num_m2s_rings; i++) { - ring = memif_get_ring(pmd, MEMIF_RING_M2S, i); - ring->head = 0; - ring->tail = 0; + ring = memif_get_ring(pmd, proc_private, MEMIF_RING_M2S, i); + __atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED); + __atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED); ring->cookie = MEMIF_COOKIE; ring->flags = 0; + + if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) + continue; + for (j = 0; j < (1 << pmd->run.log2_ring_size); j++) { slot = (i + pmd->run.num_s2m_rings) * (1 << pmd->run.log2_ring_size) + j; ring->desc[j].region = 0; - ring->desc[j].offset = pmd->regions[0]->pkt_buffer_offset + + ring->desc[j].offset = + proc_private->regions[0]->pkt_buffer_offset + (uint32_t)(slot * pmd->run.pkt_buffer_size); ring->desc[j].length = pmd->run.pkt_buffer_size; } @@ -502,7 +1026,7 @@ memif_init_rings(struct rte_eth_dev *dev) } /* called only by slave */ -static void +static int memif_init_queues(struct rte_eth_dev *dev) { struct pmd_internals *pmd = dev->data->dev_private; @@ -511,39 +1035,50 @@ memif_init_queues(struct rte_eth_dev *dev) for (i = 0; i < pmd->run.num_s2m_rings; i++) { mq = dev->data->tx_queues[i]; - mq->ring = memif_get_ring(pmd, MEMIF_RING_S2M, i); mq->log2_ring_size = pmd->run.log2_ring_size; /* queues located only in region 0 */ mq->region = 0; - mq->ring_offset = (uint8_t *)mq->ring - (uint8_t *)pmd->regions[0]->addr; + mq->ring_offset = memif_get_ring_offset(dev, mq, MEMIF_RING_S2M, i); mq->last_head = 0; mq->last_tail = 0; mq->intr_handle.fd = eventfd(0, EFD_NONBLOCK); if (mq->intr_handle.fd < 0) { MIF_LOG(WARNING, - "%s: Failed to create eventfd for tx queue %d: %s.", - rte_vdev_device_name(pmd->vdev), i, + "Failed to create eventfd for tx queue %d: %s.", i, strerror(errno)); } + mq->buffers = NULL; + if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) { + mq->buffers = rte_zmalloc("bufs", sizeof(struct rte_mbuf *) * + (1 << mq->log2_ring_size), 0); + if (mq->buffers == NULL) + return -ENOMEM; + } } for (i = 0; i < pmd->run.num_m2s_rings; i++) { mq = dev->data->rx_queues[i]; - mq->ring = memif_get_ring(pmd, MEMIF_RING_M2S, i); mq->log2_ring_size = pmd->run.log2_ring_size; /* queues located only in region 0 */ mq->region = 0; - mq->ring_offset = (uint8_t *)mq->ring - (uint8_t *)pmd->regions[0]->addr; + mq->ring_offset = memif_get_ring_offset(dev, mq, MEMIF_RING_M2S, i); mq->last_head = 0; mq->last_tail = 0; mq->intr_handle.fd = eventfd(0, EFD_NONBLOCK); if (mq->intr_handle.fd < 0) { MIF_LOG(WARNING, - "%s: Failed to create eventfd for rx queue %d: %s.", - rte_vdev_device_name(pmd->vdev), i, + "Failed to create eventfd for rx queue %d: %s.", i, strerror(errno)); } + mq->buffers = NULL; + if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) { + mq->buffers = rte_zmalloc("bufs", sizeof(struct rte_mbuf *) * + (1 << mq->log2_ring_size), 0); + if (mq->buffers == NULL) + return -ENOMEM; + } } + return 0; } int @@ -551,13 +1086,15 @@ memif_init_regions_and_queues(struct rte_eth_dev *dev) { int ret; - ret = memif_regions_init(dev->data->dev_private); + ret = memif_regions_init(dev); if (ret < 0) return ret; memif_init_rings(dev); - memif_init_queues(dev); + ret = memif_init_queues(dev); + if (ret < 0) + return ret; return 0; } @@ -566,12 +1103,14 @@ int memif_connect(struct rte_eth_dev *dev) { struct pmd_internals *pmd = dev->data->dev_private; + struct pmd_process_private *proc_private = dev->process_private; struct memif_region *mr; struct memif_queue *mq; + memif_ring_t *ring; int i; - for (i = 0; i < pmd->regions_num; i++) { - mr = pmd->regions[i]; + for (i = 0; i < proc_private->regions_num; i++) { + mr = proc_private->regions[i]; if (mr != NULL) { if (mr->addr == NULL) { if (mr->fd < 0) @@ -579,53 +1118,59 @@ memif_connect(struct rte_eth_dev *dev) mr->addr = mmap(NULL, mr->region_size, PROT_READ | PROT_WRITE, MAP_SHARED, mr->fd, 0); - if (mr->addr == NULL) + if (mr->addr == MAP_FAILED) { + MIF_LOG(ERR, "mmap failed: %s\n", + strerror(errno)); return -1; + } + } + if (i > 0 && (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY)) { + /* close memseg file */ + close(mr->fd); + mr->fd = -1; } } } - for (i = 0; i < pmd->run.num_s2m_rings; i++) { - mq = (pmd->role == MEMIF_ROLE_SLAVE) ? - dev->data->tx_queues[i] : dev->data->rx_queues[i]; - mq->ring = (memif_ring_t *)((uint8_t *)pmd->regions[mq->region]->addr + - mq->ring_offset); - if (mq->ring->cookie != MEMIF_COOKIE) { - MIF_LOG(ERR, "%s: Wrong cookie", - rte_vdev_device_name(pmd->vdev)); - return -1; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + for (i = 0; i < pmd->run.num_s2m_rings; i++) { + mq = (pmd->role == MEMIF_ROLE_SLAVE) ? + dev->data->tx_queues[i] : dev->data->rx_queues[i]; + ring = memif_get_ring_from_queue(proc_private, mq); + if (ring == NULL || ring->cookie != MEMIF_COOKIE) { + MIF_LOG(ERR, "Wrong ring"); + return -1; + } + __atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED); + __atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED); + mq->last_head = 0; + mq->last_tail = 0; + /* enable polling mode */ + if (pmd->role == MEMIF_ROLE_MASTER) + ring->flags = MEMIF_RING_FLAG_MASK_INT; } - mq->ring->head = 0; - mq->ring->tail = 0; - mq->last_head = 0; - mq->last_tail = 0; - /* enable polling mode */ - if (pmd->role == MEMIF_ROLE_MASTER) - mq->ring->flags = MEMIF_RING_FLAG_MASK_INT; - } - for (i = 0; i < pmd->run.num_m2s_rings; i++) { - mq = (pmd->role == MEMIF_ROLE_SLAVE) ? - dev->data->rx_queues[i] : dev->data->tx_queues[i]; - mq->ring = (memif_ring_t *)((uint8_t *)pmd->regions[mq->region]->addr + - mq->ring_offset); - if (mq->ring->cookie != MEMIF_COOKIE) { - MIF_LOG(ERR, "%s: Wrong cookie", - rte_vdev_device_name(pmd->vdev)); - return -1; + for (i = 0; i < pmd->run.num_m2s_rings; i++) { + mq = (pmd->role == MEMIF_ROLE_SLAVE) ? + dev->data->rx_queues[i] : dev->data->tx_queues[i]; + ring = memif_get_ring_from_queue(proc_private, mq); + if (ring == NULL || ring->cookie != MEMIF_COOKIE) { + MIF_LOG(ERR, "Wrong ring"); + return -1; + } + __atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED); + __atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED); + mq->last_head = 0; + mq->last_tail = 0; + /* enable polling mode */ + if (pmd->role == MEMIF_ROLE_SLAVE) + ring->flags = MEMIF_RING_FLAG_MASK_INT; } - mq->ring->head = 0; - mq->ring->tail = 0; - mq->last_head = 0; - mq->last_tail = 0; - /* enable polling mode */ - if (pmd->role == MEMIF_ROLE_SLAVE) - mq->ring->flags = MEMIF_RING_FLAG_MASK_INT; - } - pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING; - pmd->flags |= ETH_MEMIF_FLAG_CONNECTED; - dev->data->dev_link.link_status = ETH_LINK_UP; - MIF_LOG(INFO, "%s: Connected.", rte_vdev_device_name(pmd->vdev)); + pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING; + pmd->flags |= ETH_MEMIF_FLAG_CONNECTED; + dev->data->dev_link.link_status = ETH_LINK_UP; + } + MIF_LOG(INFO, "Connected."); return 0; } @@ -643,8 +1188,7 @@ memif_dev_start(struct rte_eth_dev *dev) ret = memif_connect_master(dev); break; default: - MIF_LOG(ERR, "%s: Unknown role: %d.", - rte_vdev_device_name(pmd->vdev), pmd->role); + MIF_LOG(ERR, "Unknown role: %d.", pmd->role); ret = -1; break; } @@ -658,15 +1202,21 @@ memif_dev_close(struct rte_eth_dev *dev) struct pmd_internals *pmd = dev->data->dev_private; int i; - memif_msg_enq_disconnect(pmd->cc, "Device closed", 0); - memif_disconnect(dev); + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + memif_msg_enq_disconnect(pmd->cc, "Device closed", 0); + memif_disconnect(dev); + + for (i = 0; i < dev->data->nb_rx_queues; i++) + (*dev->dev_ops->rx_queue_release)(dev->data->rx_queues[i]); + for (i = 0; i < dev->data->nb_tx_queues; i++) + (*dev->dev_ops->tx_queue_release)(dev->data->tx_queues[i]); - for (i = 0; i < dev->data->nb_rx_queues; i++) - (*dev->dev_ops->rx_queue_release)(dev->data->rx_queues[i]); - for (i = 0; i < dev->data->nb_tx_queues; i++) - (*dev->dev_ops->tx_queue_release)(dev->data->tx_queues[i]); + memif_socket_remove_device(dev); + } else { + memif_disconnect(dev); + } - memif_socket_remove_device(dev); + rte_free(dev->process_private); } static int @@ -703,8 +1253,7 @@ memif_tx_queue_setup(struct rte_eth_dev *dev, mq = rte_zmalloc("tx-queue", sizeof(struct memif_queue), 0); if (mq == NULL) { - MIF_LOG(ERR, "%s: Failed to allocate tx queue id: %u", - rte_vdev_device_name(pmd->vdev), qid); + MIF_LOG(ERR, "Failed to allocate tx queue id: %u", qid); return -ENOMEM; } @@ -712,10 +1261,9 @@ memif_tx_queue_setup(struct rte_eth_dev *dev, (pmd->role == MEMIF_ROLE_SLAVE) ? MEMIF_RING_S2M : MEMIF_RING_M2S; mq->n_pkts = 0; mq->n_bytes = 0; - mq->n_err = 0; mq->intr_handle.fd = -1; mq->intr_handle.type = RTE_INTR_HANDLE_EXT; - mq->pmd = pmd; + mq->in_port = dev->data->port_id; dev->data->tx_queues[qid] = mq; return 0; @@ -734,20 +1282,17 @@ memif_rx_queue_setup(struct rte_eth_dev *dev, mq = rte_zmalloc("rx-queue", sizeof(struct memif_queue), 0); if (mq == NULL) { - MIF_LOG(ERR, "%s: Failed to allocate rx queue id: %u", - rte_vdev_device_name(pmd->vdev), qid); + MIF_LOG(ERR, "Failed to allocate rx queue id: %u", qid); return -ENOMEM; } mq->type = (pmd->role == MEMIF_ROLE_SLAVE) ? MEMIF_RING_M2S : MEMIF_RING_S2M; mq->n_pkts = 0; mq->n_bytes = 0; - mq->n_err = 0; mq->intr_handle.fd = -1; mq->intr_handle.type = RTE_INTR_HANDLE_EXT; mq->mempool = mb_pool; mq->in_port = dev->data->port_id; - mq->pmd = pmd; dev->data->rx_queues[qid] = mq; return 0; @@ -765,9 +1310,21 @@ memif_queue_release(void *queue) } static int -memif_link_update(struct rte_eth_dev *dev __rte_unused, +memif_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) { + struct pmd_process_private *proc_private; + + if (rte_eal_process_type() == RTE_PROC_SECONDARY) { + proc_private = dev->process_private; + if (dev->data->dev_link.link_status == ETH_LINK_UP && + proc_private->regions_num == 0) { + memif_mp_request_regions(dev); + } else if (dev->data->dev_link.link_status == ETH_LINK_DOWN && + proc_private->regions_num > 0) { + memif_free_regions(dev); + } + } return 0; } @@ -783,7 +1340,6 @@ memif_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->ibytes = 0; stats->opackets = 0; stats->obytes = 0; - stats->oerrors = 0; tmp = (pmd->role == MEMIF_ROLE_SLAVE) ? pmd->run.num_s2m_rings : pmd->run.num_m2s_rings; @@ -811,12 +1367,11 @@ memif_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->q_obytes[i] = mq->n_bytes; stats->opackets += mq->n_pkts; stats->obytes += mq->n_bytes; - stats->oerrors += mq->n_err; } return 0; } -static void +static int memif_stats_reset(struct rte_eth_dev *dev) { struct pmd_internals *pmd = dev->data->dev_private; @@ -828,24 +1383,22 @@ memif_stats_reset(struct rte_eth_dev *dev) dev->data->rx_queues[i]; mq->n_pkts = 0; mq->n_bytes = 0; - mq->n_err = 0; } for (i = 0; i < pmd->run.num_m2s_rings; i++) { mq = (pmd->role == MEMIF_ROLE_SLAVE) ? dev->data->rx_queues[i] : dev->data->tx_queues[i]; mq->n_pkts = 0; mq->n_bytes = 0; - mq->n_err = 0; } + + return 0; } static int -memif_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t qid __rte_unused) +memif_rx_queue_intr_enable(struct rte_eth_dev *dev __rte_unused, + uint16_t qid __rte_unused) { - struct pmd_internals *pmd = dev->data->dev_private; - - MIF_LOG(WARNING, "%s: Interrupt mode not supported.", - rte_vdev_device_name(pmd->vdev)); + MIF_LOG(WARNING, "Interrupt mode not supported."); return -1; } @@ -886,28 +1439,36 @@ memif_create(struct rte_vdev_device *vdev, enum memif_role_t role, struct rte_eth_dev *eth_dev; struct rte_eth_dev_data *data; struct pmd_internals *pmd; + struct pmd_process_private *process_private; const unsigned int numa_node = vdev->device.numa_node; const char *name = rte_vdev_device_name(vdev); - if (flags & ETH_MEMIF_FLAG_ZERO_COPY) { - MIF_LOG(ERR, "Zero-copy slave not supported."); - return -1; - } - eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*pmd)); if (eth_dev == NULL) { MIF_LOG(ERR, "%s: Unable to allocate device struct.", name); return -1; } + process_private = (struct pmd_process_private *) + rte_zmalloc(name, sizeof(struct pmd_process_private), + RTE_CACHE_LINE_SIZE); + + if (process_private == NULL) { + MIF_LOG(ERR, "Failed to alloc memory for process private"); + return -1; + } + eth_dev->process_private = process_private; + pmd = eth_dev->data->dev_private; memset(pmd, 0, sizeof(*pmd)); - pmd->vdev = vdev; pmd->id = id; pmd->flags = flags; pmd->flags |= ETH_MEMIF_FLAG_DISABLED; pmd->role = role; + /* Zero-copy flag irelevant to master. */ + if (pmd->role == MEMIF_ROLE_MASTER) + pmd->flags &= ~ETH_MEMIF_FLAG_ZERO_COPY; ret = memif_socket_init(eth_dev, socket_filename); if (ret < 0) @@ -931,10 +1492,16 @@ memif_create(struct rte_vdev_device *vdev, enum memif_role_t role, eth_dev->dev_ops = &ops; eth_dev->device = &vdev->device; - eth_dev->rx_pkt_burst = eth_memif_rx; - eth_dev->tx_pkt_burst = eth_memif_tx; + if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) { + eth_dev->rx_pkt_burst = eth_memif_rx_zc; + eth_dev->tx_pkt_burst = eth_memif_tx_zc; + } else { + eth_dev->rx_pkt_burst = eth_memif_rx; + eth_dev->tx_pkt_burst = eth_memif_tx; + } + - eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; + eth_dev->data->dev_flags &= RTE_ETH_DEV_CLOSE_REMOVE; rte_eth_dev_probing_finish(eth_dev); @@ -964,6 +1531,10 @@ memif_set_zc(const char *key __rte_unused, const char *value, void *extra_args) uint32_t *flags = (uint32_t *)extra_args; if (strstr(value, "yes") != NULL) { + if (!rte_mcfg_get_single_file_segments()) { + MIF_LOG(ERR, "Zero-copy doesn't support multi-file segments."); + return -ENOTSUP; + } *flags |= ETH_MEMIF_FLAG_ZERO_COPY; } else if (strstr(value, "no") != NULL) { *flags &= ~ETH_MEMIF_FLAG_ZERO_COPY; @@ -1024,6 +1595,11 @@ memif_check_socket_filename(const char *filename) uint32_t idx; int ret = 0; + if (strlen(filename) >= MEMIF_SOCKET_UN_SIZE) { + MIF_LOG(ERR, "Unix socket address too long (max 108)."); + return -1; + } + tmp = strrchr(filename, '/'); if (tmp != NULL) { idx = tmp - filename; @@ -1061,13 +1637,8 @@ static int memif_set_mac(const char *key __rte_unused, const char *value, void *extra_args) { struct rte_ether_addr *ether_addr = (struct rte_ether_addr *)extra_args; - int ret = 0; - ret = sscanf(value, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", - ðer_addr->addr_bytes[0], ðer_addr->addr_bytes[1], - ðer_addr->addr_bytes[2], ðer_addr->addr_bytes[3], - ðer_addr->addr_bytes[4], ðer_addr->addr_bytes[5]); - if (ret != 6) + if (rte_ether_unformat_addr(value, ether_addr) < 0) MIF_LOG(WARNING, "Failed to parse mac '%s'.", value); return 0; } @@ -1098,22 +1669,53 @@ rte_pmd_memif_probe(struct rte_vdev_device *vdev) const char *secret = NULL; struct rte_ether_addr *ether_addr = rte_zmalloc("", sizeof(struct rte_ether_addr), 0); + struct rte_eth_dev *eth_dev; rte_eth_random_addr(ether_addr->addr_bytes); MIF_LOG(INFO, "Initialize MEMIF: %s.", name); if (rte_eal_process_type() == RTE_PROC_SECONDARY) { - MIF_LOG(ERR, "Multi-processing not supported for memif."); - /* TODO: - * Request connection information. - * - * Once memif in the primary process is connected, - * broadcast connection information. - */ - return -1; + eth_dev = rte_eth_dev_attach_secondary(name); + if (!eth_dev) { + MIF_LOG(ERR, "Failed to probe %s", name); + return -1; + } + + eth_dev->dev_ops = &ops; + eth_dev->device = &vdev->device; + eth_dev->rx_pkt_burst = eth_memif_rx; + eth_dev->tx_pkt_burst = eth_memif_tx; + + if (!rte_eal_primary_proc_alive(NULL)) { + MIF_LOG(ERR, "Primary process is missing"); + return -1; + } + + eth_dev->process_private = (struct pmd_process_private *) + rte_zmalloc(name, + sizeof(struct pmd_process_private), + RTE_CACHE_LINE_SIZE); + if (eth_dev->process_private == NULL) { + MIF_LOG(ERR, + "Failed to alloc memory for process private"); + return -1; + } + + rte_eth_dev_probing_finish(eth_dev); + + return 0; } + ret = rte_mp_action_register(MEMIF_MP_SEND_REGION, memif_mp_send_region); + /* + * Primary process can continue probing, but secondary process won't + * be able to get memory regions information + */ + if (ret < 0 && rte_errno != EEXIST) + MIF_LOG(WARNING, "Failed to register mp action callback: %s", + strerror(rte_errno)); + kvlist = rte_kvargs_parse(rte_vdev_device_args(vdev), valid_arguments); /* parse parameters */