drivers/net/memif/rte_eth_memif.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/un.h>
  11 #include <sys/ioctl.h>
  12 #include <sys/mman.h>
  13 #include <linux/if_ether.h>
  14 #include <errno.h>
  15 #include <sys/eventfd.h>
  16
  17 #include <rte_version.h>
  18 #include <rte_mbuf.h>
  19 #include <rte_ether.h>
  20 #include <rte_ethdev_driver.h>
  21 #include <rte_ethdev_vdev.h>
  22 #include <rte_malloc.h>
  23 #include <rte_kvargs.h>
  24 #include <rte_bus_vdev.h>
  25 #include <rte_string_fns.h>
  26 #include <rte_errno.h>
  27 #include <rte_memory.h>
  28 #include <rte_memzone.h>
  29 #include <rte_eal_memconfig.h>
  30
  31 #include "rte_eth_memif.h"
  32 #include "memif_socket.h"
  33
  34 #define ETH_MEMIF_ID_ARG                "id"
  35 #define ETH_MEMIF_ROLE_ARG              "role"
  36 #define ETH_MEMIF_PKT_BUFFER_SIZE_ARG   "bsize"
  37 #define ETH_MEMIF_RING_SIZE_ARG         "rsize"
  38 #define ETH_MEMIF_SOCKET_ARG            "socket"
  39 #define ETH_MEMIF_MAC_ARG               "mac"
  40 #define ETH_MEMIF_ZC_ARG                "zero-copy"
  41 #define ETH_MEMIF_SECRET_ARG            "secret"
  42
  43 static const char * const valid_arguments[] = {
  44         ETH_MEMIF_ID_ARG,
  45         ETH_MEMIF_ROLE_ARG,
  46         ETH_MEMIF_PKT_BUFFER_SIZE_ARG,
  47         ETH_MEMIF_RING_SIZE_ARG,
  48         ETH_MEMIF_SOCKET_ARG,
  49         ETH_MEMIF_MAC_ARG,
  50         ETH_MEMIF_ZC_ARG,
  51         ETH_MEMIF_SECRET_ARG,
  52         NULL
  53 };
  54
  55 static const struct rte_eth_link pmd_link = {
  56         .link_speed = ETH_SPEED_NUM_10G,
  57         .link_duplex = ETH_LINK_FULL_DUPLEX,
  58         .link_status = ETH_LINK_DOWN,
  59         .link_autoneg = ETH_LINK_AUTONEG
  60 };
  61
  62 #define MEMIF_MP_SEND_REGION            "memif_mp_send_region"
  63
  64
  65 static int memif_region_init_zc(const struct rte_memseg_list *msl,
  66                                 const struct rte_memseg *ms, void *arg);
  67
  68 const char *
  69 memif_version(void)
  70 {
  71         return ("memif-" RTE_STR(MEMIF_VERSION_MAJOR) "." RTE_STR(MEMIF_VERSION_MINOR));
  72 }
  73
  74 /* Message header to synchronize regions */
  75 struct mp_region_msg {
  76         char port_name[RTE_DEV_NAME_MAX_LEN];
  77         memif_region_index_t idx;
  78         memif_region_size_t size;
  79 };
  80
  81 static int
  82 memif_mp_send_region(const struct rte_mp_msg *msg, const void *peer)
  83 {
  84         struct rte_eth_dev *dev;
  85         struct pmd_process_private *proc_private;
  86         const struct mp_region_msg *msg_param = (const struct mp_region_msg *)msg->param;
  87         struct rte_mp_msg reply;
  88         struct mp_region_msg *reply_param = (struct mp_region_msg *)reply.param;
  89         uint16_t port_id;
  90         int ret;
  91
  92         /* Get requested port */
  93         ret = rte_eth_dev_get_port_by_name(msg_param->port_name, &port_id);
  94         if (ret) {
  95                 MIF_LOG(ERR, "Failed to get port id for %s",
  96                         msg_param->port_name);
  97                 return -1;
  98         }
  99         dev = &rte_eth_devices[port_id];
 100         proc_private = dev->process_private;
 101
 102         memset(&reply, 0, sizeof(reply));
 103         strlcpy(reply.name, msg->name, sizeof(reply.name));
 104         reply_param->idx = msg_param->idx;
 105         if (proc_private->regions[msg_param->idx] != NULL) {
 106                 reply_param->size = proc_private->regions[msg_param->idx]->region_size;
 107                 reply.fds[0] = proc_private->regions[msg_param->idx]->fd;
 108                 reply.num_fds = 1;
 109         }
 110         reply.len_param = sizeof(*reply_param);
 111         if (rte_mp_reply(&reply, peer) < 0) {
 112                 MIF_LOG(ERR, "Failed to reply to an add region request");
 113                 return -1;
 114         }
 115
 116         return 0;
 117 }
 118
 119 /*
 120  * Request regions
 121  * Called by secondary process, when ports link status goes up.
 122  */
 123 static int
 124 memif_mp_request_regions(struct rte_eth_dev *dev)
 125 {
 126         int ret, i;
 127         struct timespec timeout = {.tv_sec = 5, .tv_nsec = 0};
 128         struct rte_mp_msg msg, *reply;
 129         struct rte_mp_reply replies;
 130         struct mp_region_msg *msg_param = (struct mp_region_msg *)msg.param;
 131         struct mp_region_msg *reply_param;
 132         struct memif_region *r;
 133         struct pmd_process_private *proc_private = dev->process_private;
 134         struct pmd_internals *pmd = dev->data->dev_private;
 135         /* in case of zero-copy slave, only request region 0 */
 136         uint16_t max_region_num = (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) ?
 137                                    1 : ETH_MEMIF_MAX_REGION_NUM;
 138
 139         MIF_LOG(DEBUG, "Requesting memory regions");
 140
 141         for (i = 0; i < max_region_num; i++) {
 142                 /* Prepare the message */
 143                 memset(&msg, 0, sizeof(msg));
 144                 strlcpy(msg.name, MEMIF_MP_SEND_REGION, sizeof(msg.name));
 145                 strlcpy(msg_param->port_name, dev->data->name,
 146                         sizeof(msg_param->port_name));
 147                 msg_param->idx = i;
 148                 msg.len_param = sizeof(*msg_param);
 149
 150                 /* Send message */
 151                 ret = rte_mp_request_sync(&msg, &replies, &timeout);
 152                 if (ret < 0 || replies.nb_received != 1) {
 153                         MIF_LOG(ERR, "Failed to send mp msg: %d",
 154                                 rte_errno);
 155                         return -1;
 156                 }
 157
 158                 reply = &replies.msgs[0];
 159                 reply_param = (struct mp_region_msg *)reply->param;
 160
 161                 if (reply_param->size > 0) {
 162                         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 163                         if (r == NULL) {
 164                                 MIF_LOG(ERR, "Failed to alloc memif region.");
 165                                 free(reply);
 166                                 return -ENOMEM;
 167                         }
 168                         r->region_size = reply_param->size;
 169                         if (reply->num_fds < 1) {
 170                                 MIF_LOG(ERR, "Missing file descriptor.");
 171                                 free(reply);
 172                                 return -1;
 173                         }
 174                         r->fd = reply->fds[0];
 175                         r->addr = NULL;
 176
 177                         proc_private->regions[reply_param->idx] = r;
 178                         proc_private->regions_num++;
 179                 }
 180                 free(reply);
 181         }
 182
 183         if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) {
 184                 ret = rte_memseg_walk(memif_region_init_zc, (void *)proc_private);
 185                 if (ret < 0)
 186                         return ret;
 187         }
 188
 189         return memif_connect(dev);
 190 }
 191
 192 static int
 193 memif_dev_info(struct rte_eth_dev *dev __rte_unused, struct rte_eth_dev_info *dev_info)
 194 {
 195         dev_info->max_mac_addrs = 1;
 196         dev_info->max_rx_pktlen = (uint32_t)ETH_FRAME_LEN;
 197         dev_info->max_rx_queues = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 198         dev_info->max_tx_queues = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 199         dev_info->min_rx_bufsize = 0;
 200
 201         return 0;
 202 }
 203
 204 static memif_ring_t *
 205 memif_get_ring(struct pmd_internals *pmd, struct pmd_process_private *proc_private,
 206                memif_ring_type_t type, uint16_t ring_num)
 207 {
 208         /* rings only in region 0 */
 209         void *p = proc_private->regions[0]->addr;
 210         int ring_size = sizeof(memif_ring_t) + sizeof(memif_desc_t) *
 211             (1 << pmd->run.log2_ring_size);
 212
 213         p = (uint8_t *)p + (ring_num + type * pmd->run.num_s2m_rings) * ring_size;
 214
 215         return (memif_ring_t *)p;
 216 }
 217
 218 static memif_region_offset_t
 219 memif_get_ring_offset(struct rte_eth_dev *dev, struct memif_queue *mq,
 220                       memif_ring_type_t type, uint16_t num)
 221 {
 222         struct pmd_internals *pmd = dev->data->dev_private;
 223         struct pmd_process_private *proc_private = dev->process_private;
 224
 225         return ((uint8_t *)memif_get_ring(pmd, proc_private, type, num) -
 226                 (uint8_t *)proc_private->regions[mq->region]->addr);
 227 }
 228
 229 static memif_ring_t *
 230 memif_get_ring_from_queue(struct pmd_process_private *proc_private,
 231                           struct memif_queue *mq)
 232 {
 233         struct memif_region *r;
 234
 235         r = proc_private->regions[mq->region];
 236         if (r == NULL)
 237                 return NULL;
 238
 239         return (memif_ring_t *)((uint8_t *)r->addr + mq->ring_offset);
 240 }
 241
 242 static void *
 243 memif_get_buffer(struct pmd_process_private *proc_private, memif_desc_t *d)
 244 {
 245         return ((uint8_t *)proc_private->regions[d->region]->addr + d->offset);
 246 }
 247
 248 /* Free mbufs received by master */
 249 static void
 250 memif_free_stored_mbufs(struct pmd_process_private *proc_private, struct memif_queue *mq)
 251 {
 252         uint16_t cur_tail;
 253         uint16_t mask = (1 << mq->log2_ring_size) - 1;
 254         memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq);
 255
 256         /* FIXME: improve performance */
 257         /* The ring->tail acts as a guard variable between Tx and Rx
 258          * threads, so using load-acquire pairs with store-release
 259          * in function eth_memif_rx for S2M queues.
 260          */
 261         cur_tail = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE);
 262         while (mq->last_tail != cur_tail) {
 263                 RTE_MBUF_PREFETCH_TO_FREE(mq->buffers[(mq->last_tail + 1) & mask]);
 264                 /* Decrement refcnt and free mbuf. (current segment) */
 265                 rte_mbuf_refcnt_update(mq->buffers[mq->last_tail & mask], -1);
 266                 rte_pktmbuf_free_seg(mq->buffers[mq->last_tail & mask]);
 267                 mq->last_tail++;
 268         }
 269 }
 270
 271 static int
 272 memif_pktmbuf_chain(struct rte_mbuf *head, struct rte_mbuf *cur_tail,
 273                     struct rte_mbuf *tail)
 274 {
 275         /* Check for number-of-segments-overflow */
 276         if (unlikely(head->nb_segs + tail->nb_segs > RTE_MBUF_MAX_NB_SEGS))
 277                 return -EOVERFLOW;
 278
 279         /* Chain 'tail' onto the old tail */
 280         cur_tail->next = tail;
 281
 282         /* accumulate number of segments and total length. */
 283         head->nb_segs = (uint16_t)(head->nb_segs + tail->nb_segs);
 284
 285         tail->pkt_len = tail->data_len;
 286         head->pkt_len += tail->pkt_len;
 287
 288         return 0;
 289 }
 290
 291 static uint16_t
 292 eth_memif_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 293 {
 294         struct memif_queue *mq = queue;
 295         struct pmd_internals *pmd = rte_eth_devices[mq->in_port].data->dev_private;
 296         struct pmd_process_private *proc_private =
 297                 rte_eth_devices[mq->in_port].process_private;
 298         memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq);
 299         uint16_t cur_slot, last_slot, n_slots, ring_size, mask, s0;
 300         uint16_t n_rx_pkts = 0;
 301         uint16_t mbuf_size = rte_pktmbuf_data_room_size(mq->mempool) -
 302                 RTE_PKTMBUF_HEADROOM;
 303         uint16_t src_len, src_off, dst_len, dst_off, cp_len;
 304         memif_ring_type_t type = mq->type;
 305         memif_desc_t *d0;
 306         struct rte_mbuf *mbuf, *mbuf_head, *mbuf_tail;
 307         uint64_t b;
 308         ssize_t size __rte_unused;
 309         uint16_t head;
 310         int ret;
 311         struct rte_eth_link link;
 312
 313         if (unlikely((pmd->flags & ETH_MEMIF_FLAG_CONNECTED) == 0))
 314                 return 0;
 315         if (unlikely(ring == NULL)) {
 316                 /* Secondary process will attempt to request regions. */
 317                 ret = rte_eth_link_get(mq->in_port, &link);
 318                 if (ret < 0)
 319                         MIF_LOG(ERR, "Failed to get port %u link info: %s",
 320                                 mq->in_port, rte_strerror(-ret));
 321                 return 0;
 322         }
 323
 324         /* consume interrupt */
 325         if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0)
 326                 size = read(mq->intr_handle.fd, &b, sizeof(b));
 327
 328         ring_size = 1 << mq->log2_ring_size;
 329         mask = ring_size - 1;
 330
 331         if (type == MEMIF_RING_S2M) {
 332                 cur_slot = mq->last_head;
 333                 last_slot = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE);
 334         } else {
 335                 cur_slot = mq->last_tail;
 336                 last_slot = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE);
 337         }
 338
 339         if (cur_slot == last_slot)
 340                 goto refill;
 341         n_slots = last_slot - cur_slot;
 342
 343         while (n_slots && n_rx_pkts < nb_pkts) {
 344                 mbuf_head = rte_pktmbuf_alloc(mq->mempool);
 345                 if (unlikely(mbuf_head == NULL))
 346                         goto no_free_bufs;
 347                 mbuf = mbuf_head;
 348                 mbuf->port = mq->in_port;
 349
 350 next_slot:
 351                 s0 = cur_slot & mask;
 352                 d0 = &ring->desc[s0];
 353
 354                 src_len = d0->length;
 355                 dst_off = 0;
 356                 src_off = 0;
 357
 358                 do {
 359                         dst_len = mbuf_size - dst_off;
 360                         if (dst_len == 0) {
 361                                 dst_off = 0;
 362                                 dst_len = mbuf_size;
 363
 364                                 /* store pointer to tail */
 365                                 mbuf_tail = mbuf;
 366                                 mbuf = rte_pktmbuf_alloc(mq->mempool);
 367                                 if (unlikely(mbuf == NULL))
 368                                         goto no_free_bufs;
 369                                 mbuf->port = mq->in_port;
 370                                 ret = memif_pktmbuf_chain(mbuf_head, mbuf_tail, mbuf);
 371                                 if (unlikely(ret < 0)) {
 372                                         MIF_LOG(ERR, "number-of-segments-overflow");
 373                                         rte_pktmbuf_free(mbuf);
 374                                         goto no_free_bufs;
 375                                 }
 376                         }
 377                         cp_len = RTE_MIN(dst_len, src_len);
 378
 379                         rte_pktmbuf_data_len(mbuf) += cp_len;
 380                         rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf);
 381                         if (mbuf != mbuf_head)
 382                                 rte_pktmbuf_pkt_len(mbuf_head) += cp_len;
 383
 384                         memcpy(rte_pktmbuf_mtod_offset(mbuf, void *, dst_off),
 385                                (uint8_t *)memif_get_buffer(proc_private, d0) + src_off,
 386                                cp_len);
 387
 388                         src_off += cp_len;
 389                         dst_off += cp_len;
 390                         src_len -= cp_len;
 391                 } while (src_len);
 392
 393                 cur_slot++;
 394                 n_slots--;
 395
 396                 if (d0->flags & MEMIF_DESC_FLAG_NEXT)
 397                         goto next_slot;
 398
 399                 mq->n_bytes += rte_pktmbuf_pkt_len(mbuf_head);
 400                 *bufs++ = mbuf_head;
 401                 n_rx_pkts++;
 402         }
 403
 404 no_free_bufs:
 405         if (type == MEMIF_RING_S2M) {
 406                 __atomic_store_n(&ring->tail, cur_slot, __ATOMIC_RELEASE);
 407                 mq->last_head = cur_slot;
 408         } else {
 409                 mq->last_tail = cur_slot;
 410         }
 411
 412 refill:
 413         if (type == MEMIF_RING_M2S) {
 414                 /* ring->head is updated by the receiver and this function
 415                  * is called in the context of receiver thread. The loads in
 416                  * the receiver do not need to synchronize with its own stores.
 417                  */
 418                 head = __atomic_load_n(&ring->head, __ATOMIC_RELAXED);
 419                 n_slots = ring_size - head + mq->last_tail;
 420
 421                 while (n_slots--) {
 422                         s0 = head++ & mask;
 423                         d0 = &ring->desc[s0];
 424                         d0->length = pmd->run.pkt_buffer_size;
 425                 }
 426                 __atomic_store_n(&ring->head, head, __ATOMIC_RELEASE);
 427         }
 428
 429         mq->n_pkts += n_rx_pkts;
 430         return n_rx_pkts;
 431 }
 432
 433 static uint16_t
 434 eth_memif_rx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 435 {
 436         struct memif_queue *mq = queue;
 437         struct pmd_internals *pmd = rte_eth_devices[mq->in_port].data->dev_private;
 438         struct pmd_process_private *proc_private =
 439                 rte_eth_devices[mq->in_port].process_private;
 440         memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq);
 441         uint16_t cur_slot, last_slot, n_slots, ring_size, mask, s0, head;
 442         uint16_t n_rx_pkts = 0;
 443         memif_desc_t *d0;
 444         struct rte_mbuf *mbuf, *mbuf_tail;
 445         struct rte_mbuf *mbuf_head = NULL;
 446         int ret;
 447         struct rte_eth_link link;
 448
 449         if (unlikely((pmd->flags & ETH_MEMIF_FLAG_CONNECTED) == 0))
 450                 return 0;
 451         if (unlikely(ring == NULL)) {
 452                 /* Secondary process will attempt to request regions. */
 453                 rte_eth_link_get(mq->in_port, &link);
 454                 return 0;
 455         }
 456
 457         /* consume interrupt */
 458         if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0) {
 459                 uint64_t b;
 460                 ssize_t size __rte_unused;
 461                 size = read(mq->intr_handle.fd, &b, sizeof(b));
 462         }
 463
 464         ring_size = 1 << mq->log2_ring_size;
 465         mask = ring_size - 1;
 466
 467         cur_slot = mq->last_tail;
 468         /* The ring->tail acts as a guard variable between Tx and Rx
 469          * threads, so using load-acquire pairs with store-release
 470          * to synchronize it between threads.
 471          */
 472         last_slot = __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE);
 473         if (cur_slot == last_slot)
 474                 goto refill;
 475         n_slots = last_slot - cur_slot;
 476
 477         while (n_slots && n_rx_pkts < nb_pkts) {
 478                 s0 = cur_slot & mask;
 479
 480                 d0 = &ring->desc[s0];
 481                 mbuf_head = mq->buffers[s0];
 482                 mbuf = mbuf_head;
 483
 484 next_slot:
 485                 /* prefetch next descriptor */
 486                 if (n_rx_pkts + 1 < nb_pkts)
 487                         rte_prefetch0(&ring->desc[(cur_slot + 1) & mask]);
 488
 489                 mbuf->port = mq->in_port;
 490                 rte_pktmbuf_data_len(mbuf) = d0->length;
 491                 rte_pktmbuf_pkt_len(mbuf) = rte_pktmbuf_data_len(mbuf);
 492
 493                 mq->n_bytes += rte_pktmbuf_data_len(mbuf);
 494
 495                 cur_slot++;
 496                 n_slots--;
 497                 if (d0->flags & MEMIF_DESC_FLAG_NEXT) {
 498                         s0 = cur_slot & mask;
 499                         d0 = &ring->desc[s0];
 500                         mbuf_tail = mbuf;
 501                         mbuf = mq->buffers[s0];
 502                         ret = memif_pktmbuf_chain(mbuf_head, mbuf_tail, mbuf);
 503                         if (unlikely(ret < 0)) {
 504                                 MIF_LOG(ERR, "number-of-segments-overflow");
 505                                 goto refill;
 506                         }
 507                         goto next_slot;
 508                 }
 509
 510                 *bufs++ = mbuf_head;
 511                 n_rx_pkts++;
 512         }
 513
 514         mq->last_tail = cur_slot;
 515
 516 /* Supply master with new buffers */
 517 refill:
 518         /* ring->head is updated by the receiver and this function
 519          * is called in the context of receiver thread. The loads in
 520          * the receiver do not need to synchronize with its own stores.
 521          */
 522         head = __atomic_load_n(&ring->head, __ATOMIC_RELAXED);
 523         n_slots = ring_size - head + mq->last_tail;
 524
 525         if (n_slots < 32)
 526                 goto no_free_mbufs;
 527
 528         ret = rte_pktmbuf_alloc_bulk(mq->mempool, &mq->buffers[head & mask], n_slots);
 529         if (unlikely(ret < 0))
 530                 goto no_free_mbufs;
 531
 532         while (n_slots--) {
 533                 s0 = head++ & mask;
 534                 if (n_slots > 0)
 535                         rte_prefetch0(mq->buffers[head & mask]);
 536                 d0 = &ring->desc[s0];
 537                 /* store buffer header */
 538                 mbuf = mq->buffers[s0];
 539                 /* populate descriptor */
 540                 d0->length = rte_pktmbuf_data_room_size(mq->mempool) -
 541                                 RTE_PKTMBUF_HEADROOM;
 542                 d0->region = 1;
 543                 d0->offset = rte_pktmbuf_mtod(mbuf, uint8_t *) -
 544                         (uint8_t *)proc_private->regions[d0->region]->addr;
 545         }
 546 no_free_mbufs:
 547         /* The ring->head acts as a guard variable between Tx and Rx
 548          * threads, so using store-release pairs with load-acquire
 549          * in function eth_memif_tx.
 550          */
 551         __atomic_store_n(&ring->head, head, __ATOMIC_RELEASE);
 552
 553         mq->n_pkts += n_rx_pkts;
 554
 555         return n_rx_pkts;
 556 }
 557
 558 static uint16_t
 559 eth_memif_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 560 {
 561         struct memif_queue *mq = queue;
 562         struct pmd_internals *pmd = rte_eth_devices[mq->in_port].data->dev_private;
 563         struct pmd_process_private *proc_private =
 564                 rte_eth_devices[mq->in_port].process_private;
 565         memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq);
 566         uint16_t slot, saved_slot, n_free, ring_size, mask, n_tx_pkts = 0;
 567         uint16_t src_len, src_off, dst_len, dst_off, cp_len;
 568         memif_ring_type_t type = mq->type;
 569         memif_desc_t *d0;
 570         struct rte_mbuf *mbuf;
 571         struct rte_mbuf *mbuf_head;
 572         uint64_t a;
 573         ssize_t size;
 574         struct rte_eth_link link;
 575
 576         if (unlikely((pmd->flags & ETH_MEMIF_FLAG_CONNECTED) == 0))
 577                 return 0;
 578         if (unlikely(ring == NULL)) {
 579                 int ret;
 580
 581                 /* Secondary process will attempt to request regions. */
 582                 ret = rte_eth_link_get(mq->in_port, &link);
 583                 if (ret < 0)
 584                         MIF_LOG(ERR, "Failed to get port %u link info: %s",
 585                                 mq->in_port, rte_strerror(-ret));
 586                 return 0;
 587         }
 588
 589         ring_size = 1 << mq->log2_ring_size;
 590         mask = ring_size - 1;
 591
 592         if (type == MEMIF_RING_S2M) {
 593                 /* For S2M queues ring->head is updated by the sender and
 594                  * this function is called in the context of sending thread.
 595                  * The loads in the sender do not need to synchronize with
 596                  * its own stores. Hence, the following load can be a
 597                  * relaxed load.
 598                  */
 599                 slot = __atomic_load_n(&ring->head, __ATOMIC_RELAXED);
 600                 n_free = ring_size - slot +
 601                                 __atomic_load_n(&ring->tail, __ATOMIC_ACQUIRE);
 602         } else {
 603                 /* For M2S queues ring->tail is updated by the sender and
 604                  * this function is called in the context of sending thread.
 605                  * The loads in the sender do not need to synchronize with
 606                  * its own stores. Hence, the following load can be a
 607                  * relaxed load.
 608                  */
 609                 slot = __atomic_load_n(&ring->tail, __ATOMIC_RELAXED);
 610                 n_free = __atomic_load_n(&ring->head, __ATOMIC_ACQUIRE) - slot;
 611         }
 612
 613         while (n_tx_pkts < nb_pkts && n_free) {
 614                 mbuf_head = *bufs++;
 615                 mbuf = mbuf_head;
 616
 617                 saved_slot = slot;
 618                 d0 = &ring->desc[slot & mask];
 619                 dst_off = 0;
 620                 dst_len = (type == MEMIF_RING_S2M) ?
 621                         pmd->run.pkt_buffer_size : d0->length;
 622
 623 next_in_chain:
 624                 src_off = 0;
 625                 src_len = rte_pktmbuf_data_len(mbuf);
 626
 627                 while (src_len) {
 628                         if (dst_len == 0) {
 629                                 if (n_free) {
 630                                         slot++;
 631                                         n_free--;
 632                                         d0->flags |= MEMIF_DESC_FLAG_NEXT;
 633                                         d0 = &ring->desc[slot & mask];
 634                                         dst_off = 0;
 635                                         dst_len = (type == MEMIF_RING_S2M) ?
 636                                             pmd->run.pkt_buffer_size : d0->length;
 637                                         d0->flags = 0;
 638                                 } else {
 639                                         slot = saved_slot;
 640                                         goto no_free_slots;
 641                                 }
 642                         }
 643                         cp_len = RTE_MIN(dst_len, src_len);
 644
 645                         memcpy((uint8_t *)memif_get_buffer(proc_private, d0) + dst_off,
 646                                rte_pktmbuf_mtod_offset(mbuf, void *, src_off),
 647                                cp_len);
 648
 649                         mq->n_bytes += cp_len;
 650                         src_off += cp_len;
 651                         dst_off += cp_len;
 652                         src_len -= cp_len;
 653                         dst_len -= cp_len;
 654
 655                         d0->length = dst_off;
 656                 }
 657
 658                 if (rte_pktmbuf_is_contiguous(mbuf) == 0) {
 659                         mbuf = mbuf->next;
 660                         goto next_in_chain;
 661                 }
 662
 663                 n_tx_pkts++;
 664                 slot++;
 665                 n_free--;
 666                 rte_pktmbuf_free(mbuf_head);
 667         }
 668
 669 no_free_slots:
 670         if (type == MEMIF_RING_S2M)
 671                 __atomic_store_n(&ring->head, slot, __ATOMIC_RELEASE);
 672         else
 673                 __atomic_store_n(&ring->tail, slot, __ATOMIC_RELEASE);
 674
 675         if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0) {
 676                 a = 1;
 677                 size = write(mq->intr_handle.fd, &a, sizeof(a));
 678                 if (unlikely(size < 0)) {
 679                         MIF_LOG(WARNING,
 680                                 "Failed to send interrupt. %s", strerror(errno));
 681                 }
 682         }
 683
 684         mq->n_pkts += n_tx_pkts;
 685         return n_tx_pkts;
 686 }
 687
 688
 689 static int
 690 memif_tx_one_zc(struct pmd_process_private *proc_private, struct memif_queue *mq,
 691                 memif_ring_t *ring, struct rte_mbuf *mbuf, const uint16_t mask,
 692                 uint16_t slot, uint16_t n_free)
 693 {
 694         memif_desc_t *d0;
 695         int used_slots = 1;
 696
 697 next_in_chain:
 698         /* store pointer to mbuf to free it later */
 699         mq->buffers[slot & mask] = mbuf;
 700         /* Increment refcnt to make sure the buffer is not freed before master
 701          * receives it. (current segment)
 702          */
 703         rte_mbuf_refcnt_update(mbuf, 1);
 704         /* populate descriptor */
 705         d0 = &ring->desc[slot & mask];
 706         d0->length = rte_pktmbuf_data_len(mbuf);
 707         /* FIXME: get region index */
 708         d0->region = 1;
 709         d0->offset = rte_pktmbuf_mtod(mbuf, uint8_t *) -
 710                 (uint8_t *)proc_private->regions[d0->region]->addr;
 711         d0->flags = 0;
 712
 713         /* check if buffer is chained */
 714         if (rte_pktmbuf_is_contiguous(mbuf) == 0) {
 715                 if (n_free < 2)
 716                         return 0;
 717                 /* mark buffer as chained */
 718                 d0->flags |= MEMIF_DESC_FLAG_NEXT;
 719                 /* advance mbuf */
 720                 mbuf = mbuf->next;
 721                 /* update counters */
 722                 used_slots++;
 723                 slot++;
 724                 n_free--;
 725                 goto next_in_chain;
 726         }
 727         return used_slots;
 728 }
 729
 730 static uint16_t
 731 eth_memif_tx_zc(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 732 {
 733         struct memif_queue *mq = queue;
 734         struct pmd_internals *pmd = rte_eth_devices[mq->in_port].data->dev_private;
 735         struct pmd_process_private *proc_private =
 736                 rte_eth_devices[mq->in_port].process_private;
 737         memif_ring_t *ring = memif_get_ring_from_queue(proc_private, mq);
 738         uint16_t slot, n_free, ring_size, mask, n_tx_pkts = 0;
 739         struct rte_eth_link link;
 740
 741         if (unlikely((pmd->flags & ETH_MEMIF_FLAG_CONNECTED) == 0))
 742                 return 0;
 743         if (unlikely(ring == NULL)) {
 744                 /* Secondary process will attempt to request regions. */
 745                 rte_eth_link_get(mq->in_port, &link);
 746                 return 0;
 747         }
 748
 749         ring_size = 1 << mq->log2_ring_size;
 750         mask = ring_size - 1;
 751
 752         /* free mbufs received by master */
 753         memif_free_stored_mbufs(proc_private, mq);
 754
 755         /* ring type always MEMIF_RING_S2M */
 756         /* For S2M queues ring->head is updated by the sender and
 757          * this function is called in the context of sending thread.
 758          * The loads in the sender do not need to synchronize with
 759          * its own stores. Hence, the following load can be a
 760          * relaxed load.
 761          */
 762         slot = __atomic_load_n(&ring->head, __ATOMIC_RELAXED);
 763         n_free = ring_size - slot + mq->last_tail;
 764
 765         int used_slots;
 766
 767         while (n_free && (n_tx_pkts < nb_pkts)) {
 768                 while ((n_free > 4) && ((nb_pkts - n_tx_pkts) > 4)) {
 769                         if ((nb_pkts - n_tx_pkts) > 8) {
 770                                 rte_prefetch0(*bufs + 4);
 771                                 rte_prefetch0(*bufs + 5);
 772                                 rte_prefetch0(*bufs + 6);
 773                                 rte_prefetch0(*bufs + 7);
 774                         }
 775                         used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++,
 776                                 mask, slot, n_free);
 777                         if (unlikely(used_slots < 1))
 778                                 goto no_free_slots;
 779                         n_tx_pkts++;
 780                         slot += used_slots;
 781                         n_free -= used_slots;
 782
 783                         used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++,
 784                                 mask, slot, n_free);
 785                         if (unlikely(used_slots < 1))
 786                                 goto no_free_slots;
 787                         n_tx_pkts++;
 788                         slot += used_slots;
 789                         n_free -= used_slots;
 790
 791                         used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++,
 792                                 mask, slot, n_free);
 793                         if (unlikely(used_slots < 1))
 794                                 goto no_free_slots;
 795                         n_tx_pkts++;
 796                         slot += used_slots;
 797                         n_free -= used_slots;
 798
 799                         used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++,
 800                                 mask, slot, n_free);
 801                         if (unlikely(used_slots < 1))
 802                                 goto no_free_slots;
 803                         n_tx_pkts++;
 804                         slot += used_slots;
 805                         n_free -= used_slots;
 806                 }
 807                 used_slots = memif_tx_one_zc(proc_private, mq, ring, *bufs++,
 808                         mask, slot, n_free);
 809                 if (unlikely(used_slots < 1))
 810                         goto no_free_slots;
 811                 n_tx_pkts++;
 812                 slot += used_slots;
 813                 n_free -= used_slots;
 814         }
 815
 816 no_free_slots:
 817         /* ring type always MEMIF_RING_S2M */
 818         /* The ring->head acts as a guard variable between Tx and Rx
 819          * threads, so using store-release pairs with load-acquire
 820          * in function eth_memif_rx for S2M rings.
 821          */
 822         __atomic_store_n(&ring->head, slot, __ATOMIC_RELEASE);
 823
 824         /* Send interrupt, if enabled. */
 825         if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0) {
 826                 uint64_t a = 1;
 827                 ssize_t size = write(mq->intr_handle.fd, &a, sizeof(a));
 828                 if (unlikely(size < 0)) {
 829                         MIF_LOG(WARNING,
 830                                 "Failed to send interrupt. %s", strerror(errno));
 831                 }
 832         }
 833
 834         /* increment queue counters */
 835         mq->n_pkts += n_tx_pkts;
 836
 837         return n_tx_pkts;
 838 }
 839
 840 void
 841 memif_free_regions(struct rte_eth_dev *dev)
 842 {
 843         struct pmd_process_private *proc_private = dev->process_private;
 844         struct pmd_internals *pmd = dev->data->dev_private;
 845         int i;
 846         struct memif_region *r;
 847
 848         /* regions are allocated contiguously, so it's
 849          * enough to loop until 'proc_private->regions_num'
 850          */
 851         for (i = 0; i < proc_private->regions_num; i++) {
 852                 r = proc_private->regions[i];
 853                 if (r != NULL) {
 854                         /* This is memzone */
 855                         if (i > 0 && (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY)) {
 856                                 r->addr = NULL;
 857                                 if (r->fd > 0)
 858                                         close(r->fd);
 859                         }
 860                         if (r->addr != NULL) {
 861                                 munmap(r->addr, r->region_size);
 862                                 if (r->fd > 0) {
 863                                         close(r->fd);
 864                                         r->fd = -1;
 865                                 }
 866                         }
 867                         rte_free(r);
 868                         proc_private->regions[i] = NULL;
 869                 }
 870         }
 871         proc_private->regions_num = 0;
 872 }
 873
 874 static int
 875 memif_region_init_zc(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 876                      void *arg)
 877 {
 878         struct pmd_process_private *proc_private = (struct pmd_process_private *)arg;
 879         struct memif_region *r;
 880
 881         if (proc_private->regions_num < 1) {
 882                 MIF_LOG(ERR, "Missing descriptor region");
 883                 return -1;
 884         }
 885
 886         r = proc_private->regions[proc_private->regions_num - 1];
 887
 888         if (r->addr != msl->base_va)
 889                 r = proc_private->regions[++proc_private->regions_num - 1];
 890
 891         if (r == NULL) {
 892                 r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 893                 if (r == NULL) {
 894                         MIF_LOG(ERR, "Failed to alloc memif region.");
 895                         return -ENOMEM;
 896                 }
 897
 898                 r->addr = msl->base_va;
 899                 r->region_size = ms->len;
 900                 r->fd = rte_memseg_get_fd(ms);
 901                 if (r->fd < 0)
 902                         return -1;
 903                 r->pkt_buffer_offset = 0;
 904
 905                 proc_private->regions[proc_private->regions_num - 1] = r;
 906         } else {
 907                 r->region_size += ms->len;
 908         }
 909
 910         return 0;
 911 }
 912
 913 static int
 914 memif_region_init_shm(struct rte_eth_dev *dev, uint8_t has_buffers)
 915 {
 916         struct pmd_internals *pmd = dev->data->dev_private;
 917         struct pmd_process_private *proc_private = dev->process_private;
 918         char shm_name[ETH_MEMIF_SHM_NAME_SIZE];
 919         int ret = 0;
 920         struct memif_region *r;
 921
 922         if (proc_private->regions_num >= ETH_MEMIF_MAX_REGION_NUM) {
 923                 MIF_LOG(ERR, "Too many regions.");
 924                 return -1;
 925         }
 926
 927         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 928         if (r == NULL) {
 929                 MIF_LOG(ERR, "Failed to alloc memif region.");
 930                 return -ENOMEM;
 931         }
 932
 933         /* calculate buffer offset */
 934         r->pkt_buffer_offset = (pmd->run.num_s2m_rings + pmd->run.num_m2s_rings) *
 935             (sizeof(memif_ring_t) + sizeof(memif_desc_t) *
 936             (1 << pmd->run.log2_ring_size));
 937
 938         r->region_size = r->pkt_buffer_offset;
 939         /* if region has buffers, add buffers size to region_size */
 940         if (has_buffers == 1)
 941                 r->region_size += (uint32_t)(pmd->run.pkt_buffer_size *
 942                         (1 << pmd->run.log2_ring_size) *
 943                         (pmd->run.num_s2m_rings +
 944                          pmd->run.num_m2s_rings));
 945
 946         memset(shm_name, 0, sizeof(char) * ETH_MEMIF_SHM_NAME_SIZE);
 947         snprintf(shm_name, ETH_MEMIF_SHM_NAME_SIZE, "memif_region_%d",
 948                  proc_private->regions_num);
 949
 950         r->fd = memfd_create(shm_name, MFD_ALLOW_SEALING);
 951         if (r->fd < 0) {
 952                 MIF_LOG(ERR, "Failed to create shm file: %s.", strerror(errno));
 953                 ret = -1;
 954                 goto error;
 955         }
 956
 957         ret = fcntl(r->fd, F_ADD_SEALS, F_SEAL_SHRINK);
 958         if (ret < 0) {
 959                 MIF_LOG(ERR, "Failed to add seals to shm file: %s.", strerror(errno));
 960                 goto error;
 961         }
 962
 963         ret = ftruncate(r->fd, r->region_size);
 964         if (ret < 0) {
 965                 MIF_LOG(ERR, "Failed to truncate shm file: %s.", strerror(errno));
 966                 goto error;
 967         }
 968
 969         r->addr = mmap(NULL, r->region_size, PROT_READ |
 970                        PROT_WRITE, MAP_SHARED, r->fd, 0);
 971         if (r->addr == MAP_FAILED) {
 972                 MIF_LOG(ERR, "Failed to mmap shm region: %s.", strerror(ret));
 973                 ret = -1;
 974                 goto error;
 975         }
 976
 977         proc_private->regions[proc_private->regions_num] = r;
 978         proc_private->regions_num++;
 979
 980         return ret;
 981
 982 error:
 983         if (r->fd > 0)
 984                 close(r->fd);
 985         r->fd = -1;
 986
 987         return ret;
 988 }
 989
 990 static int
 991 memif_regions_init(struct rte_eth_dev *dev)
 992 {
 993         struct pmd_internals *pmd = dev->data->dev_private;
 994         int ret;
 995
 996         /*
 997          * Zero-copy exposes dpdk memory.
 998          * Each memseg list will be represented by memif region.
 999          * Zero-copy regions indexing: memseg list idx + 1,
1000          * as we already have region 0 reserved for descriptors.
1001          */
1002         if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) {
1003                 /* create region idx 0 containing descriptors */
1004                 ret = memif_region_init_shm(dev, 0);
1005                 if (ret < 0)
1006                         return ret;
1007                 ret = rte_memseg_walk(memif_region_init_zc, (void *)dev->process_private);
1008                 if (ret < 0)
1009                         return ret;
1010         } else {
1011                 /* create one memory region contaning rings and buffers */
1012                 ret = memif_region_init_shm(dev, /* has buffers */ 1);
1013                 if (ret < 0)
1014                         return ret;
1015         }
1016
1017         return 0;
1018 }
1019
1020 static void
1021 memif_init_rings(struct rte_eth_dev *dev)
1022 {
1023         struct pmd_internals *pmd = dev->data->dev_private;
1024         struct pmd_process_private *proc_private = dev->process_private;
1025         memif_ring_t *ring;
1026         int i, j;
1027         uint16_t slot;
1028
1029         for (i = 0; i < pmd->run.num_s2m_rings; i++) {
1030                 ring = memif_get_ring(pmd, proc_private, MEMIF_RING_S2M, i);
1031                 __atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED);
1032                 __atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED);
1033                 ring->cookie = MEMIF_COOKIE;
1034                 ring->flags = 0;
1035
1036                 if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY)
1037                         continue;
1038
1039                 for (j = 0; j < (1 << pmd->run.log2_ring_size); j++) {
1040                         slot = i * (1 << pmd->run.log2_ring_size) + j;
1041                         ring->desc[j].region = 0;
1042                         ring->desc[j].offset =
1043                                 proc_private->regions[0]->pkt_buffer_offset +
1044                                 (uint32_t)(slot * pmd->run.pkt_buffer_size);
1045                         ring->desc[j].length = pmd->run.pkt_buffer_size;
1046                 }
1047         }
1048
1049         for (i = 0; i < pmd->run.num_m2s_rings; i++) {
1050                 ring = memif_get_ring(pmd, proc_private, MEMIF_RING_M2S, i);
1051                 __atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED);
1052                 __atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED);
1053                 ring->cookie = MEMIF_COOKIE;
1054                 ring->flags = 0;
1055
1056                 if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY)
1057                         continue;
1058
1059                 for (j = 0; j < (1 << pmd->run.log2_ring_size); j++) {
1060                         slot = (i + pmd->run.num_s2m_rings) *
1061                             (1 << pmd->run.log2_ring_size) + j;
1062                         ring->desc[j].region = 0;
1063                         ring->desc[j].offset =
1064                                 proc_private->regions[0]->pkt_buffer_offset +
1065                                 (uint32_t)(slot * pmd->run.pkt_buffer_size);
1066                         ring->desc[j].length = pmd->run.pkt_buffer_size;
1067                 }
1068         }
1069 }
1070
1071 /* called only by slave */
1072 static int
1073 memif_init_queues(struct rte_eth_dev *dev)
1074 {
1075         struct pmd_internals *pmd = dev->data->dev_private;
1076         struct memif_queue *mq;
1077         int i;
1078
1079         for (i = 0; i < pmd->run.num_s2m_rings; i++) {
1080                 mq = dev->data->tx_queues[i];
1081                 mq->log2_ring_size = pmd->run.log2_ring_size;
1082                 /* queues located only in region 0 */
1083                 mq->region = 0;
1084                 mq->ring_offset = memif_get_ring_offset(dev, mq, MEMIF_RING_S2M, i);
1085                 mq->last_head = 0;
1086                 mq->last_tail = 0;
1087                 mq->intr_handle.fd = eventfd(0, EFD_NONBLOCK);
1088                 if (mq->intr_handle.fd < 0) {
1089                         MIF_LOG(WARNING,
1090                                 "Failed to create eventfd for tx queue %d: %s.", i,
1091                                 strerror(errno));
1092                 }
1093                 mq->buffers = NULL;
1094                 if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) {
1095                         mq->buffers = rte_zmalloc("bufs", sizeof(struct rte_mbuf *) *
1096                                                   (1 << mq->log2_ring_size), 0);
1097                         if (mq->buffers == NULL)
1098                                 return -ENOMEM;
1099                 }
1100         }
1101
1102         for (i = 0; i < pmd->run.num_m2s_rings; i++) {
1103                 mq = dev->data->rx_queues[i];
1104                 mq->log2_ring_size = pmd->run.log2_ring_size;
1105                 /* queues located only in region 0 */
1106                 mq->region = 0;
1107                 mq->ring_offset = memif_get_ring_offset(dev, mq, MEMIF_RING_M2S, i);
1108                 mq->last_head = 0;
1109                 mq->last_tail = 0;
1110                 mq->intr_handle.fd = eventfd(0, EFD_NONBLOCK);
1111                 if (mq->intr_handle.fd < 0) {
1112                         MIF_LOG(WARNING,
1113                                 "Failed to create eventfd for rx queue %d: %s.", i,
1114                                 strerror(errno));
1115                 }
1116                 mq->buffers = NULL;
1117                 if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) {
1118                         mq->buffers = rte_zmalloc("bufs", sizeof(struct rte_mbuf *) *
1119                                                   (1 << mq->log2_ring_size), 0);
1120                         if (mq->buffers == NULL)
1121                                 return -ENOMEM;
1122                 }
1123         }
1124         return 0;
1125 }
1126
1127 int
1128 memif_init_regions_and_queues(struct rte_eth_dev *dev)
1129 {
1130         int ret;
1131
1132         ret = memif_regions_init(dev);
1133         if (ret < 0)
1134                 return ret;
1135
1136         memif_init_rings(dev);
1137
1138         ret = memif_init_queues(dev);
1139         if (ret < 0)
1140                 return ret;
1141
1142         return 0;
1143 }
1144
1145 int
1146 memif_connect(struct rte_eth_dev *dev)
1147 {
1148         struct pmd_internals *pmd = dev->data->dev_private;
1149         struct pmd_process_private *proc_private = dev->process_private;
1150         struct memif_region *mr;
1151         struct memif_queue *mq;
1152         memif_ring_t *ring;
1153         int i;
1154
1155         for (i = 0; i < proc_private->regions_num; i++) {
1156                 mr = proc_private->regions[i];
1157                 if (mr != NULL) {
1158                         if (mr->addr == NULL) {
1159                                 if (mr->fd < 0)
1160                                         return -1;
1161                                 mr->addr = mmap(NULL, mr->region_size,
1162                                                 PROT_READ | PROT_WRITE,
1163                                                 MAP_SHARED, mr->fd, 0);
1164                                 if (mr->addr == MAP_FAILED) {
1165                                         MIF_LOG(ERR, "mmap failed: %s\n",
1166                                                 strerror(errno));
1167                                         return -1;
1168                                 }
1169                         }
1170                         if (i > 0 && (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY)) {
1171                                 /* close memseg file */
1172                                 close(mr->fd);
1173                                 mr->fd = -1;
1174                         }
1175                 }
1176         }
1177
1178         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
1179                 for (i = 0; i < pmd->run.num_s2m_rings; i++) {
1180                         mq = (pmd->role == MEMIF_ROLE_SLAVE) ?
1181                             dev->data->tx_queues[i] : dev->data->rx_queues[i];
1182                         ring = memif_get_ring_from_queue(proc_private, mq);
1183                         if (ring == NULL || ring->cookie != MEMIF_COOKIE) {
1184                                 MIF_LOG(ERR, "Wrong ring");
1185                                 return -1;
1186                         }
1187                         __atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED);
1188                         __atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED);
1189                         mq->last_head = 0;
1190                         mq->last_tail = 0;
1191                         /* enable polling mode */
1192                         if (pmd->role == MEMIF_ROLE_MASTER)
1193                                 ring->flags = MEMIF_RING_FLAG_MASK_INT;
1194                 }
1195                 for (i = 0; i < pmd->run.num_m2s_rings; i++) {
1196                         mq = (pmd->role == MEMIF_ROLE_SLAVE) ?
1197                             dev->data->rx_queues[i] : dev->data->tx_queues[i];
1198                         ring = memif_get_ring_from_queue(proc_private, mq);
1199                         if (ring == NULL || ring->cookie != MEMIF_COOKIE) {
1200                                 MIF_LOG(ERR, "Wrong ring");
1201                                 return -1;
1202                         }
1203                         __atomic_store_n(&ring->head, 0, __ATOMIC_RELAXED);
1204                         __atomic_store_n(&ring->tail, 0, __ATOMIC_RELAXED);
1205                         mq->last_head = 0;
1206                         mq->last_tail = 0;
1207                         /* enable polling mode */
1208                         if (pmd->role == MEMIF_ROLE_SLAVE)
1209                                 ring->flags = MEMIF_RING_FLAG_MASK_INT;
1210                 }
1211
1212                 pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
1213                 pmd->flags |= ETH_MEMIF_FLAG_CONNECTED;
1214                 dev->data->dev_link.link_status = ETH_LINK_UP;
1215         }
1216         MIF_LOG(INFO, "Connected.");
1217         return 0;
1218 }
1219
1220 static int
1221 memif_dev_start(struct rte_eth_dev *dev)
1222 {
1223         struct pmd_internals *pmd = dev->data->dev_private;
1224         int ret = 0;
1225
1226         switch (pmd->role) {
1227         case MEMIF_ROLE_SLAVE:
1228                 ret = memif_connect_slave(dev);
1229                 break;
1230         case MEMIF_ROLE_MASTER:
1231                 ret = memif_connect_master(dev);
1232                 break;
1233         default:
1234                 MIF_LOG(ERR, "Unknown role: %d.", pmd->role);
1235                 ret = -1;
1236                 break;
1237         }
1238
1239         return ret;
1240 }
1241
1242 static int
1243 memif_dev_close(struct rte_eth_dev *dev)
1244 {
1245         struct pmd_internals *pmd = dev->data->dev_private;
1246         int i;
1247
1248         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
1249                 memif_msg_enq_disconnect(pmd->cc, "Device closed", 0);
1250                 memif_disconnect(dev);
1251
1252                 for (i = 0; i < dev->data->nb_rx_queues; i++)
1253                         (*dev->dev_ops->rx_queue_release)(dev->data->rx_queues[i]);
1254                 for (i = 0; i < dev->data->nb_tx_queues; i++)
1255                         (*dev->dev_ops->tx_queue_release)(dev->data->tx_queues[i]);
1256
1257                 memif_socket_remove_device(dev);
1258         } else {
1259                 memif_disconnect(dev);
1260         }
1261
1262         rte_free(dev->process_private);
1263
1264         return 0;
1265 }
1266
1267 static int
1268 memif_dev_configure(struct rte_eth_dev *dev)
1269 {
1270         struct pmd_internals *pmd = dev->data->dev_private;
1271
1272         /*
1273          * SLAVE - TXQ
1274          * MASTER - RXQ
1275          */
1276         pmd->cfg.num_s2m_rings = (pmd->role == MEMIF_ROLE_SLAVE) ?
1277                                   dev->data->nb_tx_queues : dev->data->nb_rx_queues;
1278
1279         /*
1280          * SLAVE - RXQ
1281          * MASTER - TXQ
1282          */
1283         pmd->cfg.num_m2s_rings = (pmd->role == MEMIF_ROLE_SLAVE) ?
1284                                   dev->data->nb_rx_queues : dev->data->nb_tx_queues;
1285
1286         return 0;
1287 }
1288
1289 static int
1290 memif_tx_queue_setup(struct rte_eth_dev *dev,
1291                      uint16_t qid,
1292                      uint16_t nb_tx_desc __rte_unused,
1293                      unsigned int socket_id __rte_unused,
1294                      const struct rte_eth_txconf *tx_conf __rte_unused)
1295 {
1296         struct pmd_internals *pmd = dev->data->dev_private;
1297         struct memif_queue *mq;
1298
1299         mq = rte_zmalloc("tx-queue", sizeof(struct memif_queue), 0);
1300         if (mq == NULL) {
1301                 MIF_LOG(ERR, "Failed to allocate tx queue id: %u", qid);
1302                 return -ENOMEM;
1303         }
1304
1305         mq->type =
1306             (pmd->role == MEMIF_ROLE_SLAVE) ? MEMIF_RING_S2M : MEMIF_RING_M2S;
1307         mq->n_pkts = 0;
1308         mq->n_bytes = 0;
1309         mq->intr_handle.fd = -1;
1310         mq->intr_handle.type = RTE_INTR_HANDLE_EXT;
1311         mq->in_port = dev->data->port_id;
1312         dev->data->tx_queues[qid] = mq;
1313
1314         return 0;
1315 }
1316
1317 static int
1318 memif_rx_queue_setup(struct rte_eth_dev *dev,
1319                      uint16_t qid,
1320                      uint16_t nb_rx_desc __rte_unused,
1321                      unsigned int socket_id __rte_unused,
1322                      const struct rte_eth_rxconf *rx_conf __rte_unused,
1323                      struct rte_mempool *mb_pool)
1324 {
1325         struct pmd_internals *pmd = dev->data->dev_private;
1326         struct memif_queue *mq;
1327
1328         mq = rte_zmalloc("rx-queue", sizeof(struct memif_queue), 0);
1329         if (mq == NULL) {
1330                 MIF_LOG(ERR, "Failed to allocate rx queue id: %u", qid);
1331                 return -ENOMEM;
1332         }
1333
1334         mq->type = (pmd->role == MEMIF_ROLE_SLAVE) ? MEMIF_RING_M2S : MEMIF_RING_S2M;
1335         mq->n_pkts = 0;
1336         mq->n_bytes = 0;
1337         mq->intr_handle.fd = -1;
1338         mq->intr_handle.type = RTE_INTR_HANDLE_EXT;
1339         mq->mempool = mb_pool;
1340         mq->in_port = dev->data->port_id;
1341         dev->data->rx_queues[qid] = mq;
1342
1343         return 0;
1344 }
1345
1346 static void
1347 memif_queue_release(void *queue)
1348 {
1349         struct memif_queue *mq = (struct memif_queue *)queue;
1350
1351         if (!mq)
1352                 return;
1353
1354         rte_free(mq);
1355 }
1356
1357 static int
1358 memif_link_update(struct rte_eth_dev *dev,
1359                   int wait_to_complete __rte_unused)
1360 {
1361         struct pmd_process_private *proc_private;
1362
1363         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1364                 proc_private = dev->process_private;
1365                 if (dev->data->dev_link.link_status == ETH_LINK_UP &&
1366                                 proc_private->regions_num == 0) {
1367                         memif_mp_request_regions(dev);
1368                 } else if (dev->data->dev_link.link_status == ETH_LINK_DOWN &&
1369                                 proc_private->regions_num > 0) {
1370                         memif_free_regions(dev);
1371                 }
1372         }
1373         return 0;
1374 }
1375
1376 static int
1377 memif_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1378 {
1379         struct pmd_internals *pmd = dev->data->dev_private;
1380         struct memif_queue *mq;
1381         int i;
1382         uint8_t tmp, nq;
1383
1384         stats->ipackets = 0;
1385         stats->ibytes = 0;
1386         stats->opackets = 0;
1387         stats->obytes = 0;
1388
1389         tmp = (pmd->role == MEMIF_ROLE_SLAVE) ? pmd->run.num_s2m_rings :
1390             pmd->run.num_m2s_rings;
1391         nq = (tmp < RTE_ETHDEV_QUEUE_STAT_CNTRS) ? tmp :
1392             RTE_ETHDEV_QUEUE_STAT_CNTRS;
1393
1394         /* RX stats */
1395         for (i = 0; i < nq; i++) {
1396                 mq = dev->data->rx_queues[i];
1397                 stats->q_ipackets[i] = mq->n_pkts;
1398                 stats->q_ibytes[i] = mq->n_bytes;
1399                 stats->ipackets += mq->n_pkts;
1400                 stats->ibytes += mq->n_bytes;
1401         }
1402
1403         tmp = (pmd->role == MEMIF_ROLE_SLAVE) ? pmd->run.num_m2s_rings :
1404             pmd->run.num_s2m_rings;
1405         nq = (tmp < RTE_ETHDEV_QUEUE_STAT_CNTRS) ? tmp :
1406             RTE_ETHDEV_QUEUE_STAT_CNTRS;
1407
1408         /* TX stats */
1409         for (i = 0; i < nq; i++) {
1410                 mq = dev->data->tx_queues[i];
1411                 stats->q_opackets[i] = mq->n_pkts;
1412                 stats->q_obytes[i] = mq->n_bytes;
1413                 stats->opackets += mq->n_pkts;
1414                 stats->obytes += mq->n_bytes;
1415         }
1416         return 0;
1417 }
1418
1419 static int
1420 memif_stats_reset(struct rte_eth_dev *dev)
1421 {
1422         struct pmd_internals *pmd = dev->data->dev_private;
1423         int i;
1424         struct memif_queue *mq;
1425
1426         for (i = 0; i < pmd->run.num_s2m_rings; i++) {
1427                 mq = (pmd->role == MEMIF_ROLE_SLAVE) ? dev->data->tx_queues[i] :
1428                     dev->data->rx_queues[i];
1429                 mq->n_pkts = 0;
1430                 mq->n_bytes = 0;
1431         }
1432         for (i = 0; i < pmd->run.num_m2s_rings; i++) {
1433                 mq = (pmd->role == MEMIF_ROLE_SLAVE) ? dev->data->rx_queues[i] :
1434                     dev->data->tx_queues[i];
1435                 mq->n_pkts = 0;
1436                 mq->n_bytes = 0;
1437         }
1438
1439         return 0;
1440 }
1441
1442 static int
1443 memif_rx_queue_intr_enable(struct rte_eth_dev *dev __rte_unused,
1444                            uint16_t qid __rte_unused)
1445 {
1446         MIF_LOG(WARNING, "Interrupt mode not supported.");
1447
1448         return -1;
1449 }
1450
1451 static int
1452 memif_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t qid __rte_unused)
1453 {
1454         struct pmd_internals *pmd __rte_unused = dev->data->dev_private;
1455
1456         return 0;
1457 }
1458
1459 static const struct eth_dev_ops ops = {
1460         .dev_start = memif_dev_start,
1461         .dev_close = memif_dev_close,
1462         .dev_infos_get = memif_dev_info,
1463         .dev_configure = memif_dev_configure,
1464         .tx_queue_setup = memif_tx_queue_setup,
1465         .rx_queue_setup = memif_rx_queue_setup,
1466         .rx_queue_release = memif_queue_release,
1467         .tx_queue_release = memif_queue_release,
1468         .rx_queue_intr_enable = memif_rx_queue_intr_enable,
1469         .rx_queue_intr_disable = memif_rx_queue_intr_disable,
1470         .link_update = memif_link_update,
1471         .stats_get = memif_stats_get,
1472         .stats_reset = memif_stats_reset,
1473 };
1474
1475 static int
1476 memif_create(struct rte_vdev_device *vdev, enum memif_role_t role,
1477              memif_interface_id_t id, uint32_t flags,
1478              const char *socket_filename,
1479              memif_log2_ring_size_t log2_ring_size,
1480              uint16_t pkt_buffer_size, const char *secret,
1481              struct rte_ether_addr *ether_addr)
1482 {
1483         int ret = 0;
1484         struct rte_eth_dev *eth_dev;
1485         struct rte_eth_dev_data *data;
1486         struct pmd_internals *pmd;
1487         struct pmd_process_private *process_private;
1488         const unsigned int numa_node = vdev->device.numa_node;
1489         const char *name = rte_vdev_device_name(vdev);
1490
1491         eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*pmd));
1492         if (eth_dev == NULL) {
1493                 MIF_LOG(ERR, "%s: Unable to allocate device struct.", name);
1494                 return -1;
1495         }
1496
1497         process_private = (struct pmd_process_private *)
1498                 rte_zmalloc(name, sizeof(struct pmd_process_private),
1499                             RTE_CACHE_LINE_SIZE);
1500
1501         if (process_private == NULL) {
1502                 MIF_LOG(ERR, "Failed to alloc memory for process private");
1503                 return -1;
1504         }
1505         eth_dev->process_private = process_private;
1506
1507         pmd = eth_dev->data->dev_private;
1508         memset(pmd, 0, sizeof(*pmd));
1509
1510         pmd->id = id;
1511         pmd->flags = flags;
1512         pmd->flags |= ETH_MEMIF_FLAG_DISABLED;
1513         pmd->role = role;
1514         /* Zero-copy flag irelevant to master. */
1515         if (pmd->role == MEMIF_ROLE_MASTER)
1516                 pmd->flags &= ~ETH_MEMIF_FLAG_ZERO_COPY;
1517
1518         ret = memif_socket_init(eth_dev, socket_filename);
1519         if (ret < 0)
1520                 return ret;
1521
1522         memset(pmd->secret, 0, sizeof(char) * ETH_MEMIF_SECRET_SIZE);
1523         if (secret != NULL)
1524                 strlcpy(pmd->secret, secret, sizeof(pmd->secret));
1525
1526         pmd->cfg.log2_ring_size = log2_ring_size;
1527         /* set in .dev_configure() */
1528         pmd->cfg.num_s2m_rings = 0;
1529         pmd->cfg.num_m2s_rings = 0;
1530
1531         pmd->cfg.pkt_buffer_size = pkt_buffer_size;
1532         rte_spinlock_init(&pmd->cc_lock);
1533
1534         data = eth_dev->data;
1535         data->dev_private = pmd;
1536         data->numa_node = numa_node;
1537         data->dev_link = pmd_link;
1538         data->mac_addrs = ether_addr;
1539         data->promiscuous = 1;
1540
1541         eth_dev->dev_ops = &ops;
1542         eth_dev->device = &vdev->device;
1543         if (pmd->flags & ETH_MEMIF_FLAG_ZERO_COPY) {
1544                 eth_dev->rx_pkt_burst = eth_memif_rx_zc;
1545                 eth_dev->tx_pkt_burst = eth_memif_tx_zc;
1546         } else {
1547                 eth_dev->rx_pkt_burst = eth_memif_rx;
1548                 eth_dev->tx_pkt_burst = eth_memif_tx;
1549         }
1550
1551         rte_eth_dev_probing_finish(eth_dev);
1552
1553         return 0;
1554 }
1555
1556 static int
1557 memif_set_role(const char *key __rte_unused, const char *value,
1558                void *extra_args)
1559 {
1560         enum memif_role_t *role = (enum memif_role_t *)extra_args;
1561
1562         if (strstr(value, "master") != NULL) {
1563                 *role = MEMIF_ROLE_MASTER;
1564         } else if (strstr(value, "slave") != NULL) {
1565                 *role = MEMIF_ROLE_SLAVE;
1566         } else {
1567                 MIF_LOG(ERR, "Unknown role: %s.", value);
1568                 return -EINVAL;
1569         }
1570         return 0;
1571 }
1572
1573 static int
1574 memif_set_zc(const char *key __rte_unused, const char *value, void *extra_args)
1575 {
1576         uint32_t *flags = (uint32_t *)extra_args;
1577
1578         if (strstr(value, "yes") != NULL) {
1579                 if (!rte_mcfg_get_single_file_segments()) {
1580                         MIF_LOG(ERR, "Zero-copy doesn't support multi-file segments.");
1581                         return -ENOTSUP;
1582                 }
1583                 *flags |= ETH_MEMIF_FLAG_ZERO_COPY;
1584         } else if (strstr(value, "no") != NULL) {
1585                 *flags &= ~ETH_MEMIF_FLAG_ZERO_COPY;
1586         } else {
1587                 MIF_LOG(ERR, "Failed to parse zero-copy param: %s.", value);
1588                 return -EINVAL;
1589         }
1590         return 0;
1591 }
1592
1593 static int
1594 memif_set_id(const char *key __rte_unused, const char *value, void *extra_args)
1595 {
1596         memif_interface_id_t *id = (memif_interface_id_t *)extra_args;
1597
1598         /* even if parsing fails, 0 is a valid id */
1599         *id = strtoul(value, NULL, 10);
1600         return 0;
1601 }
1602
1603 static int
1604 memif_set_bs(const char *key __rte_unused, const char *value, void *extra_args)
1605 {
1606         unsigned long tmp;
1607         uint16_t *pkt_buffer_size = (uint16_t *)extra_args;
1608
1609         tmp = strtoul(value, NULL, 10);
1610         if (tmp == 0 || tmp > 0xFFFF) {
1611                 MIF_LOG(ERR, "Invalid buffer size: %s.", value);
1612                 return -EINVAL;
1613         }
1614         *pkt_buffer_size = tmp;
1615         return 0;
1616 }
1617
1618 static int
1619 memif_set_rs(const char *key __rte_unused, const char *value, void *extra_args)
1620 {
1621         unsigned long tmp;
1622         memif_log2_ring_size_t *log2_ring_size =
1623             (memif_log2_ring_size_t *)extra_args;
1624
1625         tmp = strtoul(value, NULL, 10);
1626         if (tmp == 0 || tmp > ETH_MEMIF_MAX_LOG2_RING_SIZE) {
1627                 MIF_LOG(ERR, "Invalid ring size: %s (max %u).",
1628                         value, ETH_MEMIF_MAX_LOG2_RING_SIZE);
1629                 return -EINVAL;
1630         }
1631         *log2_ring_size = tmp;
1632         return 0;
1633 }
1634
1635 /* check if directory exists and if we have permission to read/write */
1636 static int
1637 memif_check_socket_filename(const char *filename)
1638 {
1639         char *dir = NULL, *tmp;
1640         uint32_t idx;
1641         int ret = 0;
1642
1643         if (strlen(filename) >= MEMIF_SOCKET_UN_SIZE) {
1644                 MIF_LOG(ERR, "Unix socket address too long (max 108).");
1645                 return -1;
1646         }
1647
1648         tmp = strrchr(filename, '/');
1649         if (tmp != NULL) {
1650                 idx = tmp - filename;
1651                 dir = rte_zmalloc("memif_tmp", sizeof(char) * (idx + 1), 0);
1652                 if (dir == NULL) {
1653                         MIF_LOG(ERR, "Failed to allocate memory.");
1654                         return -1;
1655                 }
1656                 strlcpy(dir, filename, sizeof(char) * (idx + 1));
1657         }
1658
1659         if (dir == NULL || (faccessat(-1, dir, F_OK | R_OK |
1660                                         W_OK, AT_EACCESS) < 0)) {
1661                 MIF_LOG(ERR, "Invalid socket directory.");
1662                 ret = -EINVAL;
1663         }
1664
1665         if (dir != NULL)
1666                 rte_free(dir);
1667
1668         return ret;
1669 }
1670
1671 static int
1672 memif_set_socket_filename(const char *key __rte_unused, const char *value,
1673                           void *extra_args)
1674 {
1675         const char **socket_filename = (const char **)extra_args;
1676
1677         *socket_filename = value;
1678         return memif_check_socket_filename(*socket_filename);
1679 }
1680
1681 static int
1682 memif_set_mac(const char *key __rte_unused, const char *value, void *extra_args)
1683 {
1684         struct rte_ether_addr *ether_addr = (struct rte_ether_addr *)extra_args;
1685
1686         if (rte_ether_unformat_addr(value, ether_addr) < 0)
1687                 MIF_LOG(WARNING, "Failed to parse mac '%s'.", value);
1688         return 0;
1689 }
1690
1691 static int
1692 memif_set_secret(const char *key __rte_unused, const char *value, void *extra_args)
1693 {
1694         const char **secret = (const char **)extra_args;
1695
1696         *secret = value;
1697         return 0;
1698 }
1699
1700 static int
1701 rte_pmd_memif_probe(struct rte_vdev_device *vdev)
1702 {
1703         RTE_BUILD_BUG_ON(sizeof(memif_msg_t) != 128);
1704         RTE_BUILD_BUG_ON(sizeof(memif_desc_t) != 16);
1705         int ret = 0;
1706         struct rte_kvargs *kvlist;
1707         const char *name = rte_vdev_device_name(vdev);
1708         enum memif_role_t role = MEMIF_ROLE_SLAVE;
1709         memif_interface_id_t id = 0;
1710         uint16_t pkt_buffer_size = ETH_MEMIF_DEFAULT_PKT_BUFFER_SIZE;
1711         memif_log2_ring_size_t log2_ring_size = ETH_MEMIF_DEFAULT_RING_SIZE;
1712         const char *socket_filename = ETH_MEMIF_DEFAULT_SOCKET_FILENAME;
1713         uint32_t flags = 0;
1714         const char *secret = NULL;
1715         struct rte_ether_addr *ether_addr = rte_zmalloc("",
1716                 sizeof(struct rte_ether_addr), 0);
1717         struct rte_eth_dev *eth_dev;
1718
1719         rte_eth_random_addr(ether_addr->addr_bytes);
1720
1721         MIF_LOG(INFO, "Initialize MEMIF: %s.", name);
1722
1723         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1724                 eth_dev = rte_eth_dev_attach_secondary(name);
1725                 if (!eth_dev) {
1726                         MIF_LOG(ERR, "Failed to probe %s", name);
1727                         return -1;
1728                 }
1729
1730                 eth_dev->dev_ops = &ops;
1731                 eth_dev->device = &vdev->device;
1732                 eth_dev->rx_pkt_burst = eth_memif_rx;
1733                 eth_dev->tx_pkt_burst = eth_memif_tx;
1734
1735                 if (!rte_eal_primary_proc_alive(NULL)) {
1736                         MIF_LOG(ERR, "Primary process is missing");
1737                         return -1;
1738                 }
1739
1740                 eth_dev->process_private = (struct pmd_process_private *)
1741                         rte_zmalloc(name,
1742                                 sizeof(struct pmd_process_private),
1743                                 RTE_CACHE_LINE_SIZE);
1744                 if (eth_dev->process_private == NULL) {
1745                         MIF_LOG(ERR,
1746                                 "Failed to alloc memory for process private");
1747                         return -1;
1748                 }
1749
1750                 rte_eth_dev_probing_finish(eth_dev);
1751
1752                 return 0;
1753         }
1754
1755         ret = rte_mp_action_register(MEMIF_MP_SEND_REGION, memif_mp_send_region);
1756         /*
1757          * Primary process can continue probing, but secondary process won't
1758          * be able to get memory regions information
1759          */
1760         if (ret < 0 && rte_errno != EEXIST)
1761                 MIF_LOG(WARNING, "Failed to register mp action callback: %s",
1762                         strerror(rte_errno));
1763
1764         kvlist = rte_kvargs_parse(rte_vdev_device_args(vdev), valid_arguments);
1765
1766         /* parse parameters */
1767         if (kvlist != NULL) {
1768                 ret = rte_kvargs_process(kvlist, ETH_MEMIF_ROLE_ARG,
1769                                          &memif_set_role, &role);
1770                 if (ret < 0)
1771                         goto exit;
1772                 ret = rte_kvargs_process(kvlist, ETH_MEMIF_ID_ARG,
1773                                          &memif_set_id, &id);
1774                 if (ret < 0)
1775                         goto exit;
1776                 ret = rte_kvargs_process(kvlist, ETH_MEMIF_PKT_BUFFER_SIZE_ARG,
1777                                          &memif_set_bs, &pkt_buffer_size);
1778                 if (ret < 0)
1779                         goto exit;
1780                 ret = rte_kvargs_process(kvlist, ETH_MEMIF_RING_SIZE_ARG,
1781                                          &memif_set_rs, &log2_ring_size);
1782                 if (ret < 0)
1783                         goto exit;
1784                 ret = rte_kvargs_process(kvlist, ETH_MEMIF_SOCKET_ARG,
1785                                          &memif_set_socket_filename,
1786                                          (void *)(&socket_filename));
1787                 if (ret < 0)
1788                         goto exit;
1789                 ret = rte_kvargs_process(kvlist, ETH_MEMIF_MAC_ARG,
1790                                          &memif_set_mac, ether_addr);
1791                 if (ret < 0)
1792                         goto exit;
1793                 ret = rte_kvargs_process(kvlist, ETH_MEMIF_ZC_ARG,
1794                                          &memif_set_zc, &flags);
1795                 if (ret < 0)
1796                         goto exit;
1797                 ret = rte_kvargs_process(kvlist, ETH_MEMIF_SECRET_ARG,
1798                                          &memif_set_secret, (void *)(&secret));
1799                 if (ret < 0)
1800                         goto exit;
1801         }
1802
1803         /* create interface */
1804         ret = memif_create(vdev, role, id, flags, socket_filename,
1805                            log2_ring_size, pkt_buffer_size, secret, ether_addr);
1806
1807 exit:
1808         if (kvlist != NULL)
1809                 rte_kvargs_free(kvlist);
1810         return ret;
1811 }
1812
1813 static int
1814 rte_pmd_memif_remove(struct rte_vdev_device *vdev)
1815 {
1816         struct rte_eth_dev *eth_dev;
1817
1818         eth_dev = rte_eth_dev_allocated(rte_vdev_device_name(vdev));
1819         if (eth_dev == NULL)
1820                 return 0;
1821
1822         rte_eth_dev_close(eth_dev->data->port_id);
1823
1824         return 0;
1825 }
1826
1827 static struct rte_vdev_driver pmd_memif_drv = {
1828         .probe = rte_pmd_memif_probe,
1829         .remove = rte_pmd_memif_remove,
1830 };
1831
1832 RTE_PMD_REGISTER_VDEV(net_memif, pmd_memif_drv);
1833
1834 RTE_PMD_REGISTER_PARAM_STRING(net_memif,
1835                               ETH_MEMIF_ID_ARG "=<int>"
1836                               ETH_MEMIF_ROLE_ARG "=master|slave"
1837                               ETH_MEMIF_PKT_BUFFER_SIZE_ARG "=<int>"
1838                               ETH_MEMIF_RING_SIZE_ARG "=<int>"
1839                               ETH_MEMIF_SOCKET_ARG "=<string>"
1840                               ETH_MEMIF_MAC_ARG "=xx:xx:xx:xx:xx:xx"
1841                               ETH_MEMIF_ZC_ARG "=yes|no"
1842                               ETH_MEMIF_SECRET_ARG "=<string>");
1843
1844 RTE_LOG_REGISTER(memif_logtype, pmd.net.memif, NOTICE);