drivers/net/memif/memif_socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/un.h>
  11 #include <sys/ioctl.h>
  12 #include <errno.h>
  13
  14 #include <rte_version.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_ether.h>
  17 #include <rte_ethdev_driver.h>
  18 #include <rte_ethdev_vdev.h>
  19 #include <rte_malloc.h>
  20 #include <rte_kvargs.h>
  21 #include <rte_bus_vdev.h>
  22 #include <rte_hash.h>
  23 #include <rte_jhash.h>
  24 #include <rte_string_fns.h>
  25
  26 #include "rte_eth_memif.h"
  27 #include "memif_socket.h"
  28
  29 static void memif_intr_handler(void *arg);
  30
  31 static ssize_t
  32 memif_msg_send(int fd, memif_msg_t *msg, int afd)
  33 {
  34         struct msghdr mh = { 0 };
  35         struct iovec iov[1];
  36         struct cmsghdr *cmsg;
  37         char ctl[CMSG_SPACE(sizeof(int))];
  38
  39         iov[0].iov_base = msg;
  40         iov[0].iov_len = sizeof(memif_msg_t);
  41         mh.msg_iov = iov;
  42         mh.msg_iovlen = 1;
  43
  44         if (afd > 0) {
  45                 memset(&ctl, 0, sizeof(ctl));
  46                 mh.msg_control = ctl;
  47                 mh.msg_controllen = sizeof(ctl);
  48                 cmsg = CMSG_FIRSTHDR(&mh);
  49                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  50                 cmsg->cmsg_level = SOL_SOCKET;
  51                 cmsg->cmsg_type = SCM_RIGHTS;
  52                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
  53         }
  54
  55         return sendmsg(fd, &mh, 0);
  56 }
  57
  58 static int
  59 memif_msg_send_from_queue(struct memif_control_channel *cc)
  60 {
  61         ssize_t size;
  62         int ret = 0;
  63         struct memif_msg_queue_elt *e;
  64
  65         e = TAILQ_FIRST(&cc->msg_queue);
  66         if (e == NULL)
  67                 return 0;
  68
  69         size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
  70         if (size != sizeof(memif_msg_t)) {
  71                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
  72                 ret = -1;
  73         } else {
  74                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
  75         }
  76         TAILQ_REMOVE(&cc->msg_queue, e, next);
  77         rte_free(e);
  78
  79         return ret;
  80 }
  81
  82 static struct memif_msg_queue_elt *
  83 memif_msg_enq(struct memif_control_channel *cc)
  84 {
  85         struct memif_msg_queue_elt *e;
  86
  87         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
  88         if (e == NULL) {
  89                 MIF_LOG(ERR, "Failed to allocate control message.");
  90                 return NULL;
  91         }
  92
  93         e->fd = -1;
  94         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
  95
  96         return e;
  97 }
  98
  99 void
 100 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
 101                          int err_code)
 102 {
 103         struct memif_msg_queue_elt *e;
 104         struct pmd_internals *pmd;
 105         memif_msg_disconnect_t *d;
 106
 107         if (cc == NULL) {
 108                 MIF_LOG(DEBUG, "Missing control channel.");
 109                 return;
 110         }
 111
 112         e = memif_msg_enq(cc);
 113         if (e == NULL) {
 114                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
 115                 return;
 116         }
 117
 118         d = &e->msg.disconnect;
 119
 120         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
 121         d->code = err_code;
 122
 123         if (reason != NULL) {
 124                 strlcpy((char *)d->string, reason, sizeof(d->string));
 125                 if (cc->dev != NULL) {
 126                         pmd = cc->dev->data->dev_private;
 127                         strlcpy(pmd->local_disc_string, reason,
 128                                 sizeof(pmd->local_disc_string));
 129                 }
 130         }
 131 }
 132
 133 static int
 134 memif_msg_enq_hello(struct memif_control_channel *cc)
 135 {
 136         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
 137         memif_msg_hello_t *h;
 138
 139         if (e == NULL)
 140                 return -1;
 141
 142         h = &e->msg.hello;
 143
 144         e->msg.type = MEMIF_MSG_TYPE_HELLO;
 145         h->min_version = MEMIF_VERSION;
 146         h->max_version = MEMIF_VERSION;
 147         h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 148         h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 149         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
 150         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
 151
 152         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
 153
 154         return 0;
 155 }
 156
 157 static int
 158 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
 159 {
 160         struct pmd_internals *pmd = dev->data->dev_private;
 161         memif_msg_hello_t *h = &msg->hello;
 162
 163         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
 164                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
 165                 return -1;
 166         }
 167
 168         /* Set parameters for active connection */
 169         pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1,
 170                                            pmd->cfg.num_s2m_rings);
 171         pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1,
 172                                            pmd->cfg.num_m2s_rings);
 173         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
 174                                             pmd->cfg.log2_ring_size);
 175         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
 176
 177         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
 178
 179         MIF_LOG(DEBUG, "%s: Connecting to %s.",
 180                 rte_vdev_device_name(pmd->vdev), pmd->remote_name);
 181
 182         return 0;
 183 }
 184
 185 static int
 186 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
 187 {
 188         memif_msg_init_t *i = &msg->init;
 189         struct memif_socket_dev_list_elt *elt;
 190         struct pmd_internals *pmd;
 191         struct rte_eth_dev *dev;
 192
 193         if (i->version != MEMIF_VERSION) {
 194                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
 195                 return -1;
 196         }
 197
 198         if (cc->socket == NULL) {
 199                 memif_msg_enq_disconnect(cc, "Device error", 0);
 200                 return -1;
 201         }
 202
 203         /* Find device with requested ID */
 204         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
 205                 dev = elt->dev;
 206                 pmd = dev->data->dev_private;
 207                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
 208                     pmd->id == i->id) {
 209                         /* assign control channel to device */
 210                         cc->dev = dev;
 211                         pmd->cc = cc;
 212
 213                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
 214                                 memif_msg_enq_disconnect(pmd->cc,
 215                                                          "Only ethernet mode supported",
 216                                                          0);
 217                                 return -1;
 218                         }
 219
 220                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
 221                                            ETH_MEMIF_FLAG_CONNECTED)) {
 222                                 memif_msg_enq_disconnect(pmd->cc,
 223                                                          "Already connected", 0);
 224                                 return -1;
 225                         }
 226                         strlcpy(pmd->remote_name, (char *)i->name,
 227                                 sizeof(pmd->remote_name));
 228
 229                         if (*pmd->secret != '\0') {
 230                                 if (*i->secret == '\0') {
 231                                         memif_msg_enq_disconnect(pmd->cc,
 232                                                                  "Secret required", 0);
 233                                         return -1;
 234                                 }
 235                                 if (strncmp(pmd->secret, (char *)i->secret,
 236                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
 237                                         memif_msg_enq_disconnect(pmd->cc,
 238                                                                  "Incorrect secret", 0);
 239                                         return -1;
 240                                 }
 241                         }
 242
 243                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
 244                         return 0;
 245                 }
 246         }
 247
 248         /* ID not found on this socket */
 249         MIF_LOG(DEBUG, "ID %u not found.", i->id);
 250         memif_msg_enq_disconnect(cc, "ID not found", 0);
 251         return -1;
 252 }
 253
 254 static int
 255 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
 256                              int fd)
 257 {
 258         struct pmd_internals *pmd = dev->data->dev_private;
 259         memif_msg_add_region_t *ar = &msg->add_region;
 260         struct memif_region *r;
 261
 262         if (fd < 0) {
 263                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
 264                 return -1;
 265         }
 266
 267         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM || ar->index != pmd->regions_num ||
 268                         pmd->regions[ar->index] != NULL) {
 269                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
 270                 return -1;
 271         }
 272
 273         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 274         if (r == NULL) {
 275                 MIF_LOG(ERR, "%s: Failed to alloc memif region.",
 276                         rte_vdev_device_name(pmd->vdev));
 277                 return -ENOMEM;
 278         }
 279
 280         r->fd = fd;
 281         r->region_size = ar->size;
 282         r->addr = NULL;
 283
 284         pmd->regions[ar->index] = r;
 285         pmd->regions_num++;
 286
 287         return 0;
 288 }
 289
 290 static int
 291 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
 292 {
 293         struct pmd_internals *pmd = dev->data->dev_private;
 294         memif_msg_add_ring_t *ar = &msg->add_ring;
 295         struct memif_queue *mq;
 296
 297         if (fd < 0) {
 298                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
 299                 return -1;
 300         }
 301
 302         /* check if we have enough queues */
 303         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) {
 304                 if (ar->index >= pmd->cfg.num_s2m_rings) {
 305                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 306                         return -1;
 307                 }
 308                 pmd->run.num_s2m_rings++;
 309         } else {
 310                 if (ar->index >= pmd->cfg.num_m2s_rings) {
 311                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 312                         return -1;
 313                 }
 314                 pmd->run.num_m2s_rings++;
 315         }
 316
 317         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ?
 318             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
 319
 320         mq->intr_handle.fd = fd;
 321         mq->log2_ring_size = ar->log2_ring_size;
 322         mq->region = ar->region;
 323         mq->ring_offset = ar->offset;
 324
 325         return 0;
 326 }
 327
 328 static int
 329 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
 330 {
 331         struct pmd_internals *pmd = dev->data->dev_private;
 332         memif_msg_connect_t *c = &msg->connect;
 333         int ret;
 334
 335         ret = memif_connect(dev);
 336         if (ret < 0)
 337                 return ret;
 338
 339         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 340                 sizeof(pmd->remote_if_name));
 341         MIF_LOG(INFO, "%s: Remote interface %s connected.",
 342                 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name);
 343
 344         return 0;
 345 }
 346
 347 static int
 348 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
 349 {
 350         struct pmd_internals *pmd = dev->data->dev_private;
 351         memif_msg_connected_t *c = &msg->connected;
 352         int ret;
 353
 354         ret = memif_connect(dev);
 355         if (ret < 0)
 356                 return ret;
 357
 358         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 359                 sizeof(pmd->remote_if_name));
 360         MIF_LOG(INFO, "%s: Remote interface %s connected.",
 361                 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name);
 362
 363         return 0;
 364 }
 365
 366 static int
 367 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
 368 {
 369         struct pmd_internals *pmd = dev->data->dev_private;
 370         memif_msg_disconnect_t *d = &msg->disconnect;
 371
 372         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
 373         strlcpy(pmd->remote_disc_string, (char *)d->string,
 374                 sizeof(pmd->remote_disc_string));
 375
 376         MIF_LOG(INFO, "%s: Disconnect received: %s",
 377                 rte_vdev_device_name(pmd->vdev), pmd->remote_disc_string);
 378
 379         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
 380         memif_disconnect(rte_eth_dev_allocated
 381                          (rte_vdev_device_name(pmd->vdev)));
 382         return 0;
 383 }
 384
 385 static int
 386 memif_msg_enq_ack(struct rte_eth_dev *dev)
 387 {
 388         struct pmd_internals *pmd = dev->data->dev_private;
 389         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 390         if (e == NULL)
 391                 return -1;
 392
 393         e->msg.type = MEMIF_MSG_TYPE_ACK;
 394
 395         return 0;
 396 }
 397
 398 static int
 399 memif_msg_enq_init(struct rte_eth_dev *dev)
 400 {
 401         struct pmd_internals *pmd = dev->data->dev_private;
 402         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 403         memif_msg_init_t *i = &e->msg.init;
 404
 405         if (e == NULL)
 406                 return -1;
 407
 408         i = &e->msg.init;
 409         e->msg.type = MEMIF_MSG_TYPE_INIT;
 410         i->version = MEMIF_VERSION;
 411         i->id = pmd->id;
 412         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
 413
 414         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
 415
 416         if (*pmd->secret != '\0')
 417                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
 418
 419         return 0;
 420 }
 421
 422 static int
 423 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
 424 {
 425         struct pmd_internals *pmd = dev->data->dev_private;
 426         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 427         memif_msg_add_region_t *ar;
 428         struct memif_region *mr = pmd->regions[idx];
 429
 430         if (e == NULL)
 431                 return -1;
 432
 433         ar = &e->msg.add_region;
 434         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
 435         e->fd = mr->fd;
 436         ar->index = idx;
 437         ar->size = mr->region_size;
 438
 439         return 0;
 440 }
 441
 442 static int
 443 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
 444                        memif_ring_type_t type)
 445 {
 446         struct pmd_internals *pmd = dev->data->dev_private;
 447         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 448         struct memif_queue *mq;
 449         memif_msg_add_ring_t *ar;
 450
 451         if (e == NULL)
 452                 return -1;
 453
 454         ar = &e->msg.add_ring;
 455         mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] :
 456             dev->data->rx_queues[idx];
 457
 458         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
 459         e->fd = mq->intr_handle.fd;
 460         ar->index = idx;
 461         ar->offset = mq->ring_offset;
 462         ar->region = mq->region;
 463         ar->log2_ring_size = mq->log2_ring_size;
 464         ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
 465         ar->private_hdr_size = 0;
 466
 467         return 0;
 468 }
 469
 470 static int
 471 memif_msg_enq_connect(struct rte_eth_dev *dev)
 472 {
 473         struct pmd_internals *pmd = dev->data->dev_private;
 474         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 475         const char *name = rte_vdev_device_name(pmd->vdev);
 476         memif_msg_connect_t *c;
 477
 478         if (e == NULL)
 479                 return -1;
 480
 481         c = &e->msg.connect;
 482         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
 483         strlcpy((char *)c->if_name, name, sizeof(c->if_name));
 484
 485         return 0;
 486 }
 487
 488 static int
 489 memif_msg_enq_connected(struct rte_eth_dev *dev)
 490 {
 491         struct pmd_internals *pmd = dev->data->dev_private;
 492         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 493         const char *name = rte_vdev_device_name(pmd->vdev);
 494         memif_msg_connected_t *c;
 495
 496         if (e == NULL)
 497                 return -1;
 498
 499         c = &e->msg.connected;
 500         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
 501         strlcpy((char *)c->if_name, name, sizeof(c->if_name));
 502
 503         return 0;
 504 }
 505
 506 static void
 507 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
 508 {
 509         struct memif_msg_queue_elt *elt;
 510         struct memif_control_channel *cc = arg;
 511
 512         /* close control channel fd */
 513         close(intr_handle->fd);
 514         /* clear message queue */
 515         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
 516                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
 517                 rte_free(elt);
 518         }
 519         /* free control channel */
 520         rte_free(cc);
 521 }
 522
 523 void
 524 memif_disconnect(struct rte_eth_dev *dev)
 525 {
 526         struct pmd_internals *pmd = dev->data->dev_private;
 527         struct memif_msg_queue_elt *elt, *next;
 528         struct memif_queue *mq;
 529         struct rte_intr_handle *ih;
 530         int i;
 531         int ret;
 532
 533         if (pmd->cc != NULL) {
 534                 /* Clear control message queue (except disconnect message if any). */
 535                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
 536                         next = TAILQ_NEXT(elt, next);
 537                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
 538                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
 539                                 rte_free(elt);
 540                         }
 541                 }
 542                 /* send disconnect message (if there is any in queue) */
 543                 memif_msg_send_from_queue(pmd->cc);
 544
 545                 /* at this point, there should be no more messages in queue */
 546                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
 547                         MIF_LOG(WARNING,
 548                                 "%s: Unexpected message(s) in message queue.",
 549                                 rte_vdev_device_name(pmd->vdev));
 550                 }
 551
 552                 ih = &pmd->cc->intr_handle;
 553                 if (ih->fd > 0) {
 554                         ret = rte_intr_callback_unregister(ih,
 555                                                         memif_intr_handler,
 556                                                         pmd->cc);
 557                         /*
 558                          * If callback is active (disconnecting based on
 559                          * received control message).
 560                          */
 561                         if (ret == -EAGAIN) {
 562                                 ret = rte_intr_callback_unregister_pending(ih,
 563                                                         memif_intr_handler,
 564                                                         pmd->cc,
 565                                                         memif_intr_unregister_handler);
 566                         } else if (ret > 0) {
 567                                 close(ih->fd);
 568                                 rte_free(pmd->cc);
 569                         }
 570                         pmd->cc = NULL;
 571                         if (ret <= 0)
 572                                 MIF_LOG(WARNING, "%s: Failed to unregister "
 573                                         "control channel callback.",
 574                                         rte_vdev_device_name(pmd->vdev));
 575                 }
 576         }
 577
 578         /* unconfig interrupts */
 579         for (i = 0; i < pmd->cfg.num_s2m_rings; i++) {
 580                 if (pmd->role == MEMIF_ROLE_SLAVE) {
 581                         if (dev->data->tx_queues != NULL)
 582                                 mq = dev->data->tx_queues[i];
 583                         else
 584                                 continue;
 585                 } else {
 586                         if (dev->data->rx_queues != NULL)
 587                                 mq = dev->data->rx_queues[i];
 588                         else
 589                                 continue;
 590                 }
 591                 if (mq->intr_handle.fd > 0) {
 592                         close(mq->intr_handle.fd);
 593                         mq->intr_handle.fd = -1;
 594                 }
 595                 mq->ring = NULL;
 596         }
 597         for (i = 0; i < pmd->cfg.num_m2s_rings; i++) {
 598                 if (pmd->role == MEMIF_ROLE_MASTER) {
 599                         if (dev->data->tx_queues != NULL)
 600                                 mq = dev->data->tx_queues[i];
 601                         else
 602                                 continue;
 603                 } else {
 604                         if (dev->data->rx_queues != NULL)
 605                                 mq = dev->data->rx_queues[i];
 606                         else
 607                                 continue;
 608                 }
 609                 if (mq->intr_handle.fd > 0) {
 610                         close(mq->intr_handle.fd);
 611                         mq->intr_handle.fd = -1;
 612                 }
 613                 mq->ring = NULL;
 614         }
 615
 616         memif_free_regions(pmd);
 617
 618         /* reset connection configuration */
 619         memset(&pmd->run, 0, sizeof(pmd->run));
 620
 621         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 622         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
 623         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
 624         MIF_LOG(DEBUG, "%s: Disconnected.", rte_vdev_device_name(pmd->vdev));
 625 }
 626
 627 static int
 628 memif_msg_receive(struct memif_control_channel *cc)
 629 {
 630         char ctl[CMSG_SPACE(sizeof(int)) +
 631                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
 632         struct msghdr mh = { 0 };
 633         struct iovec iov[1];
 634         memif_msg_t msg = { 0 };
 635         ssize_t size;
 636         int ret = 0;
 637         struct ucred *cr __rte_unused;
 638         cr = 0;
 639         struct cmsghdr *cmsg;
 640         int afd = -1;
 641         int i;
 642         struct pmd_internals *pmd;
 643
 644         iov[0].iov_base = (void *)&msg;
 645         iov[0].iov_len = sizeof(memif_msg_t);
 646         mh.msg_iov = iov;
 647         mh.msg_iovlen = 1;
 648         mh.msg_control = ctl;
 649         mh.msg_controllen = sizeof(ctl);
 650
 651         size = recvmsg(cc->intr_handle.fd, &mh, 0);
 652         if (size != sizeof(memif_msg_t)) {
 653                 MIF_LOG(DEBUG, "Invalid message size.");
 654                 memif_msg_enq_disconnect(cc, "Invalid message size", 0);
 655                 return -1;
 656         }
 657         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
 658
 659         cmsg = CMSG_FIRSTHDR(&mh);
 660         while (cmsg) {
 661                 if (cmsg->cmsg_level == SOL_SOCKET) {
 662                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
 663                                 cr = (struct ucred *)CMSG_DATA(cmsg);
 664                         else if (cmsg->cmsg_type == SCM_RIGHTS)
 665                                 memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
 666                 }
 667                 cmsg = CMSG_NXTHDR(&mh, cmsg);
 668         }
 669
 670         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
 671                 MIF_LOG(DEBUG, "Unexpected message.");
 672                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
 673                 return -1;
 674         }
 675
 676         /* get device from hash data */
 677         switch (msg.type) {
 678         case MEMIF_MSG_TYPE_ACK:
 679                 break;
 680         case MEMIF_MSG_TYPE_HELLO:
 681                 ret = memif_msg_receive_hello(cc->dev, &msg);
 682                 if (ret < 0)
 683                         goto exit;
 684                 ret = memif_init_regions_and_queues(cc->dev);
 685                 if (ret < 0)
 686                         goto exit;
 687                 ret = memif_msg_enq_init(cc->dev);
 688                 if (ret < 0)
 689                         goto exit;
 690                 pmd = cc->dev->data->dev_private;
 691                 for (i = 0; i < pmd->regions_num; i++) {
 692                         ret = memif_msg_enq_add_region(cc->dev, i);
 693                         if (ret < 0)
 694                                 goto exit;
 695                 }
 696                 for (i = 0; i < pmd->run.num_s2m_rings; i++) {
 697                         ret = memif_msg_enq_add_ring(cc->dev, i,
 698                                                      MEMIF_RING_S2M);
 699                         if (ret < 0)
 700                                 goto exit;
 701                 }
 702                 for (i = 0; i < pmd->run.num_m2s_rings; i++) {
 703                         ret = memif_msg_enq_add_ring(cc->dev, i,
 704                                                      MEMIF_RING_M2S);
 705                         if (ret < 0)
 706                                 goto exit;
 707                 }
 708                 ret = memif_msg_enq_connect(cc->dev);
 709                 if (ret < 0)
 710                         goto exit;
 711                 break;
 712         case MEMIF_MSG_TYPE_INIT:
 713                 /*
 714                  * This cc does not have an interface asociated with it.
 715                  * If suitable interface is found it will be assigned here.
 716                  */
 717                 ret = memif_msg_receive_init(cc, &msg);
 718                 if (ret < 0)
 719                         goto exit;
 720                 ret = memif_msg_enq_ack(cc->dev);
 721                 if (ret < 0)
 722                         goto exit;
 723                 break;
 724         case MEMIF_MSG_TYPE_ADD_REGION:
 725                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
 726                 if (ret < 0)
 727                         goto exit;
 728                 ret = memif_msg_enq_ack(cc->dev);
 729                 if (ret < 0)
 730                         goto exit;
 731                 break;
 732         case MEMIF_MSG_TYPE_ADD_RING:
 733                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
 734                 if (ret < 0)
 735                         goto exit;
 736                 ret = memif_msg_enq_ack(cc->dev);
 737                 if (ret < 0)
 738                         goto exit;
 739                 break;
 740         case MEMIF_MSG_TYPE_CONNECT:
 741                 ret = memif_msg_receive_connect(cc->dev, &msg);
 742                 if (ret < 0)
 743                         goto exit;
 744                 ret = memif_msg_enq_connected(cc->dev);
 745                 if (ret < 0)
 746                         goto exit;
 747                 break;
 748         case MEMIF_MSG_TYPE_CONNECTED:
 749                 ret = memif_msg_receive_connected(cc->dev, &msg);
 750                 break;
 751         case MEMIF_MSG_TYPE_DISCONNECT:
 752                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
 753                 if (ret < 0)
 754                         goto exit;
 755                 break;
 756         default:
 757                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
 758                 ret = -1;
 759                 goto exit;
 760         }
 761
 762  exit:
 763         return ret;
 764 }
 765
 766 static void
 767 memif_intr_handler(void *arg)
 768 {
 769         struct memif_control_channel *cc = arg;
 770         int ret;
 771
 772         ret = memif_msg_receive(cc);
 773         /* if driver failed to assign device */
 774         if (cc->dev == NULL) {
 775                 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
 776                                                            memif_intr_handler,
 777                                                            cc,
 778                                                            memif_intr_unregister_handler);
 779                 if (ret < 0)
 780                         MIF_LOG(WARNING,
 781                                 "Failed to unregister control channel callback.");
 782                 return;
 783         }
 784         /* if memif_msg_receive failed */
 785         if (ret < 0)
 786                 goto disconnect;
 787
 788         ret = memif_msg_send_from_queue(cc);
 789         if (ret < 0)
 790                 goto disconnect;
 791
 792         return;
 793
 794  disconnect:
 795         if (cc->dev == NULL) {
 796                 MIF_LOG(WARNING, "eth dev not allocated");
 797                 return;
 798         }
 799         memif_disconnect(cc->dev);
 800 }
 801
 802 static void
 803 memif_listener_handler(void *arg)
 804 {
 805         struct memif_socket *socket = arg;
 806         int sockfd;
 807         int addr_len;
 808         struct sockaddr_un client;
 809         struct memif_control_channel *cc;
 810         int ret;
 811
 812         addr_len = sizeof(client);
 813         sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
 814                         (socklen_t *)&addr_len);
 815         if (sockfd < 0) {
 816                 MIF_LOG(ERR,
 817                         "Failed to accept connection request on socket fd %d",
 818                         socket->intr_handle.fd);
 819                 return;
 820         }
 821
 822         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
 823
 824         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
 825         if (cc == NULL) {
 826                 MIF_LOG(ERR, "Failed to allocate control channel.");
 827                 goto error;
 828         }
 829
 830         cc->intr_handle.fd = sockfd;
 831         cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
 832         cc->socket = socket;
 833         cc->dev = NULL;
 834         TAILQ_INIT(&cc->msg_queue);
 835
 836         ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
 837         if (ret < 0) {
 838                 MIF_LOG(ERR, "Failed to register control channel callback.");
 839                 goto error;
 840         }
 841
 842         ret = memif_msg_enq_hello(cc);
 843         if (ret < 0) {
 844                 MIF_LOG(ERR, "Failed to enqueue hello message.");
 845                 goto error;
 846         }
 847         ret = memif_msg_send_from_queue(cc);
 848         if (ret < 0)
 849                 goto error;
 850
 851         return;
 852
 853  error:
 854         if (sockfd > 0) {
 855                 close(sockfd);
 856                 sockfd = -1;
 857         }
 858         if (cc != NULL)
 859                 rte_free(cc);
 860 }
 861
 862 static struct memif_socket *
 863 memif_socket_create(struct pmd_internals *pmd, char *key, uint8_t listener)
 864 {
 865         struct memif_socket *sock;
 866         struct sockaddr_un un;
 867         int sockfd;
 868         int ret;
 869         int on = 1;
 870
 871         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
 872         if (sock == NULL) {
 873                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
 874                 return NULL;
 875         }
 876
 877         sock->listener = listener;
 878         rte_memcpy(sock->filename, key, 256);
 879         TAILQ_INIT(&sock->dev_queue);
 880
 881         if (listener != 0) {
 882                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
 883                 if (sockfd < 0)
 884                         goto error;
 885
 886                 un.sun_family = AF_UNIX;
 887                 memcpy(un.sun_path, sock->filename,
 888                         sizeof(un.sun_path) - 1);
 889
 890                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
 891                                  sizeof(on));
 892                 if (ret < 0)
 893                         goto error;
 894                 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un));
 895                 if (ret < 0)
 896                         goto error;
 897                 ret = listen(sockfd, 1);
 898                 if (ret < 0)
 899                         goto error;
 900
 901                 MIF_LOG(DEBUG, "%s: Memif listener socket %s created.",
 902                         rte_vdev_device_name(pmd->vdev), sock->filename);
 903
 904                 sock->intr_handle.fd = sockfd;
 905                 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
 906                 ret = rte_intr_callback_register(&sock->intr_handle,
 907                                                  memif_listener_handler, sock);
 908                 if (ret < 0) {
 909                         MIF_LOG(ERR, "%s: Failed to register interrupt "
 910                                 "callback for listener socket",
 911                                 rte_vdev_device_name(pmd->vdev));
 912                         return NULL;
 913                 }
 914         }
 915
 916         return sock;
 917
 918  error:
 919         MIF_LOG(ERR, "%s: Failed to setup socket %s: %s",
 920                 rte_vdev_device_name(pmd->vdev), key, strerror(errno));
 921         if (sock != NULL)
 922                 rte_free(sock);
 923         return NULL;
 924 }
 925
 926 static struct rte_hash *
 927 memif_create_socket_hash(void)
 928 {
 929         struct rte_hash_parameters params = { 0 };
 930         params.name = MEMIF_SOCKET_HASH_NAME;
 931         params.entries = 256;
 932         params.key_len = 256;
 933         params.hash_func = rte_jhash;
 934         params.hash_func_init_val = 0;
 935         return rte_hash_create(&params);
 936 }
 937
 938 int
 939 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
 940 {
 941         struct pmd_internals *pmd = dev->data->dev_private;
 942         struct memif_socket *socket = NULL;
 943         struct memif_socket_dev_list_elt *elt;
 944         struct pmd_internals *tmp_pmd;
 945         struct rte_hash *hash;
 946         int ret;
 947         char key[256];
 948
 949         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
 950         if (hash == NULL) {
 951                 hash = memif_create_socket_hash();
 952                 if (hash == NULL) {
 953                         MIF_LOG(ERR, "Failed to create memif socket hash.");
 954                         return -1;
 955                 }
 956         }
 957
 958         memset(key, 0, 256);
 959         rte_memcpy(key, socket_filename, strlen(socket_filename));
 960         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
 961         if (ret < 0) {
 962                 socket = memif_socket_create(pmd, key,
 963                                              (pmd->role ==
 964                                               MEMIF_ROLE_SLAVE) ? 0 : 1);
 965                 if (socket == NULL)
 966                         return -1;
 967                 ret = rte_hash_add_key_data(hash, key, socket);
 968                 if (ret < 0) {
 969                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
 970                         return ret;
 971                 }
 972         }
 973         pmd->socket_filename = socket->filename;
 974
 975         if (socket->listener != 0 && pmd->role == MEMIF_ROLE_SLAVE) {
 976                 MIF_LOG(ERR, "Socket is a listener.");
 977                 return -1;
 978         } else if ((socket->listener == 0) && (pmd->role == MEMIF_ROLE_MASTER)) {
 979                 MIF_LOG(ERR, "Socket is not a listener.");
 980                 return -1;
 981         }
 982
 983         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
 984                 tmp_pmd = elt->dev->data->dev_private;
 985                 if (tmp_pmd->id == pmd->id) {
 986                         MIF_LOG(ERR, "Memif device with id %d already "
 987                                 "exists on socket %s",
 988                                 pmd->id, socket->filename);
 989                         return -1;
 990                 }
 991         }
 992
 993         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
 994         if (elt == NULL) {
 995                 MIF_LOG(ERR, "%s: Failed to add device to socket device list.",
 996                         rte_vdev_device_name(pmd->vdev));
 997                 return -1;
 998         }
 999         elt->dev = dev;
1000         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1001
1002         return 0;
1003 }
1004
1005 void
1006 memif_socket_remove_device(struct rte_eth_dev *dev)
1007 {
1008         struct pmd_internals *pmd = dev->data->dev_private;
1009         struct memif_socket *socket = NULL;
1010         struct memif_socket_dev_list_elt *elt, *next;
1011         struct rte_hash *hash;
1012
1013         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1014         if (hash == NULL)
1015                 return;
1016
1017         if (pmd->socket_filename == NULL)
1018                 return;
1019
1020         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1021                 return;
1022
1023         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1024                 next = TAILQ_NEXT(elt, next);
1025                 if (elt->dev == dev) {
1026                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1027                         rte_free(elt);
1028                         pmd->socket_filename = NULL;
1029                 }
1030         }
1031
1032         /* remove socket, if this was the last device using it */
1033         if (TAILQ_EMPTY(&socket->dev_queue)) {
1034                 rte_hash_del_key(hash, socket->filename);
1035                 if (socket->listener) {
1036                         /* remove listener socket file,
1037                          * so we can create new one later.
1038                          */
1039                         remove(socket->filename);
1040                 }
1041                 rte_free(socket);
1042         }
1043 }
1044
1045 int
1046 memif_connect_master(struct rte_eth_dev *dev)
1047 {
1048         struct pmd_internals *pmd = dev->data->dev_private;
1049
1050         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1051         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1052         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1053         return 0;
1054 }
1055
1056 int
1057 memif_connect_slave(struct rte_eth_dev *dev)
1058 {
1059         int sockfd;
1060         int ret;
1061         struct sockaddr_un sun;
1062         struct pmd_internals *pmd = dev->data->dev_private;
1063
1064         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1065         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1066         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1067
1068         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1069         if (sockfd < 0) {
1070                 MIF_LOG(ERR, "%s: Failed to open socket.",
1071                         rte_vdev_device_name(pmd->vdev));
1072                 return -1;
1073         }
1074
1075         sun.sun_family = AF_UNIX;
1076
1077         memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1);
1078
1079         ret = connect(sockfd, (struct sockaddr *)&sun,
1080                       sizeof(struct sockaddr_un));
1081         if (ret < 0) {
1082                 MIF_LOG(ERR, "%s: Failed to connect socket: %s.",
1083                         rte_vdev_device_name(pmd->vdev), pmd->socket_filename);
1084                 goto error;
1085         }
1086
1087         MIF_LOG(DEBUG, "%s: Memif socket: %s connected.",
1088                 rte_vdev_device_name(pmd->vdev), pmd->socket_filename);
1089
1090         pmd->cc = rte_zmalloc("memif-cc",
1091                               sizeof(struct memif_control_channel), 0);
1092         if (pmd->cc == NULL) {
1093                 MIF_LOG(ERR, "%s: Failed to allocate control channel.",
1094                         rte_vdev_device_name(pmd->vdev));
1095                 goto error;
1096         }
1097
1098         pmd->cc->intr_handle.fd = sockfd;
1099         pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1100         pmd->cc->socket = NULL;
1101         pmd->cc->dev = dev;
1102         TAILQ_INIT(&pmd->cc->msg_queue);
1103
1104         ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1105                                          memif_intr_handler, pmd->cc);
1106         if (ret < 0) {
1107                 MIF_LOG(ERR, "%s: Failed to register interrupt callback "
1108                         "for control fd", rte_vdev_device_name(pmd->vdev));
1109                 goto error;
1110         }
1111
1112         return 0;
1113
1114  error:
1115         if (sockfd > 0) {
1116                 close(sockfd);
1117                 sockfd = -1;
1118         }
1119         if (pmd->cc != NULL) {
1120                 rte_free(pmd->cc);
1121                 pmd->cc = NULL;
1122         }
1123         return -1;
1124 }