drivers/net/memif/memif_socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/un.h>
  11 #include <sys/ioctl.h>
  12 #include <errno.h>
  13
  14 #include <rte_version.h>
  15 #include <rte_mbuf.h>
  16 #include <rte_ether.h>
  17 #include <rte_ethdev_driver.h>
  18 #include <rte_ethdev_vdev.h>
  19 #include <rte_malloc.h>
  20 #include <rte_kvargs.h>
  21 #include <rte_bus_vdev.h>
  22 #include <rte_hash.h>
  23 #include <rte_jhash.h>
  24 #include <rte_string_fns.h>
  25
  26 #include "rte_eth_memif.h"
  27 #include "memif_socket.h"
  28
  29 static void memif_intr_handler(void *arg);
  30
  31 static ssize_t
  32 memif_msg_send(int fd, memif_msg_t *msg, int afd)
  33 {
  34         struct msghdr mh = { 0 };
  35         struct iovec iov[1];
  36         struct cmsghdr *cmsg;
  37         char ctl[CMSG_SPACE(sizeof(int))];
  38
  39         iov[0].iov_base = msg;
  40         iov[0].iov_len = sizeof(memif_msg_t);
  41         mh.msg_iov = iov;
  42         mh.msg_iovlen = 1;
  43
  44         if (afd > 0) {
  45                 memset(&ctl, 0, sizeof(ctl));
  46                 mh.msg_control = ctl;
  47                 mh.msg_controllen = sizeof(ctl);
  48                 cmsg = CMSG_FIRSTHDR(&mh);
  49                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  50                 cmsg->cmsg_level = SOL_SOCKET;
  51                 cmsg->cmsg_type = SCM_RIGHTS;
  52                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
  53         }
  54
  55         return sendmsg(fd, &mh, 0);
  56 }
  57
  58 static int
  59 memif_msg_send_from_queue(struct memif_control_channel *cc)
  60 {
  61         ssize_t size;
  62         int ret = 0;
  63         struct memif_msg_queue_elt *e;
  64
  65         e = TAILQ_FIRST(&cc->msg_queue);
  66         if (e == NULL)
  67                 return 0;
  68
  69         size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
  70         if (size != sizeof(memif_msg_t)) {
  71                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
  72                 ret = -1;
  73         } else {
  74                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
  75         }
  76         TAILQ_REMOVE(&cc->msg_queue, e, next);
  77         rte_free(e);
  78
  79         return ret;
  80 }
  81
  82 static struct memif_msg_queue_elt *
  83 memif_msg_enq(struct memif_control_channel *cc)
  84 {
  85         struct memif_msg_queue_elt *e;
  86
  87         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
  88         if (e == NULL) {
  89                 MIF_LOG(ERR, "Failed to allocate control message.");
  90                 return NULL;
  91         }
  92
  93         e->fd = -1;
  94         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
  95
  96         return e;
  97 }
  98
  99 void
 100 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
 101                          int err_code)
 102 {
 103         struct memif_msg_queue_elt *e;
 104         struct pmd_internals *pmd;
 105         memif_msg_disconnect_t *d;
 106
 107         if (cc == NULL) {
 108                 MIF_LOG(DEBUG, "Missing control channel.");
 109                 return;
 110         }
 111
 112         e = memif_msg_enq(cc);
 113         if (e == NULL) {
 114                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
 115                 return;
 116         }
 117
 118         d = &e->msg.disconnect;
 119
 120         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
 121         d->code = err_code;
 122
 123         if (reason != NULL) {
 124                 strlcpy((char *)d->string, reason, sizeof(d->string));
 125                 if (cc->dev != NULL) {
 126                         pmd = cc->dev->data->dev_private;
 127                         strlcpy(pmd->local_disc_string, reason,
 128                                 sizeof(pmd->local_disc_string));
 129                 }
 130         }
 131 }
 132
 133 static int
 134 memif_msg_enq_hello(struct memif_control_channel *cc)
 135 {
 136         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
 137         memif_msg_hello_t *h;
 138
 139         if (e == NULL)
 140                 return -1;
 141
 142         h = &e->msg.hello;
 143
 144         e->msg.type = MEMIF_MSG_TYPE_HELLO;
 145         h->min_version = MEMIF_VERSION;
 146         h->max_version = MEMIF_VERSION;
 147         h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 148         h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 149         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
 150         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
 151
 152         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
 153
 154         return 0;
 155 }
 156
 157 static int
 158 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
 159 {
 160         struct pmd_internals *pmd = dev->data->dev_private;
 161         memif_msg_hello_t *h = &msg->hello;
 162
 163         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
 164                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
 165                 return -1;
 166         }
 167
 168         /* Set parameters for active connection */
 169         pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1,
 170                                            pmd->cfg.num_s2m_rings);
 171         pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1,
 172                                            pmd->cfg.num_m2s_rings);
 173         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
 174                                             pmd->cfg.log2_ring_size);
 175         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
 176
 177         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
 178
 179         MIF_LOG(DEBUG, "%s: Connecting to %s.",
 180                 rte_vdev_device_name(pmd->vdev), pmd->remote_name);
 181
 182         return 0;
 183 }
 184
 185 static int
 186 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
 187 {
 188         memif_msg_init_t *i = &msg->init;
 189         struct memif_socket_dev_list_elt *elt;
 190         struct pmd_internals *pmd;
 191         struct rte_eth_dev *dev;
 192
 193         if (i->version != MEMIF_VERSION) {
 194                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
 195                 return -1;
 196         }
 197
 198         if (cc->socket == NULL) {
 199                 memif_msg_enq_disconnect(cc, "Device error", 0);
 200                 return -1;
 201         }
 202
 203         /* Find device with requested ID */
 204         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
 205                 dev = elt->dev;
 206                 pmd = dev->data->dev_private;
 207                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
 208                     pmd->id == i->id) {
 209                         /* assign control channel to device */
 210                         cc->dev = dev;
 211                         pmd->cc = cc;
 212
 213                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
 214                                 memif_msg_enq_disconnect(pmd->cc,
 215                                                          "Only ethernet mode supported",
 216                                                          0);
 217                                 return -1;
 218                         }
 219
 220                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
 221                                            ETH_MEMIF_FLAG_CONNECTED)) {
 222                                 memif_msg_enq_disconnect(pmd->cc,
 223                                                          "Already connected", 0);
 224                                 return -1;
 225                         }
 226                         strlcpy(pmd->remote_name, (char *)i->name,
 227                                 sizeof(pmd->remote_name));
 228
 229                         if (*pmd->secret != '\0') {
 230                                 if (*i->secret == '\0') {
 231                                         memif_msg_enq_disconnect(pmd->cc,
 232                                                                  "Secret required", 0);
 233                                         return -1;
 234                                 }
 235                                 if (strncmp(pmd->secret, (char *)i->secret,
 236                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
 237                                         memif_msg_enq_disconnect(pmd->cc,
 238                                                                  "Incorrect secret", 0);
 239                                         return -1;
 240                                 }
 241                         }
 242
 243                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
 244                         return 0;
 245                 }
 246         }
 247
 248         /* ID not found on this socket */
 249         MIF_LOG(DEBUG, "ID %u not found.", i->id);
 250         memif_msg_enq_disconnect(cc, "ID not found", 0);
 251         return -1;
 252 }
 253
 254 static int
 255 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
 256                              int fd)
 257 {
 258         struct pmd_internals *pmd = dev->data->dev_private;
 259         struct pmd_process_private *proc_private = dev->process_private;
 260         memif_msg_add_region_t *ar = &msg->add_region;
 261         struct memif_region *r;
 262
 263         if (fd < 0) {
 264                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
 265                 return -1;
 266         }
 267
 268         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
 269                         ar->index != proc_private->regions_num ||
 270                         proc_private->regions[ar->index] != NULL) {
 271                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
 272                 return -1;
 273         }
 274
 275         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 276         if (r == NULL) {
 277                 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
 278                 return -ENOMEM;
 279         }
 280
 281         r->fd = fd;
 282         r->region_size = ar->size;
 283         r->addr = NULL;
 284
 285         proc_private->regions[ar->index] = r;
 286         proc_private->regions_num++;
 287
 288         return 0;
 289 }
 290
 291 static int
 292 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
 293 {
 294         struct pmd_internals *pmd = dev->data->dev_private;
 295         memif_msg_add_ring_t *ar = &msg->add_ring;
 296         struct memif_queue *mq;
 297
 298         if (fd < 0) {
 299                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
 300                 return -1;
 301         }
 302
 303         /* check if we have enough queues */
 304         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) {
 305                 if (ar->index >= pmd->cfg.num_s2m_rings) {
 306                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 307                         return -1;
 308                 }
 309                 pmd->run.num_s2m_rings++;
 310         } else {
 311                 if (ar->index >= pmd->cfg.num_m2s_rings) {
 312                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 313                         return -1;
 314                 }
 315                 pmd->run.num_m2s_rings++;
 316         }
 317
 318         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ?
 319             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
 320
 321         mq->intr_handle.fd = fd;
 322         mq->log2_ring_size = ar->log2_ring_size;
 323         mq->region = ar->region;
 324         mq->ring_offset = ar->offset;
 325
 326         return 0;
 327 }
 328
 329 static int
 330 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
 331 {
 332         struct pmd_internals *pmd = dev->data->dev_private;
 333         memif_msg_connect_t *c = &msg->connect;
 334         int ret;
 335
 336         ret = memif_connect(dev);
 337         if (ret < 0)
 338                 return ret;
 339
 340         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 341                 sizeof(pmd->remote_if_name));
 342         MIF_LOG(INFO, "%s: Remote interface %s connected.",
 343                 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name);
 344
 345         return 0;
 346 }
 347
 348 static int
 349 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
 350 {
 351         struct pmd_internals *pmd = dev->data->dev_private;
 352         memif_msg_connected_t *c = &msg->connected;
 353         int ret;
 354
 355         ret = memif_connect(dev);
 356         if (ret < 0)
 357                 return ret;
 358
 359         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 360                 sizeof(pmd->remote_if_name));
 361         MIF_LOG(INFO, "%s: Remote interface %s connected.",
 362                 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name);
 363
 364         return 0;
 365 }
 366
 367 static int
 368 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
 369 {
 370         struct pmd_internals *pmd = dev->data->dev_private;
 371         memif_msg_disconnect_t *d = &msg->disconnect;
 372
 373         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
 374         strlcpy(pmd->remote_disc_string, (char *)d->string,
 375                 sizeof(pmd->remote_disc_string));
 376
 377         MIF_LOG(INFO, "%s: Disconnect received: %s",
 378                 rte_vdev_device_name(pmd->vdev), pmd->remote_disc_string);
 379
 380         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
 381         memif_disconnect(dev);
 382         return 0;
 383 }
 384
 385 static int
 386 memif_msg_enq_ack(struct rte_eth_dev *dev)
 387 {
 388         struct pmd_internals *pmd = dev->data->dev_private;
 389         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 390         if (e == NULL)
 391                 return -1;
 392
 393         e->msg.type = MEMIF_MSG_TYPE_ACK;
 394
 395         return 0;
 396 }
 397
 398 static int
 399 memif_msg_enq_init(struct rte_eth_dev *dev)
 400 {
 401         struct pmd_internals *pmd = dev->data->dev_private;
 402         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 403         memif_msg_init_t *i = &e->msg.init;
 404
 405         if (e == NULL)
 406                 return -1;
 407
 408         i = &e->msg.init;
 409         e->msg.type = MEMIF_MSG_TYPE_INIT;
 410         i->version = MEMIF_VERSION;
 411         i->id = pmd->id;
 412         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
 413
 414         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
 415
 416         if (*pmd->secret != '\0')
 417                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
 418
 419         return 0;
 420 }
 421
 422 static int
 423 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
 424 {
 425         struct pmd_internals *pmd = dev->data->dev_private;
 426         struct pmd_process_private *proc_private = dev->process_private;
 427         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 428         memif_msg_add_region_t *ar;
 429         struct memif_region *mr = proc_private->regions[idx];
 430
 431         if (e == NULL)
 432                 return -1;
 433
 434         ar = &e->msg.add_region;
 435         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
 436         e->fd = mr->fd;
 437         ar->index = idx;
 438         ar->size = mr->region_size;
 439
 440         return 0;
 441 }
 442
 443 static int
 444 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
 445                        memif_ring_type_t type)
 446 {
 447         struct pmd_internals *pmd = dev->data->dev_private;
 448         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 449         struct memif_queue *mq;
 450         memif_msg_add_ring_t *ar;
 451
 452         if (e == NULL)
 453                 return -1;
 454
 455         ar = &e->msg.add_ring;
 456         mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] :
 457             dev->data->rx_queues[idx];
 458
 459         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
 460         e->fd = mq->intr_handle.fd;
 461         ar->index = idx;
 462         ar->offset = mq->ring_offset;
 463         ar->region = mq->region;
 464         ar->log2_ring_size = mq->log2_ring_size;
 465         ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
 466         ar->private_hdr_size = 0;
 467
 468         return 0;
 469 }
 470
 471 static int
 472 memif_msg_enq_connect(struct rte_eth_dev *dev)
 473 {
 474         struct pmd_internals *pmd = dev->data->dev_private;
 475         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 476         const char *name = rte_vdev_device_name(pmd->vdev);
 477         memif_msg_connect_t *c;
 478
 479         if (e == NULL)
 480                 return -1;
 481
 482         c = &e->msg.connect;
 483         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
 484         strlcpy((char *)c->if_name, name, sizeof(c->if_name));
 485
 486         return 0;
 487 }
 488
 489 static int
 490 memif_msg_enq_connected(struct rte_eth_dev *dev)
 491 {
 492         struct pmd_internals *pmd = dev->data->dev_private;
 493         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 494         const char *name = rte_vdev_device_name(pmd->vdev);
 495         memif_msg_connected_t *c;
 496
 497         if (e == NULL)
 498                 return -1;
 499
 500         c = &e->msg.connected;
 501         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
 502         strlcpy((char *)c->if_name, name, sizeof(c->if_name));
 503
 504         return 0;
 505 }
 506
 507 static void
 508 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
 509 {
 510         struct memif_msg_queue_elt *elt;
 511         struct memif_control_channel *cc = arg;
 512
 513         /* close control channel fd */
 514         close(intr_handle->fd);
 515         /* clear message queue */
 516         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
 517                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
 518                 rte_free(elt);
 519         }
 520         /* free control channel */
 521         rte_free(cc);
 522 }
 523
 524 void
 525 memif_disconnect(struct rte_eth_dev *dev)
 526 {
 527         struct pmd_internals *pmd = dev->data->dev_private;
 528         struct pmd_process_private *proc_private = dev->process_private;
 529         struct memif_msg_queue_elt *elt, *next;
 530         struct memif_queue *mq;
 531         struct rte_intr_handle *ih;
 532         int i;
 533         int ret;
 534
 535         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 536         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
 537         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
 538
 539         if (pmd->cc != NULL) {
 540                 /* Clear control message queue (except disconnect message if any). */
 541                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
 542                         next = TAILQ_NEXT(elt, next);
 543                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
 544                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
 545                                 rte_free(elt);
 546                         }
 547                 }
 548                 /* send disconnect message (if there is any in queue) */
 549                 memif_msg_send_from_queue(pmd->cc);
 550
 551                 /* at this point, there should be no more messages in queue */
 552                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
 553                         MIF_LOG(WARNING,
 554                                 "Unexpected message(s) in message queue.");
 555                 }
 556
 557                 ih = &pmd->cc->intr_handle;
 558                 if (ih->fd > 0) {
 559                         ret = rte_intr_callback_unregister(ih,
 560                                                         memif_intr_handler,
 561                                                         pmd->cc);
 562                         /*
 563                          * If callback is active (disconnecting based on
 564                          * received control message).
 565                          */
 566                         if (ret == -EAGAIN) {
 567                                 ret = rte_intr_callback_unregister_pending(ih,
 568                                                         memif_intr_handler,
 569                                                         pmd->cc,
 570                                                         memif_intr_unregister_handler);
 571                         } else if (ret > 0) {
 572                                 close(ih->fd);
 573                                 rte_free(pmd->cc);
 574                         }
 575                         pmd->cc = NULL;
 576                         if (ret <= 0)
 577                                 MIF_LOG(WARNING,
 578                                         "Failed to unregister control channel callback.");
 579                 }
 580         }
 581
 582         /* unconfig interrupts */
 583         for (i = 0; i < pmd->cfg.num_s2m_rings; i++) {
 584                 if (pmd->role == MEMIF_ROLE_SLAVE) {
 585                         if (dev->data->tx_queues != NULL)
 586                                 mq = dev->data->tx_queues[i];
 587                         else
 588                                 continue;
 589                 } else {
 590                         if (dev->data->rx_queues != NULL)
 591                                 mq = dev->data->rx_queues[i];
 592                         else
 593                                 continue;
 594                 }
 595                 if (mq->intr_handle.fd > 0) {
 596                         close(mq->intr_handle.fd);
 597                         mq->intr_handle.fd = -1;
 598                 }
 599         }
 600         for (i = 0; i < pmd->cfg.num_m2s_rings; i++) {
 601                 if (pmd->role == MEMIF_ROLE_MASTER) {
 602                         if (dev->data->tx_queues != NULL)
 603                                 mq = dev->data->tx_queues[i];
 604                         else
 605                                 continue;
 606                 } else {
 607                         if (dev->data->rx_queues != NULL)
 608                                 mq = dev->data->rx_queues[i];
 609                         else
 610                                 continue;
 611                 }
 612                 if (mq->intr_handle.fd > 0) {
 613                         close(mq->intr_handle.fd);
 614                         mq->intr_handle.fd = -1;
 615                 }
 616         }
 617
 618         memif_free_regions(proc_private);
 619
 620         /* reset connection configuration */
 621         memset(&pmd->run, 0, sizeof(pmd->run));
 622
 623         MIF_LOG(DEBUG, "Disconnected.");
 624 }
 625
 626 static int
 627 memif_msg_receive(struct memif_control_channel *cc)
 628 {
 629         char ctl[CMSG_SPACE(sizeof(int)) +
 630                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
 631         struct msghdr mh = { 0 };
 632         struct iovec iov[1];
 633         memif_msg_t msg = { 0 };
 634         ssize_t size;
 635         int ret = 0;
 636         struct ucred *cr __rte_unused;
 637         cr = 0;
 638         struct cmsghdr *cmsg;
 639         int afd = -1;
 640         int i;
 641         struct pmd_internals *pmd;
 642         struct pmd_process_private *proc_private;
 643
 644         iov[0].iov_base = (void *)&msg;
 645         iov[0].iov_len = sizeof(memif_msg_t);
 646         mh.msg_iov = iov;
 647         mh.msg_iovlen = 1;
 648         mh.msg_control = ctl;
 649         mh.msg_controllen = sizeof(ctl);
 650
 651         size = recvmsg(cc->intr_handle.fd, &mh, 0);
 652         if (size != sizeof(memif_msg_t)) {
 653                 MIF_LOG(DEBUG, "Invalid message size.");
 654                 memif_msg_enq_disconnect(cc, "Invalid message size", 0);
 655                 return -1;
 656         }
 657         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
 658
 659         cmsg = CMSG_FIRSTHDR(&mh);
 660         while (cmsg) {
 661                 if (cmsg->cmsg_level == SOL_SOCKET) {
 662                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
 663                                 cr = (struct ucred *)CMSG_DATA(cmsg);
 664                         else if (cmsg->cmsg_type == SCM_RIGHTS)
 665                                 memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
 666                 }
 667                 cmsg = CMSG_NXTHDR(&mh, cmsg);
 668         }
 669
 670         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
 671                 MIF_LOG(DEBUG, "Unexpected message.");
 672                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
 673                 return -1;
 674         }
 675
 676         /* get device from hash data */
 677         switch (msg.type) {
 678         case MEMIF_MSG_TYPE_ACK:
 679                 break;
 680         case MEMIF_MSG_TYPE_HELLO:
 681                 ret = memif_msg_receive_hello(cc->dev, &msg);
 682                 if (ret < 0)
 683                         goto exit;
 684                 ret = memif_init_regions_and_queues(cc->dev);
 685                 if (ret < 0)
 686                         goto exit;
 687                 ret = memif_msg_enq_init(cc->dev);
 688                 if (ret < 0)
 689                         goto exit;
 690                 pmd = cc->dev->data->dev_private;
 691                 proc_private = cc->dev->process_private;
 692                 for (i = 0; i < proc_private->regions_num; i++) {
 693                         ret = memif_msg_enq_add_region(cc->dev, i);
 694                         if (ret < 0)
 695                                 goto exit;
 696                 }
 697                 for (i = 0; i < pmd->run.num_s2m_rings; i++) {
 698                         ret = memif_msg_enq_add_ring(cc->dev, i,
 699                                                      MEMIF_RING_S2M);
 700                         if (ret < 0)
 701                                 goto exit;
 702                 }
 703                 for (i = 0; i < pmd->run.num_m2s_rings; i++) {
 704                         ret = memif_msg_enq_add_ring(cc->dev, i,
 705                                                      MEMIF_RING_M2S);
 706                         if (ret < 0)
 707                                 goto exit;
 708                 }
 709                 ret = memif_msg_enq_connect(cc->dev);
 710                 if (ret < 0)
 711                         goto exit;
 712                 break;
 713         case MEMIF_MSG_TYPE_INIT:
 714                 /*
 715                  * This cc does not have an interface asociated with it.
 716                  * If suitable interface is found it will be assigned here.
 717                  */
 718                 ret = memif_msg_receive_init(cc, &msg);
 719                 if (ret < 0)
 720                         goto exit;
 721                 ret = memif_msg_enq_ack(cc->dev);
 722                 if (ret < 0)
 723                         goto exit;
 724                 break;
 725         case MEMIF_MSG_TYPE_ADD_REGION:
 726                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
 727                 if (ret < 0)
 728                         goto exit;
 729                 ret = memif_msg_enq_ack(cc->dev);
 730                 if (ret < 0)
 731                         goto exit;
 732                 break;
 733         case MEMIF_MSG_TYPE_ADD_RING:
 734                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
 735                 if (ret < 0)
 736                         goto exit;
 737                 ret = memif_msg_enq_ack(cc->dev);
 738                 if (ret < 0)
 739                         goto exit;
 740                 break;
 741         case MEMIF_MSG_TYPE_CONNECT:
 742                 ret = memif_msg_receive_connect(cc->dev, &msg);
 743                 if (ret < 0)
 744                         goto exit;
 745                 ret = memif_msg_enq_connected(cc->dev);
 746                 if (ret < 0)
 747                         goto exit;
 748                 break;
 749         case MEMIF_MSG_TYPE_CONNECTED:
 750                 ret = memif_msg_receive_connected(cc->dev, &msg);
 751                 break;
 752         case MEMIF_MSG_TYPE_DISCONNECT:
 753                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
 754                 if (ret < 0)
 755                         goto exit;
 756                 break;
 757         default:
 758                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
 759                 ret = -1;
 760                 goto exit;
 761         }
 762
 763  exit:
 764         return ret;
 765 }
 766
 767 static void
 768 memif_intr_handler(void *arg)
 769 {
 770         struct memif_control_channel *cc = arg;
 771         int ret;
 772
 773         ret = memif_msg_receive(cc);
 774         /* if driver failed to assign device */
 775         if (cc->dev == NULL) {
 776                 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
 777                                                            memif_intr_handler,
 778                                                            cc,
 779                                                            memif_intr_unregister_handler);
 780                 if (ret < 0)
 781                         MIF_LOG(WARNING,
 782                                 "Failed to unregister control channel callback.");
 783                 return;
 784         }
 785         /* if memif_msg_receive failed */
 786         if (ret < 0)
 787                 goto disconnect;
 788
 789         ret = memif_msg_send_from_queue(cc);
 790         if (ret < 0)
 791                 goto disconnect;
 792
 793         return;
 794
 795  disconnect:
 796         if (cc->dev == NULL) {
 797                 MIF_LOG(WARNING, "eth dev not allocated");
 798                 return;
 799         }
 800         memif_disconnect(cc->dev);
 801 }
 802
 803 static void
 804 memif_listener_handler(void *arg)
 805 {
 806         struct memif_socket *socket = arg;
 807         int sockfd;
 808         int addr_len;
 809         struct sockaddr_un client;
 810         struct memif_control_channel *cc;
 811         int ret;
 812
 813         addr_len = sizeof(client);
 814         sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
 815                         (socklen_t *)&addr_len);
 816         if (sockfd < 0) {
 817                 MIF_LOG(ERR,
 818                         "Failed to accept connection request on socket fd %d",
 819                         socket->intr_handle.fd);
 820                 return;
 821         }
 822
 823         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
 824
 825         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
 826         if (cc == NULL) {
 827                 MIF_LOG(ERR, "Failed to allocate control channel.");
 828                 goto error;
 829         }
 830
 831         cc->intr_handle.fd = sockfd;
 832         cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
 833         cc->socket = socket;
 834         cc->dev = NULL;
 835         TAILQ_INIT(&cc->msg_queue);
 836
 837         ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
 838         if (ret < 0) {
 839                 MIF_LOG(ERR, "Failed to register control channel callback.");
 840                 goto error;
 841         }
 842
 843         ret = memif_msg_enq_hello(cc);
 844         if (ret < 0) {
 845                 MIF_LOG(ERR, "Failed to enqueue hello message.");
 846                 goto error;
 847         }
 848         ret = memif_msg_send_from_queue(cc);
 849         if (ret < 0)
 850                 goto error;
 851
 852         return;
 853
 854  error:
 855         if (sockfd >= 0) {
 856                 close(sockfd);
 857                 sockfd = -1;
 858         }
 859         if (cc != NULL)
 860                 rte_free(cc);
 861 }
 862
 863 static struct memif_socket *
 864 memif_socket_create(struct pmd_internals *pmd, char *key, uint8_t listener)
 865 {
 866         struct memif_socket *sock;
 867         struct sockaddr_un un;
 868         int sockfd;
 869         int ret;
 870         int on = 1;
 871
 872         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
 873         if (sock == NULL) {
 874                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
 875                 return NULL;
 876         }
 877
 878         sock->listener = listener;
 879         rte_memcpy(sock->filename, key, 256);
 880         TAILQ_INIT(&sock->dev_queue);
 881
 882         if (listener != 0) {
 883                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
 884                 if (sockfd < 0)
 885                         goto error;
 886
 887                 un.sun_family = AF_UNIX;
 888                 memcpy(un.sun_path, sock->filename,
 889                         sizeof(un.sun_path) - 1);
 890
 891                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
 892                                  sizeof(on));
 893                 if (ret < 0)
 894                         goto error;
 895                 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un));
 896                 if (ret < 0)
 897                         goto error;
 898                 ret = listen(sockfd, 1);
 899                 if (ret < 0)
 900                         goto error;
 901
 902                 MIF_LOG(DEBUG, "%s: Memif listener socket %s created.",
 903                         rte_vdev_device_name(pmd->vdev), sock->filename);
 904
 905                 sock->intr_handle.fd = sockfd;
 906                 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
 907                 ret = rte_intr_callback_register(&sock->intr_handle,
 908                                                  memif_listener_handler, sock);
 909                 if (ret < 0) {
 910                         MIF_LOG(ERR, "%s: Failed to register interrupt "
 911                                 "callback for listener socket",
 912                                 rte_vdev_device_name(pmd->vdev));
 913                         return NULL;
 914                 }
 915         }
 916
 917         return sock;
 918
 919  error:
 920         MIF_LOG(ERR, "%s: Failed to setup socket %s: %s",
 921                 rte_vdev_device_name(pmd->vdev) ?
 922                 rte_vdev_device_name(pmd->vdev) : "NULL", key, strerror(errno));
 923         if (sock != NULL)
 924                 rte_free(sock);
 925         if (sockfd >= 0)
 926                 close(sockfd);
 927         return NULL;
 928 }
 929
 930 static struct rte_hash *
 931 memif_create_socket_hash(void)
 932 {
 933         struct rte_hash_parameters params = { 0 };
 934         params.name = MEMIF_SOCKET_HASH_NAME;
 935         params.entries = 256;
 936         params.key_len = 256;
 937         params.hash_func = rte_jhash;
 938         params.hash_func_init_val = 0;
 939         return rte_hash_create(&params);
 940 }
 941
 942 int
 943 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
 944 {
 945         struct pmd_internals *pmd = dev->data->dev_private;
 946         struct memif_socket *socket = NULL;
 947         struct memif_socket_dev_list_elt *elt;
 948         struct pmd_internals *tmp_pmd;
 949         struct rte_hash *hash;
 950         int ret;
 951         char key[256];
 952
 953         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
 954         if (hash == NULL) {
 955                 hash = memif_create_socket_hash();
 956                 if (hash == NULL) {
 957                         MIF_LOG(ERR, "Failed to create memif socket hash.");
 958                         return -1;
 959                 }
 960         }
 961
 962         memset(key, 0, 256);
 963         rte_memcpy(key, socket_filename, strlen(socket_filename));
 964         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
 965         if (ret < 0) {
 966                 socket = memif_socket_create(pmd, key,
 967                                              (pmd->role ==
 968                                               MEMIF_ROLE_SLAVE) ? 0 : 1);
 969                 if (socket == NULL)
 970                         return -1;
 971                 ret = rte_hash_add_key_data(hash, key, socket);
 972                 if (ret < 0) {
 973                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
 974                         return ret;
 975                 }
 976         }
 977         pmd->socket_filename = socket->filename;
 978
 979         if (socket->listener != 0 && pmd->role == MEMIF_ROLE_SLAVE) {
 980                 MIF_LOG(ERR, "Socket is a listener.");
 981                 return -1;
 982         } else if ((socket->listener == 0) && (pmd->role == MEMIF_ROLE_MASTER)) {
 983                 MIF_LOG(ERR, "Socket is not a listener.");
 984                 return -1;
 985         }
 986
 987         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
 988                 tmp_pmd = elt->dev->data->dev_private;
 989                 if (tmp_pmd->id == pmd->id) {
 990                         MIF_LOG(ERR, "Memif device with id %d already "
 991                                 "exists on socket %s",
 992                                 pmd->id, socket->filename);
 993                         return -1;
 994                 }
 995         }
 996
 997         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
 998         if (elt == NULL) {
 999                 MIF_LOG(ERR, "%s: Failed to add device to socket device list.",
1000                         rte_vdev_device_name(pmd->vdev));
1001                 return -1;
1002         }
1003         elt->dev = dev;
1004         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1005
1006         return 0;
1007 }
1008
1009 void
1010 memif_socket_remove_device(struct rte_eth_dev *dev)
1011 {
1012         struct pmd_internals *pmd = dev->data->dev_private;
1013         struct memif_socket *socket = NULL;
1014         struct memif_socket_dev_list_elt *elt, *next;
1015         struct rte_hash *hash;
1016         int ret;
1017
1018         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1019         if (hash == NULL)
1020                 return;
1021
1022         if (pmd->socket_filename == NULL)
1023                 return;
1024
1025         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1026                 return;
1027
1028         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1029                 next = TAILQ_NEXT(elt, next);
1030                 if (elt->dev == dev) {
1031                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1032                         rte_free(elt);
1033                         pmd->socket_filename = NULL;
1034                 }
1035         }
1036
1037         /* remove socket, if this was the last device using it */
1038         if (TAILQ_EMPTY(&socket->dev_queue)) {
1039                 rte_hash_del_key(hash, socket->filename);
1040                 if (socket->listener) {
1041                         /* remove listener socket file,
1042                          * so we can create new one later.
1043                          */
1044                         ret = remove(socket->filename);
1045                         if (ret < 0)
1046                                 MIF_LOG(ERR, "Failed to remove socket file: %s",
1047                                         socket->filename);
1048                 }
1049                 rte_free(socket);
1050         }
1051 }
1052
1053 int
1054 memif_connect_master(struct rte_eth_dev *dev)
1055 {
1056         struct pmd_internals *pmd = dev->data->dev_private;
1057
1058         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1059         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1060         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1061         return 0;
1062 }
1063
1064 int
1065 memif_connect_slave(struct rte_eth_dev *dev)
1066 {
1067         int sockfd;
1068         int ret;
1069         struct sockaddr_un sun;
1070         struct pmd_internals *pmd = dev->data->dev_private;
1071
1072         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1073         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1074         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1075
1076         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1077         if (sockfd < 0) {
1078                 MIF_LOG(ERR, "%s: Failed to open socket.",
1079                         rte_vdev_device_name(pmd->vdev));
1080                 return -1;
1081         }
1082
1083         sun.sun_family = AF_UNIX;
1084
1085         memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1);
1086
1087         ret = connect(sockfd, (struct sockaddr *)&sun,
1088                       sizeof(struct sockaddr_un));
1089         if (ret < 0) {
1090                 MIF_LOG(ERR, "%s: Failed to connect socket: %s.",
1091                         rte_vdev_device_name(pmd->vdev), pmd->socket_filename);
1092                 goto error;
1093         }
1094
1095         MIF_LOG(DEBUG, "%s: Memif socket: %s connected.",
1096                 rte_vdev_device_name(pmd->vdev), pmd->socket_filename);
1097
1098         pmd->cc = rte_zmalloc("memif-cc",
1099                               sizeof(struct memif_control_channel), 0);
1100         if (pmd->cc == NULL) {
1101                 MIF_LOG(ERR, "%s: Failed to allocate control channel.",
1102                         rte_vdev_device_name(pmd->vdev));
1103                 goto error;
1104         }
1105
1106         pmd->cc->intr_handle.fd = sockfd;
1107         pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1108         pmd->cc->socket = NULL;
1109         pmd->cc->dev = dev;
1110         TAILQ_INIT(&pmd->cc->msg_queue);
1111
1112         ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1113                                          memif_intr_handler, pmd->cc);
1114         if (ret < 0) {
1115                 MIF_LOG(ERR, "%s: Failed to register interrupt callback "
1116                         "for control fd", rte_vdev_device_name(pmd->vdev));
1117                 goto error;
1118         }
1119
1120         return 0;
1121
1122  error:
1123         if (sockfd >= 0) {
1124                 close(sockfd);
1125                 sockfd = -1;
1126         }
1127         if (pmd->cc != NULL) {
1128                 rte_free(pmd->cc);
1129                 pmd->cc = NULL;
1130         }
1131         return -1;
1132 }