drivers/net/memif/memif_socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/ioctl.h>
  11 #include <errno.h>
  12
  13 #include <rte_version.h>
  14 #include <rte_mbuf.h>
  15 #include <rte_ether.h>
  16 #include <ethdev_driver.h>
  17 #include <ethdev_vdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_bus_vdev.h>
  21 #include <rte_hash.h>
  22 #include <rte_jhash.h>
  23 #include <rte_string_fns.h>
  24
  25 #include "rte_eth_memif.h"
  26 #include "memif_socket.h"
  27
  28 static void memif_intr_handler(void *arg);
  29
  30 static ssize_t
  31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
  32 {
  33         struct msghdr mh = { 0 };
  34         struct iovec iov[1];
  35         struct cmsghdr *cmsg;
  36         char ctl[CMSG_SPACE(sizeof(int))];
  37
  38         iov[0].iov_base = msg;
  39         iov[0].iov_len = sizeof(memif_msg_t);
  40         mh.msg_iov = iov;
  41         mh.msg_iovlen = 1;
  42
  43         if (afd > 0) {
  44                 memset(&ctl, 0, sizeof(ctl));
  45                 mh.msg_control = ctl;
  46                 mh.msg_controllen = sizeof(ctl);
  47                 cmsg = CMSG_FIRSTHDR(&mh);
  48                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  49                 cmsg->cmsg_level = SOL_SOCKET;
  50                 cmsg->cmsg_type = SCM_RIGHTS;
  51                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
  52         }
  53
  54         return sendmsg(fd, &mh, 0);
  55 }
  56
  57 static int
  58 memif_msg_send_from_queue(struct memif_control_channel *cc)
  59 {
  60         ssize_t size;
  61         int ret = 0;
  62         struct memif_msg_queue_elt *e;
  63
  64         e = TAILQ_FIRST(&cc->msg_queue);
  65         if (e == NULL)
  66                 return 0;
  67
  68         size = memif_msg_send(rte_intr_fd_get(cc->intr_handle), &e->msg,
  69                               e->fd);
  70         if (size != sizeof(memif_msg_t)) {
  71                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
  72                 ret = -1;
  73         } else {
  74                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
  75         }
  76         TAILQ_REMOVE(&cc->msg_queue, e, next);
  77         rte_free(e);
  78
  79         return ret;
  80 }
  81
  82 static struct memif_msg_queue_elt *
  83 memif_msg_enq(struct memif_control_channel *cc)
  84 {
  85         struct memif_msg_queue_elt *e;
  86
  87         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
  88         if (e == NULL) {
  89                 MIF_LOG(ERR, "Failed to allocate control message.");
  90                 return NULL;
  91         }
  92
  93         e->fd = -1;
  94         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
  95
  96         return e;
  97 }
  98
  99 void
 100 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
 101                          int err_code)
 102 {
 103         struct memif_msg_queue_elt *e;
 104         struct pmd_internals *pmd;
 105         memif_msg_disconnect_t *d;
 106
 107         if (cc == NULL) {
 108                 MIF_LOG(DEBUG, "Missing control channel.");
 109                 return;
 110         }
 111
 112         e = memif_msg_enq(cc);
 113         if (e == NULL) {
 114                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
 115                 return;
 116         }
 117
 118         d = &e->msg.disconnect;
 119
 120         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
 121         d->code = err_code;
 122
 123         if (reason != NULL) {
 124                 strlcpy((char *)d->string, reason, sizeof(d->string));
 125                 if (cc->dev != NULL) {
 126                         pmd = cc->dev->data->dev_private;
 127                         strlcpy(pmd->local_disc_string, reason,
 128                                 sizeof(pmd->local_disc_string));
 129                 }
 130         }
 131 }
 132
 133 static int
 134 memif_msg_enq_hello(struct memif_control_channel *cc)
 135 {
 136         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
 137         memif_msg_hello_t *h;
 138
 139         if (e == NULL)
 140                 return -1;
 141
 142         h = &e->msg.hello;
 143
 144         e->msg.type = MEMIF_MSG_TYPE_HELLO;
 145         h->min_version = MEMIF_VERSION;
 146         h->max_version = MEMIF_VERSION;
 147         h->max_c2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 148         h->max_s2c_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 149         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
 150         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
 151
 152         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
 153
 154         return 0;
 155 }
 156
 157 static int
 158 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
 159 {
 160         struct pmd_internals *pmd = dev->data->dev_private;
 161         memif_msg_hello_t *h = &msg->hello;
 162
 163         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
 164                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
 165                 return -1;
 166         }
 167
 168         /* Set parameters for active connection */
 169         pmd->run.num_c2s_rings = RTE_MIN(h->max_c2s_ring + 1,
 170                                            pmd->cfg.num_c2s_rings);
 171         pmd->run.num_s2c_rings = RTE_MIN(h->max_s2c_ring + 1,
 172                                            pmd->cfg.num_s2c_rings);
 173         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
 174                                             pmd->cfg.log2_ring_size);
 175         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
 176
 177         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
 178
 179         MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name);
 180
 181         return 0;
 182 }
 183
 184 static int
 185 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
 186 {
 187         memif_msg_init_t *i = &msg->init;
 188         struct memif_socket_dev_list_elt *elt;
 189         struct pmd_internals *pmd;
 190         struct rte_eth_dev *dev;
 191
 192         if (i->version != MEMIF_VERSION) {
 193                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
 194                 return -1;
 195         }
 196
 197         if (cc->socket == NULL) {
 198                 memif_msg_enq_disconnect(cc, "Device error", 0);
 199                 return -1;
 200         }
 201
 202         /* Find device with requested ID */
 203         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
 204                 dev = elt->dev;
 205                 pmd = dev->data->dev_private;
 206                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
 207                     (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_SERVER)) {
 208                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
 209                                            ETH_MEMIF_FLAG_CONNECTED)) {
 210                                 memif_msg_enq_disconnect(cc,
 211                                                          "Already connected", 0);
 212                                 return -1;
 213                         }
 214
 215                         /* assign control channel to device */
 216                         cc->dev = dev;
 217                         pmd->cc = cc;
 218
 219                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
 220                                 memif_msg_enq_disconnect(pmd->cc,
 221                                                          "Only ethernet mode supported",
 222                                                          0);
 223                                 return -1;
 224                         }
 225
 226                         strlcpy(pmd->remote_name, (char *)i->name,
 227                                 sizeof(pmd->remote_name));
 228
 229                         if (*pmd->secret != '\0') {
 230                                 if (*i->secret == '\0') {
 231                                         memif_msg_enq_disconnect(pmd->cc,
 232                                                                  "Secret required", 0);
 233                                         return -1;
 234                                 }
 235                                 if (strncmp(pmd->secret, (char *)i->secret,
 236                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
 237                                         memif_msg_enq_disconnect(pmd->cc,
 238                                                                  "Incorrect secret", 0);
 239                                         return -1;
 240                                 }
 241                         }
 242
 243                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
 244                         return 0;
 245                 }
 246         }
 247
 248         /* ID not found on this socket */
 249         MIF_LOG(DEBUG, "ID %u not found.", i->id);
 250         memif_msg_enq_disconnect(cc, "ID not found", 0);
 251         return -1;
 252 }
 253
 254 static int
 255 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
 256                              int fd)
 257 {
 258         struct pmd_internals *pmd = dev->data->dev_private;
 259         struct pmd_process_private *proc_private = dev->process_private;
 260         memif_msg_add_region_t *ar = &msg->add_region;
 261         struct memif_region *r;
 262
 263         if (fd < 0) {
 264                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
 265                 return -1;
 266         }
 267
 268         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
 269                         ar->index != proc_private->regions_num ||
 270                         proc_private->regions[ar->index] != NULL) {
 271                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
 272                 return -1;
 273         }
 274
 275         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 276         if (r == NULL) {
 277                 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
 278                 return -ENOMEM;
 279         }
 280
 281         r->fd = fd;
 282         r->region_size = ar->size;
 283         r->addr = NULL;
 284
 285         proc_private->regions[ar->index] = r;
 286         proc_private->regions_num++;
 287
 288         return 0;
 289 }
 290
 291 static int
 292 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
 293 {
 294         struct pmd_internals *pmd = dev->data->dev_private;
 295         memif_msg_add_ring_t *ar = &msg->add_ring;
 296         struct memif_queue *mq;
 297
 298         if (fd < 0) {
 299                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
 300                 return -1;
 301         }
 302
 303         /* check if we have enough queues */
 304         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) {
 305                 if (ar->index >= pmd->cfg.num_c2s_rings) {
 306                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 307                         return -1;
 308                 }
 309                 pmd->run.num_c2s_rings++;
 310         } else {
 311                 if (ar->index >= pmd->cfg.num_s2c_rings) {
 312                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 313                         return -1;
 314                 }
 315                 pmd->run.num_s2c_rings++;
 316         }
 317
 318         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) ?
 319             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
 320
 321         if (rte_intr_fd_set(mq->intr_handle, fd))
 322                 return -1;
 323
 324         mq->log2_ring_size = ar->log2_ring_size;
 325         mq->region = ar->region;
 326         mq->ring_offset = ar->offset;
 327
 328         return 0;
 329 }
 330
 331 static int
 332 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
 333 {
 334         struct pmd_internals *pmd = dev->data->dev_private;
 335         memif_msg_connect_t *c = &msg->connect;
 336         int ret;
 337
 338         ret = memif_connect(dev);
 339         if (ret < 0)
 340                 return ret;
 341
 342         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 343                 sizeof(pmd->remote_if_name));
 344         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 345
 346         return 0;
 347 }
 348
 349 static int
 350 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
 351 {
 352         struct pmd_internals *pmd = dev->data->dev_private;
 353         memif_msg_connected_t *c = &msg->connected;
 354         int ret;
 355
 356         ret = memif_connect(dev);
 357         if (ret < 0)
 358                 return ret;
 359
 360         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 361                 sizeof(pmd->remote_if_name));
 362         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 363
 364         return 0;
 365 }
 366
 367 static int
 368 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
 369 {
 370         struct pmd_internals *pmd = dev->data->dev_private;
 371         memif_msg_disconnect_t *d = &msg->disconnect;
 372
 373         memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string));
 374         strlcpy(pmd->remote_disc_string, (char *)d->string,
 375                 sizeof(pmd->remote_disc_string));
 376
 377         MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string);
 378
 379         memset(pmd->local_disc_string, 0, 96);
 380         memif_disconnect(dev);
 381         return 0;
 382 }
 383
 384 static int
 385 memif_msg_enq_ack(struct rte_eth_dev *dev)
 386 {
 387         struct pmd_internals *pmd = dev->data->dev_private;
 388         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 389         if (e == NULL)
 390                 return -1;
 391
 392         e->msg.type = MEMIF_MSG_TYPE_ACK;
 393
 394         return 0;
 395 }
 396
 397 static int
 398 memif_msg_enq_init(struct rte_eth_dev *dev)
 399 {
 400         struct pmd_internals *pmd = dev->data->dev_private;
 401         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 402         memif_msg_init_t *i = &e->msg.init;
 403
 404         if (e == NULL)
 405                 return -1;
 406
 407         i = &e->msg.init;
 408         e->msg.type = MEMIF_MSG_TYPE_INIT;
 409         i->version = MEMIF_VERSION;
 410         i->id = pmd->id;
 411         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
 412
 413         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
 414
 415         if (*pmd->secret != '\0')
 416                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
 417
 418         return 0;
 419 }
 420
 421 static int
 422 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
 423 {
 424         struct pmd_internals *pmd = dev->data->dev_private;
 425         struct pmd_process_private *proc_private = dev->process_private;
 426         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 427         memif_msg_add_region_t *ar;
 428         struct memif_region *mr = proc_private->regions[idx];
 429
 430         if (e == NULL)
 431                 return -1;
 432
 433         ar = &e->msg.add_region;
 434         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
 435         e->fd = mr->fd;
 436         ar->index = idx;
 437         ar->size = mr->region_size;
 438
 439         return 0;
 440 }
 441
 442 static int
 443 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
 444                        memif_ring_type_t type)
 445 {
 446         struct pmd_internals *pmd = dev->data->dev_private;
 447         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 448         struct memif_queue *mq;
 449         memif_msg_add_ring_t *ar;
 450
 451         if (e == NULL)
 452                 return -1;
 453
 454         ar = &e->msg.add_ring;
 455         mq = (type == MEMIF_RING_C2S) ? dev->data->tx_queues[idx] :
 456             dev->data->rx_queues[idx];
 457
 458         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
 459         e->fd = rte_intr_fd_get(mq->intr_handle);
 460         ar->index = idx;
 461         ar->offset = mq->ring_offset;
 462         ar->region = mq->region;
 463         ar->log2_ring_size = mq->log2_ring_size;
 464         ar->flags = (type == MEMIF_RING_C2S) ? MEMIF_MSG_ADD_RING_FLAG_C2S : 0;
 465         ar->private_hdr_size = 0;
 466
 467         return 0;
 468 }
 469
 470 static int
 471 memif_msg_enq_connect(struct rte_eth_dev *dev)
 472 {
 473         struct pmd_internals *pmd = dev->data->dev_private;
 474         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 475         memif_msg_connect_t *c;
 476
 477         if (e == NULL)
 478                 return -1;
 479
 480         c = &e->msg.connect;
 481         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
 482         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 483
 484         return 0;
 485 }
 486
 487 static int
 488 memif_msg_enq_connected(struct rte_eth_dev *dev)
 489 {
 490         struct pmd_internals *pmd = dev->data->dev_private;
 491         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 492         memif_msg_connected_t *c;
 493
 494         if (e == NULL)
 495                 return -1;
 496
 497         c = &e->msg.connected;
 498         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
 499         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 500
 501         return 0;
 502 }
 503
 504 static void
 505 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
 506 {
 507         struct memif_msg_queue_elt *elt;
 508         struct memif_control_channel *cc = arg;
 509
 510         /* close control channel fd */
 511         close(rte_intr_fd_get(intr_handle));
 512         /* clear message queue */
 513         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
 514                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
 515                 rte_free(elt);
 516         }
 517         rte_intr_instance_free(cc->intr_handle);
 518         /* free control channel */
 519         rte_free(cc);
 520 }
 521
 522 void
 523 memif_disconnect(struct rte_eth_dev *dev)
 524 {
 525         struct pmd_internals *pmd = dev->data->dev_private;
 526         struct memif_msg_queue_elt *elt, *next;
 527         struct memif_queue *mq;
 528         struct rte_intr_handle *ih;
 529         int i;
 530         int ret;
 531
 532         dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
 533         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
 534         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
 535
 536         rte_spinlock_lock(&pmd->cc_lock);
 537         if (pmd->cc != NULL) {
 538                 /* Clear control message queue (except disconnect message if any). */
 539                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
 540                         next = TAILQ_NEXT(elt, next);
 541                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
 542                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
 543                                 rte_free(elt);
 544                         }
 545                 }
 546                 /* send disconnect message (if there is any in queue) */
 547                 memif_msg_send_from_queue(pmd->cc);
 548
 549                 /* at this point, there should be no more messages in queue */
 550                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
 551                         MIF_LOG(WARNING,
 552                                 "Unexpected message(s) in message queue.");
 553                 }
 554
 555                 ih = pmd->cc->intr_handle;
 556                 if (rte_intr_fd_get(ih) > 0) {
 557                         ret = rte_intr_callback_unregister(ih,
 558                                                         memif_intr_handler,
 559                                                         pmd->cc);
 560                         /*
 561                          * If callback is active (disconnecting based on
 562                          * received control message).
 563                          */
 564                         if (ret == -EAGAIN) {
 565                                 ret = rte_intr_callback_unregister_pending(ih,
 566                                                         memif_intr_handler,
 567                                                         pmd->cc,
 568                                                         memif_intr_unregister_handler);
 569                         } else if (ret > 0) {
 570                                 close(rte_intr_fd_get(ih));
 571                                 rte_intr_instance_free(ih);
 572                                 rte_free(pmd->cc);
 573                         }
 574                         pmd->cc = NULL;
 575                         if (ret <= 0)
 576                                 MIF_LOG(WARNING,
 577                                         "Failed to unregister control channel callback.");
 578                 }
 579         }
 580         rte_spinlock_unlock(&pmd->cc_lock);
 581
 582         /* unconfig interrupts */
 583         for (i = 0; i < pmd->cfg.num_c2s_rings; i++) {
 584                 if (pmd->role == MEMIF_ROLE_CLIENT) {
 585                         if (dev->data->tx_queues != NULL)
 586                                 mq = dev->data->tx_queues[i];
 587                         else
 588                                 continue;
 589                 } else {
 590                         if (dev->data->rx_queues != NULL)
 591                                 mq = dev->data->rx_queues[i];
 592                         else
 593                                 continue;
 594                 }
 595
 596                 if (rte_intr_fd_get(mq->intr_handle) > 0) {
 597                         close(rte_intr_fd_get(mq->intr_handle));
 598                         rte_intr_fd_set(mq->intr_handle, -1);
 599                 }
 600         }
 601         for (i = 0; i < pmd->cfg.num_s2c_rings; i++) {
 602                 if (pmd->role == MEMIF_ROLE_SERVER) {
 603                         if (dev->data->tx_queues != NULL)
 604                                 mq = dev->data->tx_queues[i];
 605                         else
 606                                 continue;
 607                 } else {
 608                         if (dev->data->rx_queues != NULL)
 609                                 mq = dev->data->rx_queues[i];
 610                         else
 611                                 continue;
 612                 }
 613
 614                 if (rte_intr_fd_get(mq->intr_handle) > 0) {
 615                         close(rte_intr_fd_get(mq->intr_handle));
 616                         rte_intr_fd_set(mq->intr_handle, -1);
 617                 }
 618         }
 619
 620         memif_free_regions(dev);
 621
 622         /* reset connection configuration */
 623         memset(&pmd->run, 0, sizeof(pmd->run));
 624
 625         MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id,
 626                 (pmd->role == MEMIF_ROLE_SERVER) ? "server" : "client");
 627 }
 628
 629 static int
 630 memif_msg_receive(struct memif_control_channel *cc)
 631 {
 632         char ctl[CMSG_SPACE(sizeof(int)) +
 633                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
 634         struct msghdr mh = { 0 };
 635         struct iovec iov[1];
 636         memif_msg_t msg = { 0 };
 637         ssize_t size;
 638         int ret = 0;
 639         struct ucred *cr __rte_unused;
 640         cr = 0;
 641         struct cmsghdr *cmsg;
 642         int afd = -1;
 643         int i;
 644         struct pmd_internals *pmd;
 645         struct pmd_process_private *proc_private;
 646
 647         iov[0].iov_base = (void *)&msg;
 648         iov[0].iov_len = sizeof(memif_msg_t);
 649         mh.msg_iov = iov;
 650         mh.msg_iovlen = 1;
 651         mh.msg_control = ctl;
 652         mh.msg_controllen = sizeof(ctl);
 653
 654         size = recvmsg(rte_intr_fd_get(cc->intr_handle), &mh, 0);
 655         if (size != sizeof(memif_msg_t)) {
 656                 MIF_LOG(DEBUG, "Invalid message size = %zd", size);
 657                 if (size > 0)
 658                         /* 0 means end-of-file, negative size means error,
 659                          * don't send further disconnect message in such cases.
 660                          */
 661                         memif_msg_enq_disconnect(cc, "Invalid message size", 0);
 662                 return -1;
 663         }
 664         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
 665
 666         cmsg = CMSG_FIRSTHDR(&mh);
 667         while (cmsg) {
 668                 if (cmsg->cmsg_level == SOL_SOCKET) {
 669                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
 670                                 cr = (struct ucred *)CMSG_DATA(cmsg);
 671                         else if (cmsg->cmsg_type == SCM_RIGHTS)
 672                                 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
 673                 }
 674                 cmsg = CMSG_NXTHDR(&mh, cmsg);
 675         }
 676
 677         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
 678                 MIF_LOG(DEBUG, "Unexpected message.");
 679                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
 680                 return -1;
 681         }
 682
 683         /* get device from hash data */
 684         switch (msg.type) {
 685         case MEMIF_MSG_TYPE_ACK:
 686                 break;
 687         case MEMIF_MSG_TYPE_HELLO:
 688                 ret = memif_msg_receive_hello(cc->dev, &msg);
 689                 if (ret < 0)
 690                         goto exit;
 691                 ret = memif_init_regions_and_queues(cc->dev);
 692                 if (ret < 0)
 693                         goto exit;
 694                 ret = memif_msg_enq_init(cc->dev);
 695                 if (ret < 0)
 696                         goto exit;
 697                 pmd = cc->dev->data->dev_private;
 698                 proc_private = cc->dev->process_private;
 699                 for (i = 0; i < proc_private->regions_num; i++) {
 700                         ret = memif_msg_enq_add_region(cc->dev, i);
 701                         if (ret < 0)
 702                                 goto exit;
 703                 }
 704                 for (i = 0; i < pmd->run.num_c2s_rings; i++) {
 705                         ret = memif_msg_enq_add_ring(cc->dev, i,
 706                                                      MEMIF_RING_C2S);
 707                         if (ret < 0)
 708                                 goto exit;
 709                 }
 710                 for (i = 0; i < pmd->run.num_s2c_rings; i++) {
 711                         ret = memif_msg_enq_add_ring(cc->dev, i,
 712                                                      MEMIF_RING_S2C);
 713                         if (ret < 0)
 714                                 goto exit;
 715                 }
 716                 ret = memif_msg_enq_connect(cc->dev);
 717                 if (ret < 0)
 718                         goto exit;
 719                 break;
 720         case MEMIF_MSG_TYPE_INIT:
 721                 /*
 722                  * This cc does not have an interface asociated with it.
 723                  * If suitable interface is found it will be assigned here.
 724                  */
 725                 ret = memif_msg_receive_init(cc, &msg);
 726                 if (ret < 0)
 727                         goto exit;
 728                 ret = memif_msg_enq_ack(cc->dev);
 729                 if (ret < 0)
 730                         goto exit;
 731                 break;
 732         case MEMIF_MSG_TYPE_ADD_REGION:
 733                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
 734                 if (ret < 0)
 735                         goto exit;
 736                 ret = memif_msg_enq_ack(cc->dev);
 737                 if (ret < 0)
 738                         goto exit;
 739                 break;
 740         case MEMIF_MSG_TYPE_ADD_RING:
 741                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
 742                 if (ret < 0)
 743                         goto exit;
 744                 ret = memif_msg_enq_ack(cc->dev);
 745                 if (ret < 0)
 746                         goto exit;
 747                 break;
 748         case MEMIF_MSG_TYPE_CONNECT:
 749                 ret = memif_msg_receive_connect(cc->dev, &msg);
 750                 if (ret < 0)
 751                         goto exit;
 752                 ret = memif_msg_enq_connected(cc->dev);
 753                 if (ret < 0)
 754                         goto exit;
 755                 break;
 756         case MEMIF_MSG_TYPE_CONNECTED:
 757                 ret = memif_msg_receive_connected(cc->dev, &msg);
 758                 break;
 759         case MEMIF_MSG_TYPE_DISCONNECT:
 760                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
 761                 if (ret < 0)
 762                         goto exit;
 763                 break;
 764         default:
 765                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
 766                 ret = -1;
 767                 goto exit;
 768         }
 769
 770  exit:
 771         return ret;
 772 }
 773
 774 static void
 775 memif_intr_handler(void *arg)
 776 {
 777         struct memif_control_channel *cc = arg;
 778         int ret;
 779
 780         ret = memif_msg_receive(cc);
 781         /* if driver failed to assign device */
 782         if (cc->dev == NULL) {
 783                 memif_msg_send_from_queue(cc);
 784                 ret = rte_intr_callback_unregister_pending(cc->intr_handle,
 785                                                            memif_intr_handler,
 786                                                            cc,
 787                                                            memif_intr_unregister_handler);
 788                 if (ret < 0)
 789                         MIF_LOG(WARNING,
 790                                 "Failed to unregister control channel callback.");
 791                 return;
 792         }
 793         /* if memif_msg_receive failed */
 794         if (ret < 0)
 795                 goto disconnect;
 796
 797         ret = memif_msg_send_from_queue(cc);
 798         if (ret < 0)
 799                 goto disconnect;
 800
 801         return;
 802
 803  disconnect:
 804         if (cc->dev == NULL) {
 805                 MIF_LOG(WARNING, "eth dev not allocated");
 806                 return;
 807         }
 808         memif_disconnect(cc->dev);
 809 }
 810
 811 static void
 812 memif_listener_handler(void *arg)
 813 {
 814         struct memif_socket *socket = arg;
 815         int sockfd;
 816         int addr_len;
 817         struct sockaddr_un client;
 818         struct memif_control_channel *cc;
 819         int ret;
 820
 821         addr_len = sizeof(client);
 822         sockfd = accept(rte_intr_fd_get(socket->intr_handle),
 823                         (struct sockaddr *)&client, (socklen_t *)&addr_len);
 824         if (sockfd < 0) {
 825                 MIF_LOG(ERR,
 826                         "Failed to accept connection request on socket fd %d",
 827                         rte_intr_fd_get(socket->intr_handle));
 828                 return;
 829         }
 830
 831         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
 832
 833         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
 834         if (cc == NULL) {
 835                 MIF_LOG(ERR, "Failed to allocate control channel.");
 836                 goto error;
 837         }
 838
 839         /* Allocate interrupt instance */
 840         cc->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
 841         if (cc->intr_handle == NULL) {
 842                 MIF_LOG(ERR, "Failed to allocate intr handle");
 843                 goto error;
 844         }
 845
 846         if (rte_intr_fd_set(cc->intr_handle, sockfd))
 847                 goto error;
 848
 849         if (rte_intr_type_set(cc->intr_handle, RTE_INTR_HANDLE_EXT))
 850                 goto error;
 851
 852         cc->socket = socket;
 853         cc->dev = NULL;
 854         TAILQ_INIT(&cc->msg_queue);
 855
 856         ret = rte_intr_callback_register(cc->intr_handle, memif_intr_handler,
 857                                          cc);
 858         if (ret < 0) {
 859                 MIF_LOG(ERR, "Failed to register control channel callback.");
 860                 goto error;
 861         }
 862
 863         ret = memif_msg_enq_hello(cc);
 864         if (ret < 0) {
 865                 MIF_LOG(ERR, "Failed to enqueue hello message.");
 866                 goto error;
 867         }
 868         ret = memif_msg_send_from_queue(cc);
 869         if (ret < 0)
 870                 goto error;
 871
 872         return;
 873
 874  error:
 875         if (sockfd >= 0) {
 876                 close(sockfd);
 877                 sockfd = -1;
 878         }
 879         if (cc != NULL) {
 880                 rte_intr_instance_free(cc->intr_handle);
 881                 rte_free(cc);
 882         }
 883 }
 884
 885 static struct memif_socket *
 886 memif_socket_create(char *key, uint8_t listener, bool is_abstract)
 887 {
 888         struct memif_socket *sock;
 889         struct sockaddr_un un = { 0 };
 890         uint32_t sunlen;
 891         int sockfd;
 892         int ret;
 893         int on = 1;
 894
 895         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
 896         if (sock == NULL) {
 897                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
 898                 return NULL;
 899         }
 900
 901         sock->listener = listener;
 902         strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
 903         TAILQ_INIT(&sock->dev_queue);
 904
 905         if (listener != 0) {
 906                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
 907                 if (sockfd < 0)
 908                         goto error;
 909
 910                 un.sun_family = AF_UNIX;
 911                 if (is_abstract) {
 912                         /* abstract address */
 913                         un.sun_path[0] = '\0';
 914                         strlcpy(un.sun_path + 1, sock->filename, MEMIF_SOCKET_UN_SIZE - 1);
 915                         sunlen = RTE_MIN(1 + strlen(sock->filename),
 916                                          MEMIF_SOCKET_UN_SIZE) +
 917                                  sizeof(un) - sizeof(un.sun_path);
 918                 } else {
 919                         sunlen = sizeof(un);
 920                         strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
 921                 }
 922
 923                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
 924                                  sizeof(on));
 925                 if (ret < 0)
 926                         goto error;
 927
 928                 ret = bind(sockfd, (struct sockaddr *)&un, sunlen);
 929                 if (ret < 0)
 930                         goto error;
 931
 932                 ret = listen(sockfd, 1);
 933                 if (ret < 0)
 934                         goto error;
 935
 936                 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename);
 937
 938                 /* Allocate interrupt instance */
 939                 sock->intr_handle =
 940                         rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
 941                 if (sock->intr_handle == NULL) {
 942                         MIF_LOG(ERR, "Failed to allocate intr handle");
 943                         goto error;
 944                 }
 945
 946                 if (rte_intr_fd_set(sock->intr_handle, sockfd))
 947                         goto error;
 948
 949                 if (rte_intr_type_set(sock->intr_handle, RTE_INTR_HANDLE_EXT))
 950                         goto error;
 951
 952                 ret = rte_intr_callback_register(sock->intr_handle,
 953                                                  memif_listener_handler, sock);
 954                 if (ret < 0) {
 955                         MIF_LOG(ERR, "Failed to register interrupt "
 956                                 "callback for listener socket");
 957                         return NULL;
 958                 }
 959         }
 960
 961         return sock;
 962
 963  error:
 964         MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno));
 965         if (sock != NULL) {
 966                 rte_intr_instance_free(sock->intr_handle);
 967                 rte_free(sock);
 968         }
 969         if (sockfd >= 0)
 970                 close(sockfd);
 971         return NULL;
 972 }
 973
 974 static struct rte_hash *
 975 memif_create_socket_hash(void)
 976 {
 977         struct rte_hash_parameters params = { 0 };
 978
 979         params.name = MEMIF_SOCKET_HASH_NAME;
 980         params.entries = 256;
 981         params.key_len = MEMIF_SOCKET_UN_SIZE;
 982         params.hash_func = rte_jhash;
 983         params.hash_func_init_val = 0;
 984         params.socket_id = SOCKET_ID_ANY;
 985         return rte_hash_create(&params);
 986 }
 987
 988 int
 989 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
 990 {
 991         struct pmd_internals *pmd = dev->data->dev_private;
 992         struct memif_socket *socket = NULL;
 993         struct memif_socket_dev_list_elt *elt;
 994         struct pmd_internals *tmp_pmd;
 995         struct rte_hash *hash;
 996         int ret;
 997         char key[MEMIF_SOCKET_UN_SIZE];
 998
 999         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1000         if (hash == NULL) {
1001                 hash = memif_create_socket_hash();
1002                 if (hash == NULL) {
1003                         MIF_LOG(ERR, "Failed to create memif socket hash.");
1004                         return -1;
1005                 }
1006         }
1007
1008         memset(key, 0, MEMIF_SOCKET_UN_SIZE);
1009         strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
1010         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
1011         if (ret < 0) {
1012                 socket = memif_socket_create(key,
1013                         (pmd->role == MEMIF_ROLE_CLIENT) ? 0 : 1,
1014                         pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT);
1015                 if (socket == NULL)
1016                         return -1;
1017                 ret = rte_hash_add_key_data(hash, key, socket);
1018                 if (ret < 0) {
1019                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
1020                         return ret;
1021                 }
1022         }
1023         pmd->socket_filename = socket->filename;
1024
1025         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
1026                 tmp_pmd = elt->dev->data->dev_private;
1027                 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) {
1028                         MIF_LOG(ERR, "Two interfaces with the same id (%d) can "
1029                                 "not have the same role.", pmd->id);
1030                         return -1;
1031                 }
1032         }
1033
1034         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
1035         if (elt == NULL) {
1036                 MIF_LOG(ERR, "Failed to add device to socket device list.");
1037                 return -1;
1038         }
1039         elt->dev = dev;
1040         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1041
1042         return 0;
1043 }
1044
1045 void
1046 memif_socket_remove_device(struct rte_eth_dev *dev)
1047 {
1048         struct pmd_internals *pmd = dev->data->dev_private;
1049         struct memif_socket *socket = NULL;
1050         struct memif_socket_dev_list_elt *elt, *next;
1051         struct rte_hash *hash;
1052         int ret;
1053
1054         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1055         if (hash == NULL)
1056                 return;
1057
1058         if (pmd->socket_filename == NULL)
1059                 return;
1060
1061         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1062                 return;
1063
1064         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1065                 next = TAILQ_NEXT(elt, next);
1066                 if (elt->dev == dev) {
1067                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1068                         rte_free(elt);
1069                         pmd->socket_filename = NULL;
1070                 }
1071         }
1072
1073         /* remove socket, if this was the last device using it */
1074         if (TAILQ_EMPTY(&socket->dev_queue)) {
1075                 rte_hash_del_key(hash, socket->filename);
1076                 if (socket->listener && !(pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT)) {
1077                         /* remove listener socket file,
1078                          * so we can create new one later.
1079                          */
1080                         ret = remove(socket->filename);
1081                         if (ret < 0)
1082                                 MIF_LOG(ERR, "Failed to remove socket file: %s",
1083                                         socket->filename);
1084                 }
1085                 if (pmd->role != MEMIF_ROLE_CLIENT)
1086                         rte_intr_instance_free(socket->intr_handle);
1087                 rte_free(socket);
1088         }
1089 }
1090
1091 int
1092 memif_connect_server(struct rte_eth_dev *dev)
1093 {
1094         struct pmd_internals *pmd = dev->data->dev_private;
1095
1096         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1097         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1098         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1099         return 0;
1100 }
1101
1102 int
1103 memif_connect_client(struct rte_eth_dev *dev)
1104 {
1105         int sockfd;
1106         int ret;
1107         uint32_t sunlen;
1108         struct sockaddr_un sun = { 0 };
1109         struct pmd_internals *pmd = dev->data->dev_private;
1110
1111         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1112         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1113         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1114
1115         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1116         if (sockfd < 0) {
1117                 MIF_LOG(ERR, "Failed to open socket.");
1118                 return -1;
1119         }
1120
1121         sun.sun_family = AF_UNIX;
1122         sunlen = sizeof(struct sockaddr_un);
1123         if (pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT) {
1124                 /* abstract address */
1125                 sun.sun_path[0] = '\0';
1126                 strlcpy(sun.sun_path + 1,  pmd->socket_filename, MEMIF_SOCKET_UN_SIZE - 1);
1127                 sunlen = RTE_MIN(strlen(pmd->socket_filename) + 1,
1128                                  MEMIF_SOCKET_UN_SIZE) +
1129                          sizeof(sun) - sizeof(sun.sun_path);
1130         } else {
1131                 strlcpy(sun.sun_path,  pmd->socket_filename, MEMIF_SOCKET_UN_SIZE);
1132         }
1133
1134         ret = connect(sockfd, (struct sockaddr *)&sun, sunlen);
1135         if (ret < 0) {
1136                 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename);
1137                 goto error;
1138         }
1139
1140         MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename);
1141
1142         pmd->cc = rte_zmalloc("memif-cc",
1143                               sizeof(struct memif_control_channel), 0);
1144         if (pmd->cc == NULL) {
1145                 MIF_LOG(ERR, "Failed to allocate control channel.");
1146                 goto error;
1147         }
1148
1149         /* Allocate interrupt instance */
1150         pmd->cc->intr_handle =
1151                 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
1152         if (pmd->cc->intr_handle == NULL) {
1153                 MIF_LOG(ERR, "Failed to allocate intr handle");
1154                 goto error;
1155         }
1156
1157         if (rte_intr_fd_set(pmd->cc->intr_handle, sockfd))
1158                 goto error;
1159
1160         if (rte_intr_type_set(pmd->cc->intr_handle, RTE_INTR_HANDLE_EXT))
1161                 goto error;
1162
1163         pmd->cc->socket = NULL;
1164         pmd->cc->dev = dev;
1165         TAILQ_INIT(&pmd->cc->msg_queue);
1166
1167         ret = rte_intr_callback_register(pmd->cc->intr_handle,
1168                                          memif_intr_handler, pmd->cc);
1169         if (ret < 0) {
1170                 MIF_LOG(ERR, "Failed to register interrupt callback for control fd");
1171                 goto error;
1172         }
1173
1174         return 0;
1175
1176  error:
1177         if (sockfd >= 0) {
1178                 close(sockfd);
1179                 sockfd = -1;
1180         }
1181         if (pmd->cc != NULL) {
1182                 rte_intr_instance_free(pmd->cc->intr_handle);
1183                 rte_free(pmd->cc);
1184                 pmd->cc = NULL;
1185         }
1186         return -1;
1187 }