drivers/net/memif/memif_socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/ioctl.h>
  11 #include <errno.h>
  12
  13 #include <rte_version.h>
  14 #include <rte_mbuf.h>
  15 #include <rte_ether.h>
  16 #include <rte_ethdev_driver.h>
  17 #include <rte_ethdev_vdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_bus_vdev.h>
  21 #include <rte_hash.h>
  22 #include <rte_jhash.h>
  23 #include <rte_string_fns.h>
  24
  25 #include "rte_eth_memif.h"
  26 #include "memif_socket.h"
  27
  28 static void memif_intr_handler(void *arg);
  29
  30 static ssize_t
  31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
  32 {
  33         struct msghdr mh = { 0 };
  34         struct iovec iov[1];
  35         struct cmsghdr *cmsg;
  36         char ctl[CMSG_SPACE(sizeof(int))];
  37
  38         iov[0].iov_base = msg;
  39         iov[0].iov_len = sizeof(memif_msg_t);
  40         mh.msg_iov = iov;
  41         mh.msg_iovlen = 1;
  42
  43         if (afd > 0) {
  44                 memset(&ctl, 0, sizeof(ctl));
  45                 mh.msg_control = ctl;
  46                 mh.msg_controllen = sizeof(ctl);
  47                 cmsg = CMSG_FIRSTHDR(&mh);
  48                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  49                 cmsg->cmsg_level = SOL_SOCKET;
  50                 cmsg->cmsg_type = SCM_RIGHTS;
  51                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
  52         }
  53
  54         return sendmsg(fd, &mh, 0);
  55 }
  56
  57 static int
  58 memif_msg_send_from_queue(struct memif_control_channel *cc)
  59 {
  60         ssize_t size;
  61         int ret = 0;
  62         struct memif_msg_queue_elt *e;
  63
  64         e = TAILQ_FIRST(&cc->msg_queue);
  65         if (e == NULL)
  66                 return 0;
  67
  68         size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
  69         if (size != sizeof(memif_msg_t)) {
  70                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
  71                 ret = -1;
  72         } else {
  73                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
  74         }
  75         TAILQ_REMOVE(&cc->msg_queue, e, next);
  76         rte_free(e);
  77
  78         return ret;
  79 }
  80
  81 static struct memif_msg_queue_elt *
  82 memif_msg_enq(struct memif_control_channel *cc)
  83 {
  84         struct memif_msg_queue_elt *e;
  85
  86         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
  87         if (e == NULL) {
  88                 MIF_LOG(ERR, "Failed to allocate control message.");
  89                 return NULL;
  90         }
  91
  92         e->fd = -1;
  93         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
  94
  95         return e;
  96 }
  97
  98 void
  99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
 100                          int err_code)
 101 {
 102         struct memif_msg_queue_elt *e;
 103         struct pmd_internals *pmd;
 104         memif_msg_disconnect_t *d;
 105
 106         if (cc == NULL) {
 107                 MIF_LOG(DEBUG, "Missing control channel.");
 108                 return;
 109         }
 110
 111         e = memif_msg_enq(cc);
 112         if (e == NULL) {
 113                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
 114                 return;
 115         }
 116
 117         d = &e->msg.disconnect;
 118
 119         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
 120         d->code = err_code;
 121
 122         if (reason != NULL) {
 123                 strlcpy((char *)d->string, reason, sizeof(d->string));
 124                 if (cc->dev != NULL) {
 125                         pmd = cc->dev->data->dev_private;
 126                         strlcpy(pmd->local_disc_string, reason,
 127                                 sizeof(pmd->local_disc_string));
 128                 }
 129         }
 130 }
 131
 132 static int
 133 memif_msg_enq_hello(struct memif_control_channel *cc)
 134 {
 135         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
 136         memif_msg_hello_t *h;
 137
 138         if (e == NULL)
 139                 return -1;
 140
 141         h = &e->msg.hello;
 142
 143         e->msg.type = MEMIF_MSG_TYPE_HELLO;
 144         h->min_version = MEMIF_VERSION;
 145         h->max_version = MEMIF_VERSION;
 146         h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 147         h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 148         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
 149         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
 150
 151         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
 152
 153         return 0;
 154 }
 155
 156 static int
 157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
 158 {
 159         struct pmd_internals *pmd = dev->data->dev_private;
 160         memif_msg_hello_t *h = &msg->hello;
 161
 162         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
 163                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
 164                 return -1;
 165         }
 166
 167         /* Set parameters for active connection */
 168         pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1,
 169                                            pmd->cfg.num_s2m_rings);
 170         pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1,
 171                                            pmd->cfg.num_m2s_rings);
 172         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
 173                                             pmd->cfg.log2_ring_size);
 174         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
 175
 176         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
 177
 178         MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name);
 179
 180         return 0;
 181 }
 182
 183 static int
 184 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
 185 {
 186         memif_msg_init_t *i = &msg->init;
 187         struct memif_socket_dev_list_elt *elt;
 188         struct pmd_internals *pmd;
 189         struct rte_eth_dev *dev;
 190
 191         if (i->version != MEMIF_VERSION) {
 192                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
 193                 return -1;
 194         }
 195
 196         if (cc->socket == NULL) {
 197                 memif_msg_enq_disconnect(cc, "Device error", 0);
 198                 return -1;
 199         }
 200
 201         /* Find device with requested ID */
 202         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
 203                 dev = elt->dev;
 204                 pmd = dev->data->dev_private;
 205                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
 206                     (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_MASTER)) {
 207                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
 208                                            ETH_MEMIF_FLAG_CONNECTED)) {
 209                                 memif_msg_enq_disconnect(cc,
 210                                                          "Already connected", 0);
 211                                 return -1;
 212                         }
 213
 214                         /* assign control channel to device */
 215                         cc->dev = dev;
 216                         pmd->cc = cc;
 217
 218                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
 219                                 memif_msg_enq_disconnect(pmd->cc,
 220                                                          "Only ethernet mode supported",
 221                                                          0);
 222                                 return -1;
 223                         }
 224
 225                         strlcpy(pmd->remote_name, (char *)i->name,
 226                                 sizeof(pmd->remote_name));
 227
 228                         if (*pmd->secret != '\0') {
 229                                 if (*i->secret == '\0') {
 230                                         memif_msg_enq_disconnect(pmd->cc,
 231                                                                  "Secret required", 0);
 232                                         return -1;
 233                                 }
 234                                 if (strncmp(pmd->secret, (char *)i->secret,
 235                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
 236                                         memif_msg_enq_disconnect(pmd->cc,
 237                                                                  "Incorrect secret", 0);
 238                                         return -1;
 239                                 }
 240                         }
 241
 242                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
 243                         return 0;
 244                 }
 245         }
 246
 247         /* ID not found on this socket */
 248         MIF_LOG(DEBUG, "ID %u not found.", i->id);
 249         memif_msg_enq_disconnect(cc, "ID not found", 0);
 250         return -1;
 251 }
 252
 253 static int
 254 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
 255                              int fd)
 256 {
 257         struct pmd_internals *pmd = dev->data->dev_private;
 258         struct pmd_process_private *proc_private = dev->process_private;
 259         memif_msg_add_region_t *ar = &msg->add_region;
 260         struct memif_region *r;
 261
 262         if (fd < 0) {
 263                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
 264                 return -1;
 265         }
 266
 267         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
 268                         ar->index != proc_private->regions_num ||
 269                         proc_private->regions[ar->index] != NULL) {
 270                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
 271                 return -1;
 272         }
 273
 274         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 275         if (r == NULL) {
 276                 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
 277                 return -ENOMEM;
 278         }
 279
 280         r->fd = fd;
 281         r->region_size = ar->size;
 282         r->addr = NULL;
 283
 284         proc_private->regions[ar->index] = r;
 285         proc_private->regions_num++;
 286
 287         return 0;
 288 }
 289
 290 static int
 291 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
 292 {
 293         struct pmd_internals *pmd = dev->data->dev_private;
 294         memif_msg_add_ring_t *ar = &msg->add_ring;
 295         struct memif_queue *mq;
 296
 297         if (fd < 0) {
 298                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
 299                 return -1;
 300         }
 301
 302         /* check if we have enough queues */
 303         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) {
 304                 if (ar->index >= pmd->cfg.num_s2m_rings) {
 305                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 306                         return -1;
 307                 }
 308                 pmd->run.num_s2m_rings++;
 309         } else {
 310                 if (ar->index >= pmd->cfg.num_m2s_rings) {
 311                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 312                         return -1;
 313                 }
 314                 pmd->run.num_m2s_rings++;
 315         }
 316
 317         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ?
 318             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
 319
 320         mq->intr_handle.fd = fd;
 321         mq->log2_ring_size = ar->log2_ring_size;
 322         mq->region = ar->region;
 323         mq->ring_offset = ar->offset;
 324
 325         return 0;
 326 }
 327
 328 static int
 329 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
 330 {
 331         struct pmd_internals *pmd = dev->data->dev_private;
 332         memif_msg_connect_t *c = &msg->connect;
 333         int ret;
 334
 335         ret = memif_connect(dev);
 336         if (ret < 0)
 337                 return ret;
 338
 339         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 340                 sizeof(pmd->remote_if_name));
 341         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 342
 343         return 0;
 344 }
 345
 346 static int
 347 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
 348 {
 349         struct pmd_internals *pmd = dev->data->dev_private;
 350         memif_msg_connected_t *c = &msg->connected;
 351         int ret;
 352
 353         ret = memif_connect(dev);
 354         if (ret < 0)
 355                 return ret;
 356
 357         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 358                 sizeof(pmd->remote_if_name));
 359         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 360
 361         return 0;
 362 }
 363
 364 static int
 365 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
 366 {
 367         struct pmd_internals *pmd = dev->data->dev_private;
 368         memif_msg_disconnect_t *d = &msg->disconnect;
 369
 370         memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string));
 371         strlcpy(pmd->remote_disc_string, (char *)d->string,
 372                 sizeof(pmd->remote_disc_string));
 373
 374         MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string);
 375
 376         memset(pmd->local_disc_string, 0, 96);
 377         memif_disconnect(dev);
 378         return 0;
 379 }
 380
 381 static int
 382 memif_msg_enq_ack(struct rte_eth_dev *dev)
 383 {
 384         struct pmd_internals *pmd = dev->data->dev_private;
 385         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 386         if (e == NULL)
 387                 return -1;
 388
 389         e->msg.type = MEMIF_MSG_TYPE_ACK;
 390
 391         return 0;
 392 }
 393
 394 static int
 395 memif_msg_enq_init(struct rte_eth_dev *dev)
 396 {
 397         struct pmd_internals *pmd = dev->data->dev_private;
 398         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 399         memif_msg_init_t *i = &e->msg.init;
 400
 401         if (e == NULL)
 402                 return -1;
 403
 404         i = &e->msg.init;
 405         e->msg.type = MEMIF_MSG_TYPE_INIT;
 406         i->version = MEMIF_VERSION;
 407         i->id = pmd->id;
 408         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
 409
 410         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
 411
 412         if (*pmd->secret != '\0')
 413                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
 414
 415         return 0;
 416 }
 417
 418 static int
 419 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
 420 {
 421         struct pmd_internals *pmd = dev->data->dev_private;
 422         struct pmd_process_private *proc_private = dev->process_private;
 423         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 424         memif_msg_add_region_t *ar;
 425         struct memif_region *mr = proc_private->regions[idx];
 426
 427         if (e == NULL)
 428                 return -1;
 429
 430         ar = &e->msg.add_region;
 431         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
 432         e->fd = mr->fd;
 433         ar->index = idx;
 434         ar->size = mr->region_size;
 435
 436         return 0;
 437 }
 438
 439 static int
 440 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
 441                        memif_ring_type_t type)
 442 {
 443         struct pmd_internals *pmd = dev->data->dev_private;
 444         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 445         struct memif_queue *mq;
 446         memif_msg_add_ring_t *ar;
 447
 448         if (e == NULL)
 449                 return -1;
 450
 451         ar = &e->msg.add_ring;
 452         mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] :
 453             dev->data->rx_queues[idx];
 454
 455         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
 456         e->fd = mq->intr_handle.fd;
 457         ar->index = idx;
 458         ar->offset = mq->ring_offset;
 459         ar->region = mq->region;
 460         ar->log2_ring_size = mq->log2_ring_size;
 461         ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
 462         ar->private_hdr_size = 0;
 463
 464         return 0;
 465 }
 466
 467 static int
 468 memif_msg_enq_connect(struct rte_eth_dev *dev)
 469 {
 470         struct pmd_internals *pmd = dev->data->dev_private;
 471         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 472         memif_msg_connect_t *c;
 473
 474         if (e == NULL)
 475                 return -1;
 476
 477         c = &e->msg.connect;
 478         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
 479         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 480
 481         return 0;
 482 }
 483
 484 static int
 485 memif_msg_enq_connected(struct rte_eth_dev *dev)
 486 {
 487         struct pmd_internals *pmd = dev->data->dev_private;
 488         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 489         memif_msg_connected_t *c;
 490
 491         if (e == NULL)
 492                 return -1;
 493
 494         c = &e->msg.connected;
 495         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
 496         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 497
 498         return 0;
 499 }
 500
 501 static void
 502 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
 503 {
 504         struct memif_msg_queue_elt *elt;
 505         struct memif_control_channel *cc = arg;
 506
 507         /* close control channel fd */
 508         close(intr_handle->fd);
 509         /* clear message queue */
 510         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
 511                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
 512                 rte_free(elt);
 513         }
 514         /* free control channel */
 515         rte_free(cc);
 516 }
 517
 518 void
 519 memif_disconnect(struct rte_eth_dev *dev)
 520 {
 521         struct pmd_internals *pmd = dev->data->dev_private;
 522         struct memif_msg_queue_elt *elt, *next;
 523         struct memif_queue *mq;
 524         struct rte_intr_handle *ih;
 525         int i;
 526         int ret;
 527
 528         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 529         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
 530         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
 531
 532         rte_spinlock_lock(&pmd->cc_lock);
 533         if (pmd->cc != NULL) {
 534                 /* Clear control message queue (except disconnect message if any). */
 535                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
 536                         next = TAILQ_NEXT(elt, next);
 537                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
 538                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
 539                                 rte_free(elt);
 540                         }
 541                 }
 542                 /* send disconnect message (if there is any in queue) */
 543                 memif_msg_send_from_queue(pmd->cc);
 544
 545                 /* at this point, there should be no more messages in queue */
 546                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
 547                         MIF_LOG(WARNING,
 548                                 "Unexpected message(s) in message queue.");
 549                 }
 550
 551                 ih = &pmd->cc->intr_handle;
 552                 if (ih->fd > 0) {
 553                         ret = rte_intr_callback_unregister(ih,
 554                                                         memif_intr_handler,
 555                                                         pmd->cc);
 556                         /*
 557                          * If callback is active (disconnecting based on
 558                          * received control message).
 559                          */
 560                         if (ret == -EAGAIN) {
 561                                 ret = rte_intr_callback_unregister_pending(ih,
 562                                                         memif_intr_handler,
 563                                                         pmd->cc,
 564                                                         memif_intr_unregister_handler);
 565                         } else if (ret > 0) {
 566                                 close(ih->fd);
 567                                 rte_free(pmd->cc);
 568                         }
 569                         pmd->cc = NULL;
 570                         if (ret <= 0)
 571                                 MIF_LOG(WARNING,
 572                                         "Failed to unregister control channel callback.");
 573                 }
 574         }
 575         rte_spinlock_unlock(&pmd->cc_lock);
 576
 577         /* unconfig interrupts */
 578         for (i = 0; i < pmd->cfg.num_s2m_rings; i++) {
 579                 if (pmd->role == MEMIF_ROLE_SLAVE) {
 580                         if (dev->data->tx_queues != NULL)
 581                                 mq = dev->data->tx_queues[i];
 582                         else
 583                                 continue;
 584                 } else {
 585                         if (dev->data->rx_queues != NULL)
 586                                 mq = dev->data->rx_queues[i];
 587                         else
 588                                 continue;
 589                 }
 590                 if (mq->intr_handle.fd > 0) {
 591                         close(mq->intr_handle.fd);
 592                         mq->intr_handle.fd = -1;
 593                 }
 594         }
 595         for (i = 0; i < pmd->cfg.num_m2s_rings; i++) {
 596                 if (pmd->role == MEMIF_ROLE_MASTER) {
 597                         if (dev->data->tx_queues != NULL)
 598                                 mq = dev->data->tx_queues[i];
 599                         else
 600                                 continue;
 601                 } else {
 602                         if (dev->data->rx_queues != NULL)
 603                                 mq = dev->data->rx_queues[i];
 604                         else
 605                                 continue;
 606                 }
 607                 if (mq->intr_handle.fd > 0) {
 608                         close(mq->intr_handle.fd);
 609                         mq->intr_handle.fd = -1;
 610                 }
 611         }
 612
 613         memif_free_regions(dev);
 614
 615         /* reset connection configuration */
 616         memset(&pmd->run, 0, sizeof(pmd->run));
 617
 618         MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id,
 619                 (pmd->role == MEMIF_ROLE_MASTER) ? "master" : "slave");
 620 }
 621
 622 static int
 623 memif_msg_receive(struct memif_control_channel *cc)
 624 {
 625         char ctl[CMSG_SPACE(sizeof(int)) +
 626                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
 627         struct msghdr mh = { 0 };
 628         struct iovec iov[1];
 629         memif_msg_t msg = { 0 };
 630         ssize_t size;
 631         int ret = 0;
 632         struct ucred *cr __rte_unused;
 633         cr = 0;
 634         struct cmsghdr *cmsg;
 635         int afd = -1;
 636         int i;
 637         struct pmd_internals *pmd;
 638         struct pmd_process_private *proc_private;
 639
 640         iov[0].iov_base = (void *)&msg;
 641         iov[0].iov_len = sizeof(memif_msg_t);
 642         mh.msg_iov = iov;
 643         mh.msg_iovlen = 1;
 644         mh.msg_control = ctl;
 645         mh.msg_controllen = sizeof(ctl);
 646
 647         size = recvmsg(cc->intr_handle.fd, &mh, 0);
 648         if (size != sizeof(memif_msg_t)) {
 649                 MIF_LOG(DEBUG, "Invalid message size = %zd", size);
 650                 if (size > 0)
 651                         /* 0 means end-of-file, negative size means error,
 652                          * don't send further disconnect message in such cases.
 653                          */
 654                         memif_msg_enq_disconnect(cc, "Invalid message size", 0);
 655                 return -1;
 656         }
 657         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
 658
 659         cmsg = CMSG_FIRSTHDR(&mh);
 660         while (cmsg) {
 661                 if (cmsg->cmsg_level == SOL_SOCKET) {
 662                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
 663                                 cr = (struct ucred *)CMSG_DATA(cmsg);
 664                         else if (cmsg->cmsg_type == SCM_RIGHTS)
 665                                 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
 666                 }
 667                 cmsg = CMSG_NXTHDR(&mh, cmsg);
 668         }
 669
 670         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
 671                 MIF_LOG(DEBUG, "Unexpected message.");
 672                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
 673                 return -1;
 674         }
 675
 676         /* get device from hash data */
 677         switch (msg.type) {
 678         case MEMIF_MSG_TYPE_ACK:
 679                 break;
 680         case MEMIF_MSG_TYPE_HELLO:
 681                 ret = memif_msg_receive_hello(cc->dev, &msg);
 682                 if (ret < 0)
 683                         goto exit;
 684                 ret = memif_init_regions_and_queues(cc->dev);
 685                 if (ret < 0)
 686                         goto exit;
 687                 ret = memif_msg_enq_init(cc->dev);
 688                 if (ret < 0)
 689                         goto exit;
 690                 pmd = cc->dev->data->dev_private;
 691                 proc_private = cc->dev->process_private;
 692                 for (i = 0; i < proc_private->regions_num; i++) {
 693                         ret = memif_msg_enq_add_region(cc->dev, i);
 694                         if (ret < 0)
 695                                 goto exit;
 696                 }
 697                 for (i = 0; i < pmd->run.num_s2m_rings; i++) {
 698                         ret = memif_msg_enq_add_ring(cc->dev, i,
 699                                                      MEMIF_RING_S2M);
 700                         if (ret < 0)
 701                                 goto exit;
 702                 }
 703                 for (i = 0; i < pmd->run.num_m2s_rings; i++) {
 704                         ret = memif_msg_enq_add_ring(cc->dev, i,
 705                                                      MEMIF_RING_M2S);
 706                         if (ret < 0)
 707                                 goto exit;
 708                 }
 709                 ret = memif_msg_enq_connect(cc->dev);
 710                 if (ret < 0)
 711                         goto exit;
 712                 break;
 713         case MEMIF_MSG_TYPE_INIT:
 714                 /*
 715                  * This cc does not have an interface asociated with it.
 716                  * If suitable interface is found it will be assigned here.
 717                  */
 718                 ret = memif_msg_receive_init(cc, &msg);
 719                 if (ret < 0)
 720                         goto exit;
 721                 ret = memif_msg_enq_ack(cc->dev);
 722                 if (ret < 0)
 723                         goto exit;
 724                 break;
 725         case MEMIF_MSG_TYPE_ADD_REGION:
 726                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
 727                 if (ret < 0)
 728                         goto exit;
 729                 ret = memif_msg_enq_ack(cc->dev);
 730                 if (ret < 0)
 731                         goto exit;
 732                 break;
 733         case MEMIF_MSG_TYPE_ADD_RING:
 734                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
 735                 if (ret < 0)
 736                         goto exit;
 737                 ret = memif_msg_enq_ack(cc->dev);
 738                 if (ret < 0)
 739                         goto exit;
 740                 break;
 741         case MEMIF_MSG_TYPE_CONNECT:
 742                 ret = memif_msg_receive_connect(cc->dev, &msg);
 743                 if (ret < 0)
 744                         goto exit;
 745                 ret = memif_msg_enq_connected(cc->dev);
 746                 if (ret < 0)
 747                         goto exit;
 748                 break;
 749         case MEMIF_MSG_TYPE_CONNECTED:
 750                 ret = memif_msg_receive_connected(cc->dev, &msg);
 751                 break;
 752         case MEMIF_MSG_TYPE_DISCONNECT:
 753                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
 754                 if (ret < 0)
 755                         goto exit;
 756                 break;
 757         default:
 758                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
 759                 ret = -1;
 760                 goto exit;
 761         }
 762
 763  exit:
 764         return ret;
 765 }
 766
 767 static void
 768 memif_intr_handler(void *arg)
 769 {
 770         struct memif_control_channel *cc = arg;
 771         int ret;
 772
 773         ret = memif_msg_receive(cc);
 774         /* if driver failed to assign device */
 775         if (cc->dev == NULL) {
 776                 memif_msg_send_from_queue(cc);
 777                 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
 778                                                            memif_intr_handler,
 779                                                            cc,
 780                                                            memif_intr_unregister_handler);
 781                 if (ret < 0)
 782                         MIF_LOG(WARNING,
 783                                 "Failed to unregister control channel callback.");
 784                 return;
 785         }
 786         /* if memif_msg_receive failed */
 787         if (ret < 0)
 788                 goto disconnect;
 789
 790         ret = memif_msg_send_from_queue(cc);
 791         if (ret < 0)
 792                 goto disconnect;
 793
 794         return;
 795
 796  disconnect:
 797         if (cc->dev == NULL) {
 798                 MIF_LOG(WARNING, "eth dev not allocated");
 799                 return;
 800         }
 801         memif_disconnect(cc->dev);
 802 }
 803
 804 static void
 805 memif_listener_handler(void *arg)
 806 {
 807         struct memif_socket *socket = arg;
 808         int sockfd;
 809         int addr_len;
 810         struct sockaddr_un client;
 811         struct memif_control_channel *cc;
 812         int ret;
 813
 814         addr_len = sizeof(client);
 815         sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
 816                         (socklen_t *)&addr_len);
 817         if (sockfd < 0) {
 818                 MIF_LOG(ERR,
 819                         "Failed to accept connection request on socket fd %d",
 820                         socket->intr_handle.fd);
 821                 return;
 822         }
 823
 824         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
 825
 826         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
 827         if (cc == NULL) {
 828                 MIF_LOG(ERR, "Failed to allocate control channel.");
 829                 goto error;
 830         }
 831
 832         cc->intr_handle.fd = sockfd;
 833         cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
 834         cc->socket = socket;
 835         cc->dev = NULL;
 836         TAILQ_INIT(&cc->msg_queue);
 837
 838         ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
 839         if (ret < 0) {
 840                 MIF_LOG(ERR, "Failed to register control channel callback.");
 841                 goto error;
 842         }
 843
 844         ret = memif_msg_enq_hello(cc);
 845         if (ret < 0) {
 846                 MIF_LOG(ERR, "Failed to enqueue hello message.");
 847                 goto error;
 848         }
 849         ret = memif_msg_send_from_queue(cc);
 850         if (ret < 0)
 851                 goto error;
 852
 853         return;
 854
 855  error:
 856         if (sockfd >= 0) {
 857                 close(sockfd);
 858                 sockfd = -1;
 859         }
 860         if (cc != NULL)
 861                 rte_free(cc);
 862 }
 863
 864 static struct memif_socket *
 865 memif_socket_create(char *key, uint8_t listener)
 866 {
 867         struct memif_socket *sock;
 868         struct sockaddr_un un;
 869         int sockfd;
 870         int ret;
 871         int on = 1;
 872
 873         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
 874         if (sock == NULL) {
 875                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
 876                 return NULL;
 877         }
 878
 879         sock->listener = listener;
 880         strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
 881         TAILQ_INIT(&sock->dev_queue);
 882
 883         if (listener != 0) {
 884                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
 885                 if (sockfd < 0)
 886                         goto error;
 887
 888                 un.sun_family = AF_UNIX;
 889                 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
 890
 891                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
 892                                  sizeof(on));
 893                 if (ret < 0)
 894                         goto error;
 895
 896                 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un));
 897                 if (ret < 0)
 898                         goto error;
 899
 900                 ret = listen(sockfd, 1);
 901                 if (ret < 0)
 902                         goto error;
 903
 904                 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename);
 905
 906                 sock->intr_handle.fd = sockfd;
 907                 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
 908                 ret = rte_intr_callback_register(&sock->intr_handle,
 909                                                  memif_listener_handler, sock);
 910                 if (ret < 0) {
 911                         MIF_LOG(ERR, "Failed to register interrupt "
 912                                 "callback for listener socket");
 913                         return NULL;
 914                 }
 915         }
 916
 917         return sock;
 918
 919  error:
 920         MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno));
 921         if (sock != NULL)
 922                 rte_free(sock);
 923         if (sockfd >= 0)
 924                 close(sockfd);
 925         return NULL;
 926 }
 927
 928 static struct rte_hash *
 929 memif_create_socket_hash(void)
 930 {
 931         struct rte_hash_parameters params = { 0 };
 932
 933         params.name = MEMIF_SOCKET_HASH_NAME;
 934         params.entries = 256;
 935         params.key_len = MEMIF_SOCKET_UN_SIZE;
 936         params.hash_func = rte_jhash;
 937         params.hash_func_init_val = 0;
 938         return rte_hash_create(&params);
 939 }
 940
 941 int
 942 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
 943 {
 944         struct pmd_internals *pmd = dev->data->dev_private;
 945         struct memif_socket *socket = NULL;
 946         struct memif_socket_dev_list_elt *elt;
 947         struct pmd_internals *tmp_pmd;
 948         struct rte_hash *hash;
 949         int ret;
 950         char key[MEMIF_SOCKET_UN_SIZE];
 951
 952         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
 953         if (hash == NULL) {
 954                 hash = memif_create_socket_hash();
 955                 if (hash == NULL) {
 956                         MIF_LOG(ERR, "Failed to create memif socket hash.");
 957                         return -1;
 958                 }
 959         }
 960
 961         memset(key, 0, MEMIF_SOCKET_UN_SIZE);
 962         strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
 963         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
 964         if (ret < 0) {
 965                 socket = memif_socket_create(key,
 966                                              (pmd->role == MEMIF_ROLE_SLAVE) ? 0 : 1);
 967                 if (socket == NULL)
 968                         return -1;
 969                 ret = rte_hash_add_key_data(hash, key, socket);
 970                 if (ret < 0) {
 971                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
 972                         return ret;
 973                 }
 974         }
 975         pmd->socket_filename = socket->filename;
 976
 977         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
 978                 tmp_pmd = elt->dev->data->dev_private;
 979                 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) {
 980                         MIF_LOG(ERR, "Two interfaces with the same id (%d) can "
 981                                 "not have the same role.", pmd->id);
 982                         return -1;
 983                 }
 984         }
 985
 986         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
 987         if (elt == NULL) {
 988                 MIF_LOG(ERR, "Failed to add device to socket device list.");
 989                 return -1;
 990         }
 991         elt->dev = dev;
 992         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
 993
 994         return 0;
 995 }
 996
 997 void
 998 memif_socket_remove_device(struct rte_eth_dev *dev)
 999 {
1000         struct pmd_internals *pmd = dev->data->dev_private;
1001         struct memif_socket *socket = NULL;
1002         struct memif_socket_dev_list_elt *elt, *next;
1003         struct rte_hash *hash;
1004         int ret;
1005
1006         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1007         if (hash == NULL)
1008                 return;
1009
1010         if (pmd->socket_filename == NULL)
1011                 return;
1012
1013         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1014                 return;
1015
1016         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1017                 next = TAILQ_NEXT(elt, next);
1018                 if (elt->dev == dev) {
1019                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1020                         rte_free(elt);
1021                         pmd->socket_filename = NULL;
1022                 }
1023         }
1024
1025         /* remove socket, if this was the last device using it */
1026         if (TAILQ_EMPTY(&socket->dev_queue)) {
1027                 rte_hash_del_key(hash, socket->filename);
1028                 if (socket->listener) {
1029                         /* remove listener socket file,
1030                          * so we can create new one later.
1031                          */
1032                         ret = remove(socket->filename);
1033                         if (ret < 0)
1034                                 MIF_LOG(ERR, "Failed to remove socket file: %s",
1035                                         socket->filename);
1036                 }
1037                 rte_free(socket);
1038         }
1039 }
1040
1041 int
1042 memif_connect_master(struct rte_eth_dev *dev)
1043 {
1044         struct pmd_internals *pmd = dev->data->dev_private;
1045
1046         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1047         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1048         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1049         return 0;
1050 }
1051
1052 int
1053 memif_connect_slave(struct rte_eth_dev *dev)
1054 {
1055         int sockfd;
1056         int ret;
1057         struct sockaddr_un sun;
1058         struct pmd_internals *pmd = dev->data->dev_private;
1059
1060         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1061         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1062         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1063
1064         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1065         if (sockfd < 0) {
1066                 MIF_LOG(ERR, "Failed to open socket.");
1067                 return -1;
1068         }
1069
1070         sun.sun_family = AF_UNIX;
1071
1072         memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1);
1073
1074         ret = connect(sockfd, (struct sockaddr *)&sun,
1075                       sizeof(struct sockaddr_un));
1076         if (ret < 0) {
1077                 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename);
1078                 goto error;
1079         }
1080
1081         MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename);
1082
1083         pmd->cc = rte_zmalloc("memif-cc",
1084                               sizeof(struct memif_control_channel), 0);
1085         if (pmd->cc == NULL) {
1086                 MIF_LOG(ERR, "Failed to allocate control channel.");
1087                 goto error;
1088         }
1089
1090         pmd->cc->intr_handle.fd = sockfd;
1091         pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1092         pmd->cc->socket = NULL;
1093         pmd->cc->dev = dev;
1094         TAILQ_INIT(&pmd->cc->msg_queue);
1095
1096         ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1097                                          memif_intr_handler, pmd->cc);
1098         if (ret < 0) {
1099                 MIF_LOG(ERR, "Failed to register interrupt callback for control fd");
1100                 goto error;
1101         }
1102
1103         return 0;
1104
1105  error:
1106         if (sockfd >= 0) {
1107                 close(sockfd);
1108                 sockfd = -1;
1109         }
1110         if (pmd->cc != NULL) {
1111                 rte_free(pmd->cc);
1112                 pmd->cc = NULL;
1113         }
1114         return -1;
1115 }