drivers/net/memif/memif_socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/ioctl.h>
  11 #include <errno.h>
  12
  13 #include <rte_version.h>
  14 #include <rte_mbuf.h>
  15 #include <rte_ether.h>
  16 #include <rte_ethdev_driver.h>
  17 #include <rte_ethdev_vdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_bus_vdev.h>
  21 #include <rte_hash.h>
  22 #include <rte_jhash.h>
  23 #include <rte_string_fns.h>
  24
  25 #include "rte_eth_memif.h"
  26 #include "memif_socket.h"
  27
  28 static void memif_intr_handler(void *arg);
  29
  30 static ssize_t
  31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
  32 {
  33         struct msghdr mh = { 0 };
  34         struct iovec iov[1];
  35         struct cmsghdr *cmsg;
  36         char ctl[CMSG_SPACE(sizeof(int))];
  37
  38         iov[0].iov_base = msg;
  39         iov[0].iov_len = sizeof(memif_msg_t);
  40         mh.msg_iov = iov;
  41         mh.msg_iovlen = 1;
  42
  43         if (afd > 0) {
  44                 memset(&ctl, 0, sizeof(ctl));
  45                 mh.msg_control = ctl;
  46                 mh.msg_controllen = sizeof(ctl);
  47                 cmsg = CMSG_FIRSTHDR(&mh);
  48                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  49                 cmsg->cmsg_level = SOL_SOCKET;
  50                 cmsg->cmsg_type = SCM_RIGHTS;
  51                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
  52         }
  53
  54         return sendmsg(fd, &mh, 0);
  55 }
  56
  57 static int
  58 memif_msg_send_from_queue(struct memif_control_channel *cc)
  59 {
  60         ssize_t size;
  61         int ret = 0;
  62         struct memif_msg_queue_elt *e;
  63
  64         e = TAILQ_FIRST(&cc->msg_queue);
  65         if (e == NULL)
  66                 return 0;
  67
  68         size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
  69         if (size != sizeof(memif_msg_t)) {
  70                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
  71                 ret = -1;
  72         } else {
  73                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
  74         }
  75         TAILQ_REMOVE(&cc->msg_queue, e, next);
  76         rte_free(e);
  77
  78         return ret;
  79 }
  80
  81 static struct memif_msg_queue_elt *
  82 memif_msg_enq(struct memif_control_channel *cc)
  83 {
  84         struct memif_msg_queue_elt *e;
  85
  86         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
  87         if (e == NULL) {
  88                 MIF_LOG(ERR, "Failed to allocate control message.");
  89                 return NULL;
  90         }
  91
  92         e->fd = -1;
  93         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
  94
  95         return e;
  96 }
  97
  98 void
  99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
 100                          int err_code)
 101 {
 102         struct memif_msg_queue_elt *e;
 103         struct pmd_internals *pmd;
 104         memif_msg_disconnect_t *d;
 105
 106         if (cc == NULL) {
 107                 MIF_LOG(DEBUG, "Missing control channel.");
 108                 return;
 109         }
 110
 111         e = memif_msg_enq(cc);
 112         if (e == NULL) {
 113                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
 114                 return;
 115         }
 116
 117         d = &e->msg.disconnect;
 118
 119         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
 120         d->code = err_code;
 121
 122         if (reason != NULL) {
 123                 strlcpy((char *)d->string, reason, sizeof(d->string));
 124                 if (cc->dev != NULL) {
 125                         pmd = cc->dev->data->dev_private;
 126                         strlcpy(pmd->local_disc_string, reason,
 127                                 sizeof(pmd->local_disc_string));
 128                 }
 129         }
 130 }
 131
 132 static int
 133 memif_msg_enq_hello(struct memif_control_channel *cc)
 134 {
 135         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
 136         memif_msg_hello_t *h;
 137
 138         if (e == NULL)
 139                 return -1;
 140
 141         h = &e->msg.hello;
 142
 143         e->msg.type = MEMIF_MSG_TYPE_HELLO;
 144         h->min_version = MEMIF_VERSION;
 145         h->max_version = MEMIF_VERSION;
 146         h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 147         h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 148         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
 149         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
 150
 151         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
 152
 153         return 0;
 154 }
 155
 156 static int
 157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
 158 {
 159         struct pmd_internals *pmd = dev->data->dev_private;
 160         memif_msg_hello_t *h = &msg->hello;
 161
 162         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
 163                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
 164                 return -1;
 165         }
 166
 167         /* Set parameters for active connection */
 168         pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1,
 169                                            pmd->cfg.num_s2m_rings);
 170         pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1,
 171                                            pmd->cfg.num_m2s_rings);
 172         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
 173                                             pmd->cfg.log2_ring_size);
 174         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
 175
 176         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
 177
 178         MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name);
 179
 180         return 0;
 181 }
 182
 183 static int
 184 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
 185 {
 186         memif_msg_init_t *i = &msg->init;
 187         struct memif_socket_dev_list_elt *elt;
 188         struct pmd_internals *pmd;
 189         struct rte_eth_dev *dev;
 190
 191         if (i->version != MEMIF_VERSION) {
 192                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
 193                 return -1;
 194         }
 195
 196         if (cc->socket == NULL) {
 197                 memif_msg_enq_disconnect(cc, "Device error", 0);
 198                 return -1;
 199         }
 200
 201         /* Find device with requested ID */
 202         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
 203                 dev = elt->dev;
 204                 pmd = dev->data->dev_private;
 205                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
 206                     (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_MASTER)) {
 207                         /* assign control channel to device */
 208                         cc->dev = dev;
 209                         pmd->cc = cc;
 210
 211                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
 212                                 memif_msg_enq_disconnect(pmd->cc,
 213                                                          "Only ethernet mode supported",
 214                                                          0);
 215                                 return -1;
 216                         }
 217
 218                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
 219                                            ETH_MEMIF_FLAG_CONNECTED)) {
 220                                 memif_msg_enq_disconnect(pmd->cc,
 221                                                          "Already connected", 0);
 222                                 return -1;
 223                         }
 224                         strlcpy(pmd->remote_name, (char *)i->name,
 225                                 sizeof(pmd->remote_name));
 226
 227                         if (*pmd->secret != '\0') {
 228                                 if (*i->secret == '\0') {
 229                                         memif_msg_enq_disconnect(pmd->cc,
 230                                                                  "Secret required", 0);
 231                                         return -1;
 232                                 }
 233                                 if (strncmp(pmd->secret, (char *)i->secret,
 234                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
 235                                         memif_msg_enq_disconnect(pmd->cc,
 236                                                                  "Incorrect secret", 0);
 237                                         return -1;
 238                                 }
 239                         }
 240
 241                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
 242                         return 0;
 243                 }
 244         }
 245
 246         /* ID not found on this socket */
 247         MIF_LOG(DEBUG, "ID %u not found.", i->id);
 248         memif_msg_enq_disconnect(cc, "ID not found", 0);
 249         return -1;
 250 }
 251
 252 static int
 253 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
 254                              int fd)
 255 {
 256         struct pmd_internals *pmd = dev->data->dev_private;
 257         struct pmd_process_private *proc_private = dev->process_private;
 258         memif_msg_add_region_t *ar = &msg->add_region;
 259         struct memif_region *r;
 260
 261         if (fd < 0) {
 262                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
 263                 return -1;
 264         }
 265
 266         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
 267                         ar->index != proc_private->regions_num ||
 268                         proc_private->regions[ar->index] != NULL) {
 269                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
 270                 return -1;
 271         }
 272
 273         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 274         if (r == NULL) {
 275                 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
 276                 return -ENOMEM;
 277         }
 278
 279         r->fd = fd;
 280         r->region_size = ar->size;
 281         r->addr = NULL;
 282
 283         proc_private->regions[ar->index] = r;
 284         proc_private->regions_num++;
 285
 286         return 0;
 287 }
 288
 289 static int
 290 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
 291 {
 292         struct pmd_internals *pmd = dev->data->dev_private;
 293         memif_msg_add_ring_t *ar = &msg->add_ring;
 294         struct memif_queue *mq;
 295
 296         if (fd < 0) {
 297                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
 298                 return -1;
 299         }
 300
 301         /* check if we have enough queues */
 302         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) {
 303                 if (ar->index >= pmd->cfg.num_s2m_rings) {
 304                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 305                         return -1;
 306                 }
 307                 pmd->run.num_s2m_rings++;
 308         } else {
 309                 if (ar->index >= pmd->cfg.num_m2s_rings) {
 310                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 311                         return -1;
 312                 }
 313                 pmd->run.num_m2s_rings++;
 314         }
 315
 316         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ?
 317             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
 318
 319         mq->intr_handle.fd = fd;
 320         mq->log2_ring_size = ar->log2_ring_size;
 321         mq->region = ar->region;
 322         mq->ring_offset = ar->offset;
 323
 324         return 0;
 325 }
 326
 327 static int
 328 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
 329 {
 330         struct pmd_internals *pmd = dev->data->dev_private;
 331         memif_msg_connect_t *c = &msg->connect;
 332         int ret;
 333
 334         ret = memif_connect(dev);
 335         if (ret < 0)
 336                 return ret;
 337
 338         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 339                 sizeof(pmd->remote_if_name));
 340         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 341
 342         return 0;
 343 }
 344
 345 static int
 346 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
 347 {
 348         struct pmd_internals *pmd = dev->data->dev_private;
 349         memif_msg_connected_t *c = &msg->connected;
 350         int ret;
 351
 352         ret = memif_connect(dev);
 353         if (ret < 0)
 354                 return ret;
 355
 356         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 357                 sizeof(pmd->remote_if_name));
 358         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 359
 360         return 0;
 361 }
 362
 363 static int
 364 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
 365 {
 366         struct pmd_internals *pmd = dev->data->dev_private;
 367         memif_msg_disconnect_t *d = &msg->disconnect;
 368
 369         memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string));
 370         strlcpy(pmd->remote_disc_string, (char *)d->string,
 371                 sizeof(pmd->remote_disc_string));
 372
 373         MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string);
 374
 375         memset(pmd->local_disc_string, 0, 96);
 376         memif_disconnect(dev);
 377         return 0;
 378 }
 379
 380 static int
 381 memif_msg_enq_ack(struct rte_eth_dev *dev)
 382 {
 383         struct pmd_internals *pmd = dev->data->dev_private;
 384         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 385         if (e == NULL)
 386                 return -1;
 387
 388         e->msg.type = MEMIF_MSG_TYPE_ACK;
 389
 390         return 0;
 391 }
 392
 393 static int
 394 memif_msg_enq_init(struct rte_eth_dev *dev)
 395 {
 396         struct pmd_internals *pmd = dev->data->dev_private;
 397         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 398         memif_msg_init_t *i = &e->msg.init;
 399
 400         if (e == NULL)
 401                 return -1;
 402
 403         i = &e->msg.init;
 404         e->msg.type = MEMIF_MSG_TYPE_INIT;
 405         i->version = MEMIF_VERSION;
 406         i->id = pmd->id;
 407         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
 408
 409         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
 410
 411         if (*pmd->secret != '\0')
 412                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
 413
 414         return 0;
 415 }
 416
 417 static int
 418 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
 419 {
 420         struct pmd_internals *pmd = dev->data->dev_private;
 421         struct pmd_process_private *proc_private = dev->process_private;
 422         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 423         memif_msg_add_region_t *ar;
 424         struct memif_region *mr = proc_private->regions[idx];
 425
 426         if (e == NULL)
 427                 return -1;
 428
 429         ar = &e->msg.add_region;
 430         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
 431         e->fd = mr->fd;
 432         ar->index = idx;
 433         ar->size = mr->region_size;
 434
 435         return 0;
 436 }
 437
 438 static int
 439 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
 440                        memif_ring_type_t type)
 441 {
 442         struct pmd_internals *pmd = dev->data->dev_private;
 443         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 444         struct memif_queue *mq;
 445         memif_msg_add_ring_t *ar;
 446
 447         if (e == NULL)
 448                 return -1;
 449
 450         ar = &e->msg.add_ring;
 451         mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] :
 452             dev->data->rx_queues[idx];
 453
 454         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
 455         e->fd = mq->intr_handle.fd;
 456         ar->index = idx;
 457         ar->offset = mq->ring_offset;
 458         ar->region = mq->region;
 459         ar->log2_ring_size = mq->log2_ring_size;
 460         ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
 461         ar->private_hdr_size = 0;
 462
 463         return 0;
 464 }
 465
 466 static int
 467 memif_msg_enq_connect(struct rte_eth_dev *dev)
 468 {
 469         struct pmd_internals *pmd = dev->data->dev_private;
 470         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 471         memif_msg_connect_t *c;
 472
 473         if (e == NULL)
 474                 return -1;
 475
 476         c = &e->msg.connect;
 477         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
 478         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 479
 480         return 0;
 481 }
 482
 483 static int
 484 memif_msg_enq_connected(struct rte_eth_dev *dev)
 485 {
 486         struct pmd_internals *pmd = dev->data->dev_private;
 487         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 488         memif_msg_connected_t *c;
 489
 490         if (e == NULL)
 491                 return -1;
 492
 493         c = &e->msg.connected;
 494         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
 495         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 496
 497         return 0;
 498 }
 499
 500 static void
 501 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
 502 {
 503         struct memif_msg_queue_elt *elt;
 504         struct memif_control_channel *cc = arg;
 505
 506         /* close control channel fd */
 507         close(intr_handle->fd);
 508         /* clear message queue */
 509         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
 510                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
 511                 rte_free(elt);
 512         }
 513         /* free control channel */
 514         rte_free(cc);
 515 }
 516
 517 void
 518 memif_disconnect(struct rte_eth_dev *dev)
 519 {
 520         struct pmd_internals *pmd = dev->data->dev_private;
 521         struct memif_msg_queue_elt *elt, *next;
 522         struct memif_queue *mq;
 523         struct rte_intr_handle *ih;
 524         int i;
 525         int ret;
 526
 527         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 528         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
 529         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
 530
 531         rte_spinlock_lock(&pmd->cc_lock);
 532         if (pmd->cc != NULL) {
 533                 /* Clear control message queue (except disconnect message if any). */
 534                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
 535                         next = TAILQ_NEXT(elt, next);
 536                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
 537                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
 538                                 rte_free(elt);
 539                         }
 540                 }
 541                 /* send disconnect message (if there is any in queue) */
 542                 memif_msg_send_from_queue(pmd->cc);
 543
 544                 /* at this point, there should be no more messages in queue */
 545                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
 546                         MIF_LOG(WARNING,
 547                                 "Unexpected message(s) in message queue.");
 548                 }
 549
 550                 ih = &pmd->cc->intr_handle;
 551                 if (ih->fd > 0) {
 552                         ret = rte_intr_callback_unregister(ih,
 553                                                         memif_intr_handler,
 554                                                         pmd->cc);
 555                         /*
 556                          * If callback is active (disconnecting based on
 557                          * received control message).
 558                          */
 559                         if (ret == -EAGAIN) {
 560                                 ret = rte_intr_callback_unregister_pending(ih,
 561                                                         memif_intr_handler,
 562                                                         pmd->cc,
 563                                                         memif_intr_unregister_handler);
 564                         } else if (ret > 0) {
 565                                 close(ih->fd);
 566                                 rte_free(pmd->cc);
 567                         }
 568                         pmd->cc = NULL;
 569                         if (ret <= 0)
 570                                 MIF_LOG(WARNING,
 571                                         "Failed to unregister control channel callback.");
 572                 }
 573         }
 574         rte_spinlock_unlock(&pmd->cc_lock);
 575
 576         /* unconfig interrupts */
 577         for (i = 0; i < pmd->cfg.num_s2m_rings; i++) {
 578                 if (pmd->role == MEMIF_ROLE_SLAVE) {
 579                         if (dev->data->tx_queues != NULL)
 580                                 mq = dev->data->tx_queues[i];
 581                         else
 582                                 continue;
 583                 } else {
 584                         if (dev->data->rx_queues != NULL)
 585                                 mq = dev->data->rx_queues[i];
 586                         else
 587                                 continue;
 588                 }
 589                 if (mq->intr_handle.fd > 0) {
 590                         close(mq->intr_handle.fd);
 591                         mq->intr_handle.fd = -1;
 592                 }
 593         }
 594         for (i = 0; i < pmd->cfg.num_m2s_rings; i++) {
 595                 if (pmd->role == MEMIF_ROLE_MASTER) {
 596                         if (dev->data->tx_queues != NULL)
 597                                 mq = dev->data->tx_queues[i];
 598                         else
 599                                 continue;
 600                 } else {
 601                         if (dev->data->rx_queues != NULL)
 602                                 mq = dev->data->rx_queues[i];
 603                         else
 604                                 continue;
 605                 }
 606                 if (mq->intr_handle.fd > 0) {
 607                         close(mq->intr_handle.fd);
 608                         mq->intr_handle.fd = -1;
 609                 }
 610         }
 611
 612         memif_free_regions(dev);
 613
 614         /* reset connection configuration */
 615         memset(&pmd->run, 0, sizeof(pmd->run));
 616
 617         MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id,
 618                 (pmd->role == MEMIF_ROLE_MASTER) ? "master" : "slave");
 619 }
 620
 621 static int
 622 memif_msg_receive(struct memif_control_channel *cc)
 623 {
 624         char ctl[CMSG_SPACE(sizeof(int)) +
 625                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
 626         struct msghdr mh = { 0 };
 627         struct iovec iov[1];
 628         memif_msg_t msg = { 0 };
 629         ssize_t size;
 630         int ret = 0;
 631         struct ucred *cr __rte_unused;
 632         cr = 0;
 633         struct cmsghdr *cmsg;
 634         int afd = -1;
 635         int i;
 636         struct pmd_internals *pmd;
 637         struct pmd_process_private *proc_private;
 638
 639         iov[0].iov_base = (void *)&msg;
 640         iov[0].iov_len = sizeof(memif_msg_t);
 641         mh.msg_iov = iov;
 642         mh.msg_iovlen = 1;
 643         mh.msg_control = ctl;
 644         mh.msg_controllen = sizeof(ctl);
 645
 646         size = recvmsg(cc->intr_handle.fd, &mh, 0);
 647         if (size != sizeof(memif_msg_t)) {
 648                 MIF_LOG(DEBUG, "Invalid message size = %zd", size);
 649                 if (size > 0)
 650                         /* 0 means end-of-file, negative size means error,
 651                          * don't send further disconnect message in such cases.
 652                          */
 653                         memif_msg_enq_disconnect(cc, "Invalid message size", 0);
 654                 return -1;
 655         }
 656         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
 657
 658         cmsg = CMSG_FIRSTHDR(&mh);
 659         while (cmsg) {
 660                 if (cmsg->cmsg_level == SOL_SOCKET) {
 661                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
 662                                 cr = (struct ucred *)CMSG_DATA(cmsg);
 663                         else if (cmsg->cmsg_type == SCM_RIGHTS)
 664                                 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
 665                 }
 666                 cmsg = CMSG_NXTHDR(&mh, cmsg);
 667         }
 668
 669         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
 670                 MIF_LOG(DEBUG, "Unexpected message.");
 671                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
 672                 return -1;
 673         }
 674
 675         /* get device from hash data */
 676         switch (msg.type) {
 677         case MEMIF_MSG_TYPE_ACK:
 678                 break;
 679         case MEMIF_MSG_TYPE_HELLO:
 680                 ret = memif_msg_receive_hello(cc->dev, &msg);
 681                 if (ret < 0)
 682                         goto exit;
 683                 ret = memif_init_regions_and_queues(cc->dev);
 684                 if (ret < 0)
 685                         goto exit;
 686                 ret = memif_msg_enq_init(cc->dev);
 687                 if (ret < 0)
 688                         goto exit;
 689                 pmd = cc->dev->data->dev_private;
 690                 proc_private = cc->dev->process_private;
 691                 for (i = 0; i < proc_private->regions_num; i++) {
 692                         ret = memif_msg_enq_add_region(cc->dev, i);
 693                         if (ret < 0)
 694                                 goto exit;
 695                 }
 696                 for (i = 0; i < pmd->run.num_s2m_rings; i++) {
 697                         ret = memif_msg_enq_add_ring(cc->dev, i,
 698                                                      MEMIF_RING_S2M);
 699                         if (ret < 0)
 700                                 goto exit;
 701                 }
 702                 for (i = 0; i < pmd->run.num_m2s_rings; i++) {
 703                         ret = memif_msg_enq_add_ring(cc->dev, i,
 704                                                      MEMIF_RING_M2S);
 705                         if (ret < 0)
 706                                 goto exit;
 707                 }
 708                 ret = memif_msg_enq_connect(cc->dev);
 709                 if (ret < 0)
 710                         goto exit;
 711                 break;
 712         case MEMIF_MSG_TYPE_INIT:
 713                 /*
 714                  * This cc does not have an interface asociated with it.
 715                  * If suitable interface is found it will be assigned here.
 716                  */
 717                 ret = memif_msg_receive_init(cc, &msg);
 718                 if (ret < 0)
 719                         goto exit;
 720                 ret = memif_msg_enq_ack(cc->dev);
 721                 if (ret < 0)
 722                         goto exit;
 723                 break;
 724         case MEMIF_MSG_TYPE_ADD_REGION:
 725                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
 726                 if (ret < 0)
 727                         goto exit;
 728                 ret = memif_msg_enq_ack(cc->dev);
 729                 if (ret < 0)
 730                         goto exit;
 731                 break;
 732         case MEMIF_MSG_TYPE_ADD_RING:
 733                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
 734                 if (ret < 0)
 735                         goto exit;
 736                 ret = memif_msg_enq_ack(cc->dev);
 737                 if (ret < 0)
 738                         goto exit;
 739                 break;
 740         case MEMIF_MSG_TYPE_CONNECT:
 741                 ret = memif_msg_receive_connect(cc->dev, &msg);
 742                 if (ret < 0)
 743                         goto exit;
 744                 ret = memif_msg_enq_connected(cc->dev);
 745                 if (ret < 0)
 746                         goto exit;
 747                 break;
 748         case MEMIF_MSG_TYPE_CONNECTED:
 749                 ret = memif_msg_receive_connected(cc->dev, &msg);
 750                 break;
 751         case MEMIF_MSG_TYPE_DISCONNECT:
 752                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
 753                 if (ret < 0)
 754                         goto exit;
 755                 break;
 756         default:
 757                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
 758                 ret = -1;
 759                 goto exit;
 760         }
 761
 762  exit:
 763         return ret;
 764 }
 765
 766 static void
 767 memif_intr_handler(void *arg)
 768 {
 769         struct memif_control_channel *cc = arg;
 770         int ret;
 771
 772         ret = memif_msg_receive(cc);
 773         /* if driver failed to assign device */
 774         if (cc->dev == NULL) {
 775                 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
 776                                                            memif_intr_handler,
 777                                                            cc,
 778                                                            memif_intr_unregister_handler);
 779                 if (ret < 0)
 780                         MIF_LOG(WARNING,
 781                                 "Failed to unregister control channel callback.");
 782                 return;
 783         }
 784         /* if memif_msg_receive failed */
 785         if (ret < 0)
 786                 goto disconnect;
 787
 788         ret = memif_msg_send_from_queue(cc);
 789         if (ret < 0)
 790                 goto disconnect;
 791
 792         return;
 793
 794  disconnect:
 795         if (cc->dev == NULL) {
 796                 MIF_LOG(WARNING, "eth dev not allocated");
 797                 return;
 798         }
 799         memif_disconnect(cc->dev);
 800 }
 801
 802 static void
 803 memif_listener_handler(void *arg)
 804 {
 805         struct memif_socket *socket = arg;
 806         int sockfd;
 807         int addr_len;
 808         struct sockaddr_un client;
 809         struct memif_control_channel *cc;
 810         int ret;
 811
 812         addr_len = sizeof(client);
 813         sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
 814                         (socklen_t *)&addr_len);
 815         if (sockfd < 0) {
 816                 MIF_LOG(ERR,
 817                         "Failed to accept connection request on socket fd %d",
 818                         socket->intr_handle.fd);
 819                 return;
 820         }
 821
 822         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
 823
 824         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
 825         if (cc == NULL) {
 826                 MIF_LOG(ERR, "Failed to allocate control channel.");
 827                 goto error;
 828         }
 829
 830         cc->intr_handle.fd = sockfd;
 831         cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
 832         cc->socket = socket;
 833         cc->dev = NULL;
 834         TAILQ_INIT(&cc->msg_queue);
 835
 836         ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
 837         if (ret < 0) {
 838                 MIF_LOG(ERR, "Failed to register control channel callback.");
 839                 goto error;
 840         }
 841
 842         ret = memif_msg_enq_hello(cc);
 843         if (ret < 0) {
 844                 MIF_LOG(ERR, "Failed to enqueue hello message.");
 845                 goto error;
 846         }
 847         ret = memif_msg_send_from_queue(cc);
 848         if (ret < 0)
 849                 goto error;
 850
 851         return;
 852
 853  error:
 854         if (sockfd >= 0) {
 855                 close(sockfd);
 856                 sockfd = -1;
 857         }
 858         if (cc != NULL)
 859                 rte_free(cc);
 860 }
 861
 862 static struct memif_socket *
 863 memif_socket_create(char *key, uint8_t listener)
 864 {
 865         struct memif_socket *sock;
 866         struct sockaddr_un un;
 867         int sockfd;
 868         int ret;
 869         int on = 1;
 870
 871         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
 872         if (sock == NULL) {
 873                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
 874                 return NULL;
 875         }
 876
 877         sock->listener = listener;
 878         strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
 879         TAILQ_INIT(&sock->dev_queue);
 880
 881         if (listener != 0) {
 882                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
 883                 if (sockfd < 0)
 884                         goto error;
 885
 886                 un.sun_family = AF_UNIX;
 887                 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
 888
 889                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
 890                                  sizeof(on));
 891                 if (ret < 0)
 892                         goto error;
 893
 894                 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un));
 895                 if (ret < 0)
 896                         goto error;
 897
 898                 ret = listen(sockfd, 1);
 899                 if (ret < 0)
 900                         goto error;
 901
 902                 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename);
 903
 904                 sock->intr_handle.fd = sockfd;
 905                 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
 906                 ret = rte_intr_callback_register(&sock->intr_handle,
 907                                                  memif_listener_handler, sock);
 908                 if (ret < 0) {
 909                         MIF_LOG(ERR, "Failed to register interrupt "
 910                                 "callback for listener socket");
 911                         return NULL;
 912                 }
 913         }
 914
 915         return sock;
 916
 917  error:
 918         MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno));
 919         if (sock != NULL)
 920                 rte_free(sock);
 921         if (sockfd >= 0)
 922                 close(sockfd);
 923         return NULL;
 924 }
 925
 926 static struct rte_hash *
 927 memif_create_socket_hash(void)
 928 {
 929         struct rte_hash_parameters params = { 0 };
 930
 931         params.name = MEMIF_SOCKET_HASH_NAME;
 932         params.entries = 256;
 933         params.key_len = MEMIF_SOCKET_UN_SIZE;
 934         params.hash_func = rte_jhash;
 935         params.hash_func_init_val = 0;
 936         return rte_hash_create(&params);
 937 }
 938
 939 int
 940 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
 941 {
 942         struct pmd_internals *pmd = dev->data->dev_private;
 943         struct memif_socket *socket = NULL;
 944         struct memif_socket_dev_list_elt *elt;
 945         struct pmd_internals *tmp_pmd;
 946         struct rte_hash *hash;
 947         int ret;
 948         char key[MEMIF_SOCKET_UN_SIZE];
 949
 950         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
 951         if (hash == NULL) {
 952                 hash = memif_create_socket_hash();
 953                 if (hash == NULL) {
 954                         MIF_LOG(ERR, "Failed to create memif socket hash.");
 955                         return -1;
 956                 }
 957         }
 958
 959         memset(key, 0, MEMIF_SOCKET_UN_SIZE);
 960         strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
 961         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
 962         if (ret < 0) {
 963                 socket = memif_socket_create(key,
 964                                              (pmd->role == MEMIF_ROLE_SLAVE) ? 0 : 1);
 965                 if (socket == NULL)
 966                         return -1;
 967                 ret = rte_hash_add_key_data(hash, key, socket);
 968                 if (ret < 0) {
 969                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
 970                         return ret;
 971                 }
 972         }
 973         pmd->socket_filename = socket->filename;
 974
 975         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
 976                 tmp_pmd = elt->dev->data->dev_private;
 977                 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) {
 978                         MIF_LOG(ERR, "Two interfaces with the same id (%d) can "
 979                                 "not have the same role.", pmd->id);
 980                         return -1;
 981                 }
 982         }
 983
 984         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
 985         if (elt == NULL) {
 986                 MIF_LOG(ERR, "Failed to add device to socket device list.");
 987                 return -1;
 988         }
 989         elt->dev = dev;
 990         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
 991
 992         return 0;
 993 }
 994
 995 void
 996 memif_socket_remove_device(struct rte_eth_dev *dev)
 997 {
 998         struct pmd_internals *pmd = dev->data->dev_private;
 999         struct memif_socket *socket = NULL;
1000         struct memif_socket_dev_list_elt *elt, *next;
1001         struct rte_hash *hash;
1002         int ret;
1003
1004         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1005         if (hash == NULL)
1006                 return;
1007
1008         if (pmd->socket_filename == NULL)
1009                 return;
1010
1011         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1012                 return;
1013
1014         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1015                 next = TAILQ_NEXT(elt, next);
1016                 if (elt->dev == dev) {
1017                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1018                         rte_free(elt);
1019                         pmd->socket_filename = NULL;
1020                 }
1021         }
1022
1023         /* remove socket, if this was the last device using it */
1024         if (TAILQ_EMPTY(&socket->dev_queue)) {
1025                 rte_hash_del_key(hash, socket->filename);
1026                 if (socket->listener) {
1027                         /* remove listener socket file,
1028                          * so we can create new one later.
1029                          */
1030                         ret = remove(socket->filename);
1031                         if (ret < 0)
1032                                 MIF_LOG(ERR, "Failed to remove socket file: %s",
1033                                         socket->filename);
1034                 }
1035                 rte_free(socket);
1036         }
1037 }
1038
1039 int
1040 memif_connect_master(struct rte_eth_dev *dev)
1041 {
1042         struct pmd_internals *pmd = dev->data->dev_private;
1043
1044         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1045         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1046         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1047         return 0;
1048 }
1049
1050 int
1051 memif_connect_slave(struct rte_eth_dev *dev)
1052 {
1053         int sockfd;
1054         int ret;
1055         struct sockaddr_un sun;
1056         struct pmd_internals *pmd = dev->data->dev_private;
1057
1058         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1059         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1060         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1061
1062         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1063         if (sockfd < 0) {
1064                 MIF_LOG(ERR, "Failed to open socket.");
1065                 return -1;
1066         }
1067
1068         sun.sun_family = AF_UNIX;
1069
1070         memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1);
1071
1072         ret = connect(sockfd, (struct sockaddr *)&sun,
1073                       sizeof(struct sockaddr_un));
1074         if (ret < 0) {
1075                 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename);
1076                 goto error;
1077         }
1078
1079         MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename);
1080
1081         pmd->cc = rte_zmalloc("memif-cc",
1082                               sizeof(struct memif_control_channel), 0);
1083         if (pmd->cc == NULL) {
1084                 MIF_LOG(ERR, "Failed to allocate control channel.");
1085                 goto error;
1086         }
1087
1088         pmd->cc->intr_handle.fd = sockfd;
1089         pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1090         pmd->cc->socket = NULL;
1091         pmd->cc->dev = dev;
1092         TAILQ_INIT(&pmd->cc->msg_queue);
1093
1094         ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1095                                          memif_intr_handler, pmd->cc);
1096         if (ret < 0) {
1097                 MIF_LOG(ERR, "Failed to register interrupt callback for control fd");
1098                 goto error;
1099         }
1100
1101         return 0;
1102
1103  error:
1104         if (sockfd >= 0) {
1105                 close(sockfd);
1106                 sockfd = -1;
1107         }
1108         if (pmd->cc != NULL) {
1109                 rte_free(pmd->cc);
1110                 pmd->cc = NULL;
1111         }
1112         return -1;
1113 }