drivers/net/memif/memif_socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/ioctl.h>
  11 #include <errno.h>
  12
  13 #include <rte_version.h>
  14 #include <rte_mbuf.h>
  15 #include <rte_ether.h>
  16 #include <rte_ethdev_driver.h>
  17 #include <rte_ethdev_vdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_bus_vdev.h>
  21 #include <rte_hash.h>
  22 #include <rte_jhash.h>
  23 #include <rte_string_fns.h>
  24
  25 #include "rte_eth_memif.h"
  26 #include "memif_socket.h"
  27
  28 static void memif_intr_handler(void *arg);
  29
  30 static ssize_t
  31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
  32 {
  33         struct msghdr mh = { 0 };
  34         struct iovec iov[1];
  35         struct cmsghdr *cmsg;
  36         char ctl[CMSG_SPACE(sizeof(int))];
  37
  38         iov[0].iov_base = msg;
  39         iov[0].iov_len = sizeof(memif_msg_t);
  40         mh.msg_iov = iov;
  41         mh.msg_iovlen = 1;
  42
  43         if (afd > 0) {
  44                 memset(&ctl, 0, sizeof(ctl));
  45                 mh.msg_control = ctl;
  46                 mh.msg_controllen = sizeof(ctl);
  47                 cmsg = CMSG_FIRSTHDR(&mh);
  48                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  49                 cmsg->cmsg_level = SOL_SOCKET;
  50                 cmsg->cmsg_type = SCM_RIGHTS;
  51                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
  52         }
  53
  54         return sendmsg(fd, &mh, 0);
  55 }
  56
  57 static int
  58 memif_msg_send_from_queue(struct memif_control_channel *cc)
  59 {
  60         ssize_t size;
  61         int ret = 0;
  62         struct memif_msg_queue_elt *e;
  63
  64         e = TAILQ_FIRST(&cc->msg_queue);
  65         if (e == NULL)
  66                 return 0;
  67
  68         size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
  69         if (size != sizeof(memif_msg_t)) {
  70                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
  71                 ret = -1;
  72         } else {
  73                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
  74         }
  75         TAILQ_REMOVE(&cc->msg_queue, e, next);
  76         rte_free(e);
  77
  78         return ret;
  79 }
  80
  81 static struct memif_msg_queue_elt *
  82 memif_msg_enq(struct memif_control_channel *cc)
  83 {
  84         struct memif_msg_queue_elt *e;
  85
  86         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
  87         if (e == NULL) {
  88                 MIF_LOG(ERR, "Failed to allocate control message.");
  89                 return NULL;
  90         }
  91
  92         e->fd = -1;
  93         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
  94
  95         return e;
  96 }
  97
  98 void
  99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
 100                          int err_code)
 101 {
 102         struct memif_msg_queue_elt *e;
 103         struct pmd_internals *pmd;
 104         memif_msg_disconnect_t *d;
 105
 106         if (cc == NULL) {
 107                 MIF_LOG(DEBUG, "Missing control channel.");
 108                 return;
 109         }
 110
 111         e = memif_msg_enq(cc);
 112         if (e == NULL) {
 113                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
 114                 return;
 115         }
 116
 117         d = &e->msg.disconnect;
 118
 119         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
 120         d->code = err_code;
 121
 122         if (reason != NULL) {
 123                 strlcpy((char *)d->string, reason, sizeof(d->string));
 124                 if (cc->dev != NULL) {
 125                         pmd = cc->dev->data->dev_private;
 126                         strlcpy(pmd->local_disc_string, reason,
 127                                 sizeof(pmd->local_disc_string));
 128                 }
 129         }
 130 }
 131
 132 static int
 133 memif_msg_enq_hello(struct memif_control_channel *cc)
 134 {
 135         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
 136         memif_msg_hello_t *h;
 137
 138         if (e == NULL)
 139                 return -1;
 140
 141         h = &e->msg.hello;
 142
 143         e->msg.type = MEMIF_MSG_TYPE_HELLO;
 144         h->min_version = MEMIF_VERSION;
 145         h->max_version = MEMIF_VERSION;
 146         h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 147         h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 148         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
 149         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
 150
 151         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
 152
 153         return 0;
 154 }
 155
 156 static int
 157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
 158 {
 159         struct pmd_internals *pmd = dev->data->dev_private;
 160         memif_msg_hello_t *h = &msg->hello;
 161
 162         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
 163                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
 164                 return -1;
 165         }
 166
 167         /* Set parameters for active connection */
 168         pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1,
 169                                            pmd->cfg.num_s2m_rings);
 170         pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1,
 171                                            pmd->cfg.num_m2s_rings);
 172         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
 173                                             pmd->cfg.log2_ring_size);
 174         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
 175
 176         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
 177
 178         MIF_LOG(DEBUG, "%s: Connecting to %s.",
 179                 rte_vdev_device_name(pmd->vdev), pmd->remote_name);
 180
 181         return 0;
 182 }
 183
 184 static int
 185 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
 186 {
 187         memif_msg_init_t *i = &msg->init;
 188         struct memif_socket_dev_list_elt *elt;
 189         struct pmd_internals *pmd;
 190         struct rte_eth_dev *dev;
 191
 192         if (i->version != MEMIF_VERSION) {
 193                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
 194                 return -1;
 195         }
 196
 197         if (cc->socket == NULL) {
 198                 memif_msg_enq_disconnect(cc, "Device error", 0);
 199                 return -1;
 200         }
 201
 202         /* Find device with requested ID */
 203         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
 204                 dev = elt->dev;
 205                 pmd = dev->data->dev_private;
 206                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
 207                     pmd->id == i->id) {
 208                         /* assign control channel to device */
 209                         cc->dev = dev;
 210                         pmd->cc = cc;
 211
 212                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
 213                                 memif_msg_enq_disconnect(pmd->cc,
 214                                                          "Only ethernet mode supported",
 215                                                          0);
 216                                 return -1;
 217                         }
 218
 219                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
 220                                            ETH_MEMIF_FLAG_CONNECTED)) {
 221                                 memif_msg_enq_disconnect(pmd->cc,
 222                                                          "Already connected", 0);
 223                                 return -1;
 224                         }
 225                         strlcpy(pmd->remote_name, (char *)i->name,
 226                                 sizeof(pmd->remote_name));
 227
 228                         if (*pmd->secret != '\0') {
 229                                 if (*i->secret == '\0') {
 230                                         memif_msg_enq_disconnect(pmd->cc,
 231                                                                  "Secret required", 0);
 232                                         return -1;
 233                                 }
 234                                 if (strncmp(pmd->secret, (char *)i->secret,
 235                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
 236                                         memif_msg_enq_disconnect(pmd->cc,
 237                                                                  "Incorrect secret", 0);
 238                                         return -1;
 239                                 }
 240                         }
 241
 242                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
 243                         return 0;
 244                 }
 245         }
 246
 247         /* ID not found on this socket */
 248         MIF_LOG(DEBUG, "ID %u not found.", i->id);
 249         memif_msg_enq_disconnect(cc, "ID not found", 0);
 250         return -1;
 251 }
 252
 253 static int
 254 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
 255                              int fd)
 256 {
 257         struct pmd_internals *pmd = dev->data->dev_private;
 258         struct pmd_process_private *proc_private = dev->process_private;
 259         memif_msg_add_region_t *ar = &msg->add_region;
 260         struct memif_region *r;
 261
 262         if (fd < 0) {
 263                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
 264                 return -1;
 265         }
 266
 267         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
 268                         ar->index != proc_private->regions_num ||
 269                         proc_private->regions[ar->index] != NULL) {
 270                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
 271                 return -1;
 272         }
 273
 274         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 275         if (r == NULL) {
 276                 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
 277                 return -ENOMEM;
 278         }
 279
 280         r->fd = fd;
 281         r->region_size = ar->size;
 282         r->addr = NULL;
 283
 284         proc_private->regions[ar->index] = r;
 285         proc_private->regions_num++;
 286
 287         return 0;
 288 }
 289
 290 static int
 291 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
 292 {
 293         struct pmd_internals *pmd = dev->data->dev_private;
 294         memif_msg_add_ring_t *ar = &msg->add_ring;
 295         struct memif_queue *mq;
 296
 297         if (fd < 0) {
 298                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
 299                 return -1;
 300         }
 301
 302         /* check if we have enough queues */
 303         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) {
 304                 if (ar->index >= pmd->cfg.num_s2m_rings) {
 305                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 306                         return -1;
 307                 }
 308                 pmd->run.num_s2m_rings++;
 309         } else {
 310                 if (ar->index >= pmd->cfg.num_m2s_rings) {
 311                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 312                         return -1;
 313                 }
 314                 pmd->run.num_m2s_rings++;
 315         }
 316
 317         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ?
 318             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
 319
 320         mq->intr_handle.fd = fd;
 321         mq->log2_ring_size = ar->log2_ring_size;
 322         mq->region = ar->region;
 323         mq->ring_offset = ar->offset;
 324
 325         return 0;
 326 }
 327
 328 static int
 329 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
 330 {
 331         struct pmd_internals *pmd = dev->data->dev_private;
 332         memif_msg_connect_t *c = &msg->connect;
 333         int ret;
 334
 335         ret = memif_connect(dev);
 336         if (ret < 0)
 337                 return ret;
 338
 339         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 340                 sizeof(pmd->remote_if_name));
 341         MIF_LOG(INFO, "%s: Remote interface %s connected.",
 342                 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name);
 343
 344         return 0;
 345 }
 346
 347 static int
 348 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
 349 {
 350         struct pmd_internals *pmd = dev->data->dev_private;
 351         memif_msg_connected_t *c = &msg->connected;
 352         int ret;
 353
 354         ret = memif_connect(dev);
 355         if (ret < 0)
 356                 return ret;
 357
 358         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 359                 sizeof(pmd->remote_if_name));
 360         MIF_LOG(INFO, "%s: Remote interface %s connected.",
 361                 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name);
 362
 363         return 0;
 364 }
 365
 366 static int
 367 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
 368 {
 369         struct pmd_internals *pmd = dev->data->dev_private;
 370         memif_msg_disconnect_t *d = &msg->disconnect;
 371
 372         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
 373         strlcpy(pmd->remote_disc_string, (char *)d->string,
 374                 sizeof(pmd->remote_disc_string));
 375
 376         MIF_LOG(INFO, "%s: Disconnect received: %s",
 377                 rte_vdev_device_name(pmd->vdev), pmd->remote_disc_string);
 378
 379         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
 380         memif_disconnect(dev);
 381         return 0;
 382 }
 383
 384 static int
 385 memif_msg_enq_ack(struct rte_eth_dev *dev)
 386 {
 387         struct pmd_internals *pmd = dev->data->dev_private;
 388         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 389         if (e == NULL)
 390                 return -1;
 391
 392         e->msg.type = MEMIF_MSG_TYPE_ACK;
 393
 394         return 0;
 395 }
 396
 397 static int
 398 memif_msg_enq_init(struct rte_eth_dev *dev)
 399 {
 400         struct pmd_internals *pmd = dev->data->dev_private;
 401         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 402         memif_msg_init_t *i = &e->msg.init;
 403
 404         if (e == NULL)
 405                 return -1;
 406
 407         i = &e->msg.init;
 408         e->msg.type = MEMIF_MSG_TYPE_INIT;
 409         i->version = MEMIF_VERSION;
 410         i->id = pmd->id;
 411         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
 412
 413         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
 414
 415         if (*pmd->secret != '\0')
 416                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
 417
 418         return 0;
 419 }
 420
 421 static int
 422 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
 423 {
 424         struct pmd_internals *pmd = dev->data->dev_private;
 425         struct pmd_process_private *proc_private = dev->process_private;
 426         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 427         memif_msg_add_region_t *ar;
 428         struct memif_region *mr = proc_private->regions[idx];
 429
 430         if (e == NULL)
 431                 return -1;
 432
 433         ar = &e->msg.add_region;
 434         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
 435         e->fd = mr->fd;
 436         ar->index = idx;
 437         ar->size = mr->region_size;
 438
 439         return 0;
 440 }
 441
 442 static int
 443 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
 444                        memif_ring_type_t type)
 445 {
 446         struct pmd_internals *pmd = dev->data->dev_private;
 447         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 448         struct memif_queue *mq;
 449         memif_msg_add_ring_t *ar;
 450
 451         if (e == NULL)
 452                 return -1;
 453
 454         ar = &e->msg.add_ring;
 455         mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] :
 456             dev->data->rx_queues[idx];
 457
 458         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
 459         e->fd = mq->intr_handle.fd;
 460         ar->index = idx;
 461         ar->offset = mq->ring_offset;
 462         ar->region = mq->region;
 463         ar->log2_ring_size = mq->log2_ring_size;
 464         ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
 465         ar->private_hdr_size = 0;
 466
 467         return 0;
 468 }
 469
 470 static int
 471 memif_msg_enq_connect(struct rte_eth_dev *dev)
 472 {
 473         struct pmd_internals *pmd = dev->data->dev_private;
 474         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 475         const char *name = rte_vdev_device_name(pmd->vdev);
 476         memif_msg_connect_t *c;
 477
 478         if (e == NULL)
 479                 return -1;
 480
 481         c = &e->msg.connect;
 482         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
 483         strlcpy((char *)c->if_name, name, sizeof(c->if_name));
 484
 485         return 0;
 486 }
 487
 488 static int
 489 memif_msg_enq_connected(struct rte_eth_dev *dev)
 490 {
 491         struct pmd_internals *pmd = dev->data->dev_private;
 492         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 493         const char *name = rte_vdev_device_name(pmd->vdev);
 494         memif_msg_connected_t *c;
 495
 496         if (e == NULL)
 497                 return -1;
 498
 499         c = &e->msg.connected;
 500         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
 501         strlcpy((char *)c->if_name, name, sizeof(c->if_name));
 502
 503         return 0;
 504 }
 505
 506 static void
 507 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
 508 {
 509         struct memif_msg_queue_elt *elt;
 510         struct memif_control_channel *cc = arg;
 511
 512         /* close control channel fd */
 513         close(intr_handle->fd);
 514         /* clear message queue */
 515         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
 516                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
 517                 rte_free(elt);
 518         }
 519         /* free control channel */
 520         rte_free(cc);
 521 }
 522
 523 void
 524 memif_disconnect(struct rte_eth_dev *dev)
 525 {
 526         struct pmd_internals *pmd = dev->data->dev_private;
 527         struct pmd_process_private *proc_private = dev->process_private;
 528         struct memif_msg_queue_elt *elt, *next;
 529         struct memif_queue *mq;
 530         struct rte_intr_handle *ih;
 531         int i;
 532         int ret;
 533
 534         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 535         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
 536         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
 537
 538         if (pmd->cc != NULL) {
 539                 /* Clear control message queue (except disconnect message if any). */
 540                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
 541                         next = TAILQ_NEXT(elt, next);
 542                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
 543                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
 544                                 rte_free(elt);
 545                         }
 546                 }
 547                 /* send disconnect message (if there is any in queue) */
 548                 memif_msg_send_from_queue(pmd->cc);
 549
 550                 /* at this point, there should be no more messages in queue */
 551                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
 552                         MIF_LOG(WARNING,
 553                                 "Unexpected message(s) in message queue.");
 554                 }
 555
 556                 ih = &pmd->cc->intr_handle;
 557                 if (ih->fd > 0) {
 558                         ret = rte_intr_callback_unregister(ih,
 559                                                         memif_intr_handler,
 560                                                         pmd->cc);
 561                         /*
 562                          * If callback is active (disconnecting based on
 563                          * received control message).
 564                          */
 565                         if (ret == -EAGAIN) {
 566                                 ret = rte_intr_callback_unregister_pending(ih,
 567                                                         memif_intr_handler,
 568                                                         pmd->cc,
 569                                                         memif_intr_unregister_handler);
 570                         } else if (ret > 0) {
 571                                 close(ih->fd);
 572                                 rte_free(pmd->cc);
 573                         }
 574                         pmd->cc = NULL;
 575                         if (ret <= 0)
 576                                 MIF_LOG(WARNING,
 577                                         "Failed to unregister control channel callback.");
 578                 }
 579         }
 580
 581         /* unconfig interrupts */
 582         for (i = 0; i < pmd->cfg.num_s2m_rings; i++) {
 583                 if (pmd->role == MEMIF_ROLE_SLAVE) {
 584                         if (dev->data->tx_queues != NULL)
 585                                 mq = dev->data->tx_queues[i];
 586                         else
 587                                 continue;
 588                 } else {
 589                         if (dev->data->rx_queues != NULL)
 590                                 mq = dev->data->rx_queues[i];
 591                         else
 592                                 continue;
 593                 }
 594                 if (mq->intr_handle.fd > 0) {
 595                         close(mq->intr_handle.fd);
 596                         mq->intr_handle.fd = -1;
 597                 }
 598         }
 599         for (i = 0; i < pmd->cfg.num_m2s_rings; i++) {
 600                 if (pmd->role == MEMIF_ROLE_MASTER) {
 601                         if (dev->data->tx_queues != NULL)
 602                                 mq = dev->data->tx_queues[i];
 603                         else
 604                                 continue;
 605                 } else {
 606                         if (dev->data->rx_queues != NULL)
 607                                 mq = dev->data->rx_queues[i];
 608                         else
 609                                 continue;
 610                 }
 611                 if (mq->intr_handle.fd > 0) {
 612                         close(mq->intr_handle.fd);
 613                         mq->intr_handle.fd = -1;
 614                 }
 615         }
 616
 617         memif_free_regions(proc_private);
 618
 619         /* reset connection configuration */
 620         memset(&pmd->run, 0, sizeof(pmd->run));
 621
 622         MIF_LOG(DEBUG, "Disconnected.");
 623 }
 624
 625 static int
 626 memif_msg_receive(struct memif_control_channel *cc)
 627 {
 628         char ctl[CMSG_SPACE(sizeof(int)) +
 629                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
 630         struct msghdr mh = { 0 };
 631         struct iovec iov[1];
 632         memif_msg_t msg = { 0 };
 633         ssize_t size;
 634         int ret = 0;
 635         struct ucred *cr __rte_unused;
 636         cr = 0;
 637         struct cmsghdr *cmsg;
 638         int afd = -1;
 639         int i;
 640         struct pmd_internals *pmd;
 641         struct pmd_process_private *proc_private;
 642
 643         iov[0].iov_base = (void *)&msg;
 644         iov[0].iov_len = sizeof(memif_msg_t);
 645         mh.msg_iov = iov;
 646         mh.msg_iovlen = 1;
 647         mh.msg_control = ctl;
 648         mh.msg_controllen = sizeof(ctl);
 649
 650         size = recvmsg(cc->intr_handle.fd, &mh, 0);
 651         if (size != sizeof(memif_msg_t)) {
 652                 MIF_LOG(DEBUG, "Invalid message size.");
 653                 memif_msg_enq_disconnect(cc, "Invalid message size", 0);
 654                 return -1;
 655         }
 656         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
 657
 658         cmsg = CMSG_FIRSTHDR(&mh);
 659         while (cmsg) {
 660                 if (cmsg->cmsg_level == SOL_SOCKET) {
 661                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
 662                                 cr = (struct ucred *)CMSG_DATA(cmsg);
 663                         else if (cmsg->cmsg_type == SCM_RIGHTS)
 664                                 memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
 665                 }
 666                 cmsg = CMSG_NXTHDR(&mh, cmsg);
 667         }
 668
 669         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
 670                 MIF_LOG(DEBUG, "Unexpected message.");
 671                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
 672                 return -1;
 673         }
 674
 675         /* get device from hash data */
 676         switch (msg.type) {
 677         case MEMIF_MSG_TYPE_ACK:
 678                 break;
 679         case MEMIF_MSG_TYPE_HELLO:
 680                 ret = memif_msg_receive_hello(cc->dev, &msg);
 681                 if (ret < 0)
 682                         goto exit;
 683                 ret = memif_init_regions_and_queues(cc->dev);
 684                 if (ret < 0)
 685                         goto exit;
 686                 ret = memif_msg_enq_init(cc->dev);
 687                 if (ret < 0)
 688                         goto exit;
 689                 pmd = cc->dev->data->dev_private;
 690                 proc_private = cc->dev->process_private;
 691                 for (i = 0; i < proc_private->regions_num; i++) {
 692                         ret = memif_msg_enq_add_region(cc->dev, i);
 693                         if (ret < 0)
 694                                 goto exit;
 695                 }
 696                 for (i = 0; i < pmd->run.num_s2m_rings; i++) {
 697                         ret = memif_msg_enq_add_ring(cc->dev, i,
 698                                                      MEMIF_RING_S2M);
 699                         if (ret < 0)
 700                                 goto exit;
 701                 }
 702                 for (i = 0; i < pmd->run.num_m2s_rings; i++) {
 703                         ret = memif_msg_enq_add_ring(cc->dev, i,
 704                                                      MEMIF_RING_M2S);
 705                         if (ret < 0)
 706                                 goto exit;
 707                 }
 708                 ret = memif_msg_enq_connect(cc->dev);
 709                 if (ret < 0)
 710                         goto exit;
 711                 break;
 712         case MEMIF_MSG_TYPE_INIT:
 713                 /*
 714                  * This cc does not have an interface asociated with it.
 715                  * If suitable interface is found it will be assigned here.
 716                  */
 717                 ret = memif_msg_receive_init(cc, &msg);
 718                 if (ret < 0)
 719                         goto exit;
 720                 ret = memif_msg_enq_ack(cc->dev);
 721                 if (ret < 0)
 722                         goto exit;
 723                 break;
 724         case MEMIF_MSG_TYPE_ADD_REGION:
 725                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
 726                 if (ret < 0)
 727                         goto exit;
 728                 ret = memif_msg_enq_ack(cc->dev);
 729                 if (ret < 0)
 730                         goto exit;
 731                 break;
 732         case MEMIF_MSG_TYPE_ADD_RING:
 733                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
 734                 if (ret < 0)
 735                         goto exit;
 736                 ret = memif_msg_enq_ack(cc->dev);
 737                 if (ret < 0)
 738                         goto exit;
 739                 break;
 740         case MEMIF_MSG_TYPE_CONNECT:
 741                 ret = memif_msg_receive_connect(cc->dev, &msg);
 742                 if (ret < 0)
 743                         goto exit;
 744                 ret = memif_msg_enq_connected(cc->dev);
 745                 if (ret < 0)
 746                         goto exit;
 747                 break;
 748         case MEMIF_MSG_TYPE_CONNECTED:
 749                 ret = memif_msg_receive_connected(cc->dev, &msg);
 750                 break;
 751         case MEMIF_MSG_TYPE_DISCONNECT:
 752                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
 753                 if (ret < 0)
 754                         goto exit;
 755                 break;
 756         default:
 757                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
 758                 ret = -1;
 759                 goto exit;
 760         }
 761
 762  exit:
 763         return ret;
 764 }
 765
 766 static void
 767 memif_intr_handler(void *arg)
 768 {
 769         struct memif_control_channel *cc = arg;
 770         int ret;
 771
 772         ret = memif_msg_receive(cc);
 773         /* if driver failed to assign device */
 774         if (cc->dev == NULL) {
 775                 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
 776                                                            memif_intr_handler,
 777                                                            cc,
 778                                                            memif_intr_unregister_handler);
 779                 if (ret < 0)
 780                         MIF_LOG(WARNING,
 781                                 "Failed to unregister control channel callback.");
 782                 return;
 783         }
 784         /* if memif_msg_receive failed */
 785         if (ret < 0)
 786                 goto disconnect;
 787
 788         ret = memif_msg_send_from_queue(cc);
 789         if (ret < 0)
 790                 goto disconnect;
 791
 792         return;
 793
 794  disconnect:
 795         if (cc->dev == NULL) {
 796                 MIF_LOG(WARNING, "eth dev not allocated");
 797                 return;
 798         }
 799         memif_disconnect(cc->dev);
 800 }
 801
 802 static void
 803 memif_listener_handler(void *arg)
 804 {
 805         struct memif_socket *socket = arg;
 806         int sockfd;
 807         int addr_len;
 808         struct sockaddr_un client;
 809         struct memif_control_channel *cc;
 810         int ret;
 811
 812         addr_len = sizeof(client);
 813         sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
 814                         (socklen_t *)&addr_len);
 815         if (sockfd < 0) {
 816                 MIF_LOG(ERR,
 817                         "Failed to accept connection request on socket fd %d",
 818                         socket->intr_handle.fd);
 819                 return;
 820         }
 821
 822         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
 823
 824         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
 825         if (cc == NULL) {
 826                 MIF_LOG(ERR, "Failed to allocate control channel.");
 827                 goto error;
 828         }
 829
 830         cc->intr_handle.fd = sockfd;
 831         cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
 832         cc->socket = socket;
 833         cc->dev = NULL;
 834         TAILQ_INIT(&cc->msg_queue);
 835
 836         ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
 837         if (ret < 0) {
 838                 MIF_LOG(ERR, "Failed to register control channel callback.");
 839                 goto error;
 840         }
 841
 842         ret = memif_msg_enq_hello(cc);
 843         if (ret < 0) {
 844                 MIF_LOG(ERR, "Failed to enqueue hello message.");
 845                 goto error;
 846         }
 847         ret = memif_msg_send_from_queue(cc);
 848         if (ret < 0)
 849                 goto error;
 850
 851         return;
 852
 853  error:
 854         if (sockfd >= 0) {
 855                 close(sockfd);
 856                 sockfd = -1;
 857         }
 858         if (cc != NULL)
 859                 rte_free(cc);
 860 }
 861
 862 static struct memif_socket *
 863 memif_socket_create(struct pmd_internals *pmd,
 864                     const char *key, uint8_t listener)
 865 {
 866         struct memif_socket *sock;
 867         struct sockaddr_un un;
 868         int sockfd;
 869         int ret;
 870         int on = 1;
 871
 872         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
 873         if (sock == NULL) {
 874                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
 875                 return NULL;
 876         }
 877
 878         sock->listener = listener;
 879         strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
 880         TAILQ_INIT(&sock->dev_queue);
 881
 882         if (listener != 0) {
 883                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
 884                 if (sockfd < 0)
 885                         goto error;
 886
 887                 un.sun_family = AF_UNIX;
 888                 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
 889
 890                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
 891                                  sizeof(on));
 892                 if (ret < 0)
 893                         goto error;
 894
 895                 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un));
 896                 if (ret < 0)
 897                         goto error;
 898
 899                 ret = listen(sockfd, 1);
 900                 if (ret < 0)
 901                         goto error;
 902
 903                 MIF_LOG(DEBUG, "%s: Memif listener socket %s created.",
 904                         rte_vdev_device_name(pmd->vdev), sock->filename);
 905
 906                 sock->intr_handle.fd = sockfd;
 907                 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
 908                 ret = rte_intr_callback_register(&sock->intr_handle,
 909                                                  memif_listener_handler, sock);
 910                 if (ret < 0) {
 911                         MIF_LOG(ERR, "%s: Failed to register interrupt "
 912                                 "callback for listener socket",
 913                                 rte_vdev_device_name(pmd->vdev));
 914                         return NULL;
 915                 }
 916         }
 917
 918         return sock;
 919
 920  error:
 921         MIF_LOG(ERR, "%s: Failed to setup socket %s: %s",
 922                 rte_vdev_device_name(pmd->vdev) ?
 923                 rte_vdev_device_name(pmd->vdev) : "NULL", key, strerror(errno));
 924         if (sock != NULL)
 925                 rte_free(sock);
 926         if (sockfd >= 0)
 927                 close(sockfd);
 928         return NULL;
 929 }
 930
 931 static struct rte_hash *
 932 memif_create_socket_hash(void)
 933 {
 934         struct rte_hash_parameters params = { 0 };
 935
 936         params.name = MEMIF_SOCKET_HASH_NAME;
 937         params.entries = 256;
 938         params.key_len = MEMIF_SOCKET_UN_SIZE;
 939         params.hash_func = rte_jhash;
 940         params.hash_func_init_val = 0;
 941         return rte_hash_create(&params);
 942 }
 943
 944 int
 945 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
 946 {
 947         struct pmd_internals *pmd = dev->data->dev_private;
 948         struct memif_socket *socket = NULL;
 949         struct memif_socket_dev_list_elt *elt;
 950         struct pmd_internals *tmp_pmd;
 951         struct rte_hash *hash;
 952         int ret;
 953         char key[MEMIF_SOCKET_UN_SIZE];
 954
 955         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
 956         if (hash == NULL) {
 957                 hash = memif_create_socket_hash();
 958                 if (hash == NULL) {
 959                         MIF_LOG(ERR, "Failed to create memif socket hash.");
 960                         return -1;
 961                 }
 962         }
 963
 964         memset(key, 0, MEMIF_SOCKET_UN_SIZE);
 965         strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
 966         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
 967         if (ret < 0) {
 968                 socket = memif_socket_create(pmd, key,
 969                                              (pmd->role ==
 970                                               MEMIF_ROLE_SLAVE) ? 0 : 1);
 971                 if (socket == NULL)
 972                         return -1;
 973                 ret = rte_hash_add_key_data(hash, key, socket);
 974                 if (ret < 0) {
 975                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
 976                         return ret;
 977                 }
 978         }
 979         pmd->socket_filename = socket->filename;
 980
 981         if (socket->listener != 0 && pmd->role == MEMIF_ROLE_SLAVE) {
 982                 MIF_LOG(ERR, "Socket is a listener.");
 983                 return -1;
 984         } else if ((socket->listener == 0) && (pmd->role == MEMIF_ROLE_MASTER)) {
 985                 MIF_LOG(ERR, "Socket is not a listener.");
 986                 return -1;
 987         }
 988
 989         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
 990                 tmp_pmd = elt->dev->data->dev_private;
 991                 if (tmp_pmd->id == pmd->id) {
 992                         MIF_LOG(ERR, "Memif device with id %d already "
 993                                 "exists on socket %s",
 994                                 pmd->id, socket->filename);
 995                         return -1;
 996                 }
 997         }
 998
 999         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
1000         if (elt == NULL) {
1001                 MIF_LOG(ERR, "%s: Failed to add device to socket device list.",
1002                         rte_vdev_device_name(pmd->vdev));
1003                 return -1;
1004         }
1005         elt->dev = dev;
1006         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1007
1008         return 0;
1009 }
1010
1011 void
1012 memif_socket_remove_device(struct rte_eth_dev *dev)
1013 {
1014         struct pmd_internals *pmd = dev->data->dev_private;
1015         struct memif_socket *socket = NULL;
1016         struct memif_socket_dev_list_elt *elt, *next;
1017         struct rte_hash *hash;
1018         int ret;
1019
1020         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1021         if (hash == NULL)
1022                 return;
1023
1024         if (pmd->socket_filename == NULL)
1025                 return;
1026
1027         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1028                 return;
1029
1030         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1031                 next = TAILQ_NEXT(elt, next);
1032                 if (elt->dev == dev) {
1033                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1034                         rte_free(elt);
1035                         pmd->socket_filename = NULL;
1036                 }
1037         }
1038
1039         /* remove socket, if this was the last device using it */
1040         if (TAILQ_EMPTY(&socket->dev_queue)) {
1041                 rte_hash_del_key(hash, socket->filename);
1042                 if (socket->listener) {
1043                         /* remove listener socket file,
1044                          * so we can create new one later.
1045                          */
1046                         ret = remove(socket->filename);
1047                         if (ret < 0)
1048                                 MIF_LOG(ERR, "Failed to remove socket file: %s",
1049                                         socket->filename);
1050                 }
1051                 rte_free(socket);
1052         }
1053 }
1054
1055 int
1056 memif_connect_master(struct rte_eth_dev *dev)
1057 {
1058         struct pmd_internals *pmd = dev->data->dev_private;
1059
1060         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1061         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1062         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1063         return 0;
1064 }
1065
1066 int
1067 memif_connect_slave(struct rte_eth_dev *dev)
1068 {
1069         int sockfd;
1070         int ret;
1071         struct sockaddr_un sun;
1072         struct pmd_internals *pmd = dev->data->dev_private;
1073
1074         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1075         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1076         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1077
1078         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1079         if (sockfd < 0) {
1080                 MIF_LOG(ERR, "%s: Failed to open socket.",
1081                         rte_vdev_device_name(pmd->vdev));
1082                 return -1;
1083         }
1084
1085         sun.sun_family = AF_UNIX;
1086
1087         memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1);
1088
1089         ret = connect(sockfd, (struct sockaddr *)&sun,
1090                       sizeof(struct sockaddr_un));
1091         if (ret < 0) {
1092                 MIF_LOG(ERR, "%s: Failed to connect socket: %s.",
1093                         rte_vdev_device_name(pmd->vdev), pmd->socket_filename);
1094                 goto error;
1095         }
1096
1097         MIF_LOG(DEBUG, "%s: Memif socket: %s connected.",
1098                 rte_vdev_device_name(pmd->vdev), pmd->socket_filename);
1099
1100         pmd->cc = rte_zmalloc("memif-cc",
1101                               sizeof(struct memif_control_channel), 0);
1102         if (pmd->cc == NULL) {
1103                 MIF_LOG(ERR, "%s: Failed to allocate control channel.",
1104                         rte_vdev_device_name(pmd->vdev));
1105                 goto error;
1106         }
1107
1108         pmd->cc->intr_handle.fd = sockfd;
1109         pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1110         pmd->cc->socket = NULL;
1111         pmd->cc->dev = dev;
1112         TAILQ_INIT(&pmd->cc->msg_queue);
1113
1114         ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1115                                          memif_intr_handler, pmd->cc);
1116         if (ret < 0) {
1117                 MIF_LOG(ERR, "%s: Failed to register interrupt callback "
1118                         "for control fd", rte_vdev_device_name(pmd->vdev));
1119                 goto error;
1120         }
1121
1122         return 0;
1123
1124  error:
1125         if (sockfd >= 0) {
1126                 close(sockfd);
1127                 sockfd = -1;
1128         }
1129         if (pmd->cc != NULL) {
1130                 rte_free(pmd->cc);
1131                 pmd->cc = NULL;
1132         }
1133         return -1;
1134 }