drivers/net/memif/memif_socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/ioctl.h>
  11 #include <errno.h>
  12
  13 #include <rte_version.h>
  14 #include <rte_mbuf.h>
  15 #include <rte_ether.h>
  16 #include <ethdev_driver.h>
  17 #include <ethdev_vdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_bus_vdev.h>
  21 #include <rte_hash.h>
  22 #include <rte_jhash.h>
  23 #include <rte_string_fns.h>
  24
  25 #include "rte_eth_memif.h"
  26 #include "memif_socket.h"
  27
  28 static void memif_intr_handler(void *arg);
  29
  30 static ssize_t
  31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
  32 {
  33         struct msghdr mh = { 0 };
  34         struct iovec iov[1];
  35         struct cmsghdr *cmsg;
  36         char ctl[CMSG_SPACE(sizeof(int))];
  37
  38         iov[0].iov_base = msg;
  39         iov[0].iov_len = sizeof(memif_msg_t);
  40         mh.msg_iov = iov;
  41         mh.msg_iovlen = 1;
  42
  43         if (afd > 0) {
  44                 memset(&ctl, 0, sizeof(ctl));
  45                 mh.msg_control = ctl;
  46                 mh.msg_controllen = sizeof(ctl);
  47                 cmsg = CMSG_FIRSTHDR(&mh);
  48                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  49                 cmsg->cmsg_level = SOL_SOCKET;
  50                 cmsg->cmsg_type = SCM_RIGHTS;
  51                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
  52         }
  53
  54         return sendmsg(fd, &mh, 0);
  55 }
  56
  57 static int
  58 memif_msg_send_from_queue(struct memif_control_channel *cc)
  59 {
  60         ssize_t size;
  61         int ret = 0;
  62         struct memif_msg_queue_elt *e;
  63
  64         e = TAILQ_FIRST(&cc->msg_queue);
  65         if (e == NULL)
  66                 return 0;
  67
  68         if (rte_intr_fd_get(cc->intr_handle) < 0)
  69                 return -1;
  70
  71         size = memif_msg_send(rte_intr_fd_get(cc->intr_handle), &e->msg,
  72                               e->fd);
  73         if (size != sizeof(memif_msg_t)) {
  74                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
  75                 ret = -1;
  76         } else {
  77                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
  78         }
  79         TAILQ_REMOVE(&cc->msg_queue, e, next);
  80         rte_free(e);
  81
  82         return ret;
  83 }
  84
  85 static struct memif_msg_queue_elt *
  86 memif_msg_enq(struct memif_control_channel *cc)
  87 {
  88         struct memif_msg_queue_elt *e;
  89
  90         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
  91         if (e == NULL) {
  92                 MIF_LOG(ERR, "Failed to allocate control message.");
  93                 return NULL;
  94         }
  95
  96         e->fd = -1;
  97         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
  98
  99         return e;
 100 }
 101
 102 void
 103 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
 104                          int err_code)
 105 {
 106         struct memif_msg_queue_elt *e;
 107         struct pmd_internals *pmd;
 108         memif_msg_disconnect_t *d;
 109
 110         if (cc == NULL) {
 111                 MIF_LOG(DEBUG, "Missing control channel.");
 112                 return;
 113         }
 114
 115         e = memif_msg_enq(cc);
 116         if (e == NULL) {
 117                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
 118                 return;
 119         }
 120
 121         d = &e->msg.disconnect;
 122
 123         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
 124         d->code = err_code;
 125
 126         if (reason != NULL) {
 127                 strlcpy((char *)d->string, reason, sizeof(d->string));
 128                 if (cc->dev != NULL) {
 129                         pmd = cc->dev->data->dev_private;
 130                         strlcpy(pmd->local_disc_string, reason,
 131                                 sizeof(pmd->local_disc_string));
 132                 }
 133         }
 134 }
 135
 136 static int
 137 memif_msg_enq_hello(struct memif_control_channel *cc)
 138 {
 139         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
 140         memif_msg_hello_t *h;
 141
 142         if (e == NULL)
 143                 return -1;
 144
 145         h = &e->msg.hello;
 146
 147         e->msg.type = MEMIF_MSG_TYPE_HELLO;
 148         h->min_version = MEMIF_VERSION;
 149         h->max_version = MEMIF_VERSION;
 150         h->max_c2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 151         h->max_s2c_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 152         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
 153         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
 154
 155         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
 156
 157         return 0;
 158 }
 159
 160 static int
 161 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
 162 {
 163         struct pmd_internals *pmd = dev->data->dev_private;
 164         memif_msg_hello_t *h = &msg->hello;
 165
 166         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
 167                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
 168                 return -1;
 169         }
 170
 171         /* Set parameters for active connection */
 172         pmd->run.num_c2s_rings = RTE_MIN(h->max_c2s_ring + 1,
 173                                            pmd->cfg.num_c2s_rings);
 174         pmd->run.num_s2c_rings = RTE_MIN(h->max_s2c_ring + 1,
 175                                            pmd->cfg.num_s2c_rings);
 176         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
 177                                             pmd->cfg.log2_ring_size);
 178         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
 179
 180         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
 181
 182         MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name);
 183
 184         return 0;
 185 }
 186
 187 static int
 188 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
 189 {
 190         memif_msg_init_t *i = &msg->init;
 191         struct memif_socket_dev_list_elt *elt;
 192         struct pmd_internals *pmd;
 193         struct rte_eth_dev *dev;
 194
 195         if (i->version != MEMIF_VERSION) {
 196                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
 197                 return -1;
 198         }
 199
 200         if (cc->socket == NULL) {
 201                 memif_msg_enq_disconnect(cc, "Device error", 0);
 202                 return -1;
 203         }
 204
 205         /* Find device with requested ID */
 206         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
 207                 dev = elt->dev;
 208                 pmd = dev->data->dev_private;
 209                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
 210                     (pmd->id == i->id) && (pmd->role == MEMIF_ROLE_SERVER)) {
 211                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
 212                                            ETH_MEMIF_FLAG_CONNECTED)) {
 213                                 memif_msg_enq_disconnect(cc,
 214                                                          "Already connected", 0);
 215                                 return -1;
 216                         }
 217
 218                         /* assign control channel to device */
 219                         cc->dev = dev;
 220                         pmd->cc = cc;
 221
 222                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
 223                                 memif_msg_enq_disconnect(pmd->cc,
 224                                                          "Only ethernet mode supported",
 225                                                          0);
 226                                 return -1;
 227                         }
 228
 229                         strlcpy(pmd->remote_name, (char *)i->name,
 230                                 sizeof(pmd->remote_name));
 231
 232                         if (*pmd->secret != '\0') {
 233                                 if (*i->secret == '\0') {
 234                                         memif_msg_enq_disconnect(pmd->cc,
 235                                                                  "Secret required", 0);
 236                                         return -1;
 237                                 }
 238                                 if (strncmp(pmd->secret, (char *)i->secret,
 239                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
 240                                         memif_msg_enq_disconnect(pmd->cc,
 241                                                                  "Incorrect secret", 0);
 242                                         return -1;
 243                                 }
 244                         }
 245
 246                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
 247                         return 0;
 248                 }
 249         }
 250
 251         /* ID not found on this socket */
 252         MIF_LOG(DEBUG, "ID %u not found.", i->id);
 253         memif_msg_enq_disconnect(cc, "ID not found", 0);
 254         return -1;
 255 }
 256
 257 static int
 258 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
 259                              int fd)
 260 {
 261         struct pmd_internals *pmd = dev->data->dev_private;
 262         struct pmd_process_private *proc_private = dev->process_private;
 263         memif_msg_add_region_t *ar = &msg->add_region;
 264         struct memif_region *r;
 265
 266         if (fd < 0) {
 267                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
 268                 return -1;
 269         }
 270
 271         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
 272                         ar->index != proc_private->regions_num ||
 273                         proc_private->regions[ar->index] != NULL) {
 274                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
 275                 return -1;
 276         }
 277
 278         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 279         if (r == NULL) {
 280                 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
 281                 return -ENOMEM;
 282         }
 283
 284         r->fd = fd;
 285         r->region_size = ar->size;
 286         r->addr = NULL;
 287
 288         proc_private->regions[ar->index] = r;
 289         proc_private->regions_num++;
 290
 291         return 0;
 292 }
 293
 294 static int
 295 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
 296 {
 297         struct pmd_internals *pmd = dev->data->dev_private;
 298         memif_msg_add_ring_t *ar = &msg->add_ring;
 299         struct memif_queue *mq;
 300
 301         if (fd < 0) {
 302                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
 303                 return -1;
 304         }
 305
 306         /* check if we have enough queues */
 307         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) {
 308                 if (ar->index >= pmd->cfg.num_c2s_rings) {
 309                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 310                         return -1;
 311                 }
 312                 pmd->run.num_c2s_rings++;
 313         } else {
 314                 if (ar->index >= pmd->cfg.num_s2c_rings) {
 315                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 316                         return -1;
 317                 }
 318                 pmd->run.num_s2c_rings++;
 319         }
 320
 321         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_C2S) ?
 322             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
 323
 324         if (rte_intr_fd_set(mq->intr_handle, fd))
 325                 return -1;
 326
 327         mq->log2_ring_size = ar->log2_ring_size;
 328         mq->region = ar->region;
 329         mq->ring_offset = ar->offset;
 330
 331         return 0;
 332 }
 333
 334 static int
 335 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
 336 {
 337         struct pmd_internals *pmd = dev->data->dev_private;
 338         memif_msg_connect_t *c = &msg->connect;
 339         int ret;
 340
 341         ret = memif_connect(dev);
 342         if (ret < 0)
 343                 return ret;
 344
 345         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 346                 sizeof(pmd->remote_if_name));
 347         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 348
 349         return 0;
 350 }
 351
 352 static int
 353 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
 354 {
 355         struct pmd_internals *pmd = dev->data->dev_private;
 356         memif_msg_connected_t *c = &msg->connected;
 357         int ret;
 358
 359         ret = memif_connect(dev);
 360         if (ret < 0)
 361                 return ret;
 362
 363         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 364                 sizeof(pmd->remote_if_name));
 365         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 366
 367         return 0;
 368 }
 369
 370 static int
 371 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
 372 {
 373         struct pmd_internals *pmd = dev->data->dev_private;
 374         memif_msg_disconnect_t *d = &msg->disconnect;
 375
 376         memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string));
 377         strlcpy(pmd->remote_disc_string, (char *)d->string,
 378                 sizeof(pmd->remote_disc_string));
 379
 380         MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string);
 381
 382         memset(pmd->local_disc_string, 0, 96);
 383         memif_disconnect(dev);
 384         return 0;
 385 }
 386
 387 static int
 388 memif_msg_enq_ack(struct rte_eth_dev *dev)
 389 {
 390         struct pmd_internals *pmd = dev->data->dev_private;
 391         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 392         if (e == NULL)
 393                 return -1;
 394
 395         e->msg.type = MEMIF_MSG_TYPE_ACK;
 396
 397         return 0;
 398 }
 399
 400 static int
 401 memif_msg_enq_init(struct rte_eth_dev *dev)
 402 {
 403         struct pmd_internals *pmd = dev->data->dev_private;
 404         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 405         memif_msg_init_t *i;
 406
 407         if (e == NULL)
 408                 return -1;
 409         i = &e->msg.init;
 410         e->msg.type = MEMIF_MSG_TYPE_INIT;
 411         i->version = MEMIF_VERSION;
 412         i->id = pmd->id;
 413         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
 414
 415         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
 416
 417         if (*pmd->secret != '\0')
 418                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
 419
 420         return 0;
 421 }
 422
 423 static int
 424 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
 425 {
 426         struct pmd_internals *pmd = dev->data->dev_private;
 427         struct pmd_process_private *proc_private = dev->process_private;
 428         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 429         memif_msg_add_region_t *ar;
 430         struct memif_region *mr = proc_private->regions[idx];
 431
 432         if (e == NULL)
 433                 return -1;
 434
 435         ar = &e->msg.add_region;
 436         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
 437         e->fd = mr->fd;
 438         ar->index = idx;
 439         ar->size = mr->region_size;
 440
 441         return 0;
 442 }
 443
 444 static int
 445 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
 446                        memif_ring_type_t type)
 447 {
 448         struct pmd_internals *pmd = dev->data->dev_private;
 449         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 450         struct memif_queue *mq;
 451         memif_msg_add_ring_t *ar;
 452
 453         if (e == NULL)
 454                 return -1;
 455
 456         ar = &e->msg.add_ring;
 457         mq = (type == MEMIF_RING_C2S) ? dev->data->tx_queues[idx] :
 458             dev->data->rx_queues[idx];
 459
 460         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
 461         e->fd = rte_intr_fd_get(mq->intr_handle);
 462         ar->index = idx;
 463         ar->offset = mq->ring_offset;
 464         ar->region = mq->region;
 465         ar->log2_ring_size = mq->log2_ring_size;
 466         ar->flags = (type == MEMIF_RING_C2S) ? MEMIF_MSG_ADD_RING_FLAG_C2S : 0;
 467         ar->private_hdr_size = 0;
 468
 469         return 0;
 470 }
 471
 472 static int
 473 memif_msg_enq_connect(struct rte_eth_dev *dev)
 474 {
 475         struct pmd_internals *pmd = dev->data->dev_private;
 476         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 477         memif_msg_connect_t *c;
 478
 479         if (e == NULL)
 480                 return -1;
 481
 482         c = &e->msg.connect;
 483         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
 484         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 485
 486         return 0;
 487 }
 488
 489 static int
 490 memif_msg_enq_connected(struct rte_eth_dev *dev)
 491 {
 492         struct pmd_internals *pmd = dev->data->dev_private;
 493         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 494         memif_msg_connected_t *c;
 495
 496         if (e == NULL)
 497                 return -1;
 498
 499         c = &e->msg.connected;
 500         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
 501         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 502
 503         return 0;
 504 }
 505
 506 static void
 507 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
 508 {
 509         struct memif_msg_queue_elt *elt;
 510         struct memif_control_channel *cc = arg;
 511
 512         /* close control channel fd */
 513         if (rte_intr_fd_get(intr_handle) >= 0)
 514                 close(rte_intr_fd_get(intr_handle));
 515         /* clear message queue */
 516         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
 517                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
 518                 rte_free(elt);
 519         }
 520         rte_intr_instance_free(cc->intr_handle);
 521         /* free control channel */
 522         rte_free(cc);
 523 }
 524
 525 void
 526 memif_disconnect(struct rte_eth_dev *dev)
 527 {
 528         struct pmd_internals *pmd = dev->data->dev_private;
 529         struct memif_msg_queue_elt *elt, *next;
 530         struct memif_queue *mq;
 531         struct rte_intr_handle *ih;
 532         int i;
 533         int ret;
 534
 535         dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
 536         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
 537         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
 538
 539         rte_spinlock_lock(&pmd->cc_lock);
 540         if (pmd->cc != NULL) {
 541                 /* Clear control message queue (except disconnect message if any). */
 542                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
 543                         next = TAILQ_NEXT(elt, next);
 544                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
 545                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
 546                                 rte_free(elt);
 547                         }
 548                 }
 549                 /* send disconnect message (if there is any in queue) */
 550                 memif_msg_send_from_queue(pmd->cc);
 551
 552                 /* at this point, there should be no more messages in queue */
 553                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
 554                         MIF_LOG(WARNING,
 555                                 "Unexpected message(s) in message queue.");
 556                 }
 557
 558                 ih = pmd->cc->intr_handle;
 559                 if (rte_intr_fd_get(ih) > 0) {
 560                         ret = rte_intr_callback_unregister(ih,
 561                                                         memif_intr_handler,
 562                                                         pmd->cc);
 563                         /*
 564                          * If callback is active (disconnecting based on
 565                          * received control message).
 566                          */
 567                         if (ret == -EAGAIN) {
 568                                 ret = rte_intr_callback_unregister_pending(ih,
 569                                                         memif_intr_handler,
 570                                                         pmd->cc,
 571                                                         memif_intr_unregister_handler);
 572                         } else if (ret > 0) {
 573                                 close(rte_intr_fd_get(ih));
 574                                 rte_intr_instance_free(ih);
 575                                 rte_free(pmd->cc);
 576                         }
 577                         pmd->cc = NULL;
 578                         if (ret <= 0)
 579                                 MIF_LOG(WARNING,
 580                                         "Failed to unregister control channel callback.");
 581                 }
 582         }
 583         rte_spinlock_unlock(&pmd->cc_lock);
 584
 585         /* unconfig interrupts */
 586         for (i = 0; i < pmd->cfg.num_c2s_rings; i++) {
 587                 if (pmd->role == MEMIF_ROLE_CLIENT) {
 588                         if (dev->data->tx_queues != NULL)
 589                                 mq = dev->data->tx_queues[i];
 590                         else
 591                                 continue;
 592                 } else {
 593                         if (dev->data->rx_queues != NULL)
 594                                 mq = dev->data->rx_queues[i];
 595                         else
 596                                 continue;
 597                 }
 598
 599                 if (rte_intr_fd_get(mq->intr_handle) > 0) {
 600                         close(rte_intr_fd_get(mq->intr_handle));
 601                         rte_intr_fd_set(mq->intr_handle, -1);
 602                 }
 603         }
 604         for (i = 0; i < pmd->cfg.num_s2c_rings; i++) {
 605                 if (pmd->role == MEMIF_ROLE_SERVER) {
 606                         if (dev->data->tx_queues != NULL)
 607                                 mq = dev->data->tx_queues[i];
 608                         else
 609                                 continue;
 610                 } else {
 611                         if (dev->data->rx_queues != NULL)
 612                                 mq = dev->data->rx_queues[i];
 613                         else
 614                                 continue;
 615                 }
 616
 617                 if (rte_intr_fd_get(mq->intr_handle) > 0) {
 618                         close(rte_intr_fd_get(mq->intr_handle));
 619                         rte_intr_fd_set(mq->intr_handle, -1);
 620                 }
 621         }
 622
 623         memif_free_regions(dev);
 624
 625         /* reset connection configuration */
 626         memset(&pmd->run, 0, sizeof(pmd->run));
 627
 628         MIF_LOG(DEBUG, "Disconnected, id: %d, role: %s.", pmd->id,
 629                 (pmd->role == MEMIF_ROLE_SERVER) ? "server" : "client");
 630 }
 631
 632 static int
 633 memif_msg_receive(struct memif_control_channel *cc)
 634 {
 635         char ctl[CMSG_SPACE(sizeof(int)) +
 636                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
 637         struct msghdr mh = { 0 };
 638         struct iovec iov[1];
 639         memif_msg_t msg = { 0 };
 640         ssize_t size;
 641         int ret = 0;
 642         struct ucred *cr __rte_unused;
 643         cr = 0;
 644         struct cmsghdr *cmsg;
 645         int afd = -1;
 646         int i;
 647         struct pmd_internals *pmd;
 648         struct pmd_process_private *proc_private;
 649
 650         iov[0].iov_base = (void *)&msg;
 651         iov[0].iov_len = sizeof(memif_msg_t);
 652         mh.msg_iov = iov;
 653         mh.msg_iovlen = 1;
 654         mh.msg_control = ctl;
 655         mh.msg_controllen = sizeof(ctl);
 656
 657         if (rte_intr_fd_get(cc->intr_handle) < 0)
 658                 return -1;
 659
 660         size = recvmsg(rte_intr_fd_get(cc->intr_handle), &mh, 0);
 661         if (size != sizeof(memif_msg_t)) {
 662                 MIF_LOG(DEBUG, "Invalid message size = %zd", size);
 663                 if (size > 0)
 664                         /* 0 means end-of-file, negative size means error,
 665                          * don't send further disconnect message in such cases.
 666                          */
 667                         memif_msg_enq_disconnect(cc, "Invalid message size", 0);
 668                 return -1;
 669         }
 670         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
 671
 672         cmsg = CMSG_FIRSTHDR(&mh);
 673         while (cmsg) {
 674                 if (cmsg->cmsg_level == SOL_SOCKET) {
 675                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
 676                                 cr = (struct ucred *)CMSG_DATA(cmsg);
 677                         else if (cmsg->cmsg_type == SCM_RIGHTS)
 678                                 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
 679                 }
 680                 cmsg = CMSG_NXTHDR(&mh, cmsg);
 681         }
 682
 683         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
 684                 MIF_LOG(DEBUG, "Unexpected message.");
 685                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
 686                 return -1;
 687         }
 688
 689         /* get device from hash data */
 690         switch (msg.type) {
 691         case MEMIF_MSG_TYPE_ACK:
 692                 break;
 693         case MEMIF_MSG_TYPE_HELLO:
 694                 ret = memif_msg_receive_hello(cc->dev, &msg);
 695                 if (ret < 0)
 696                         goto exit;
 697                 ret = memif_init_regions_and_queues(cc->dev);
 698                 if (ret < 0)
 699                         goto exit;
 700                 ret = memif_msg_enq_init(cc->dev);
 701                 if (ret < 0)
 702                         goto exit;
 703                 pmd = cc->dev->data->dev_private;
 704                 proc_private = cc->dev->process_private;
 705                 for (i = 0; i < proc_private->regions_num; i++) {
 706                         ret = memif_msg_enq_add_region(cc->dev, i);
 707                         if (ret < 0)
 708                                 goto exit;
 709                 }
 710                 for (i = 0; i < pmd->run.num_c2s_rings; i++) {
 711                         ret = memif_msg_enq_add_ring(cc->dev, i,
 712                                                      MEMIF_RING_C2S);
 713                         if (ret < 0)
 714                                 goto exit;
 715                 }
 716                 for (i = 0; i < pmd->run.num_s2c_rings; i++) {
 717                         ret = memif_msg_enq_add_ring(cc->dev, i,
 718                                                      MEMIF_RING_S2C);
 719                         if (ret < 0)
 720                                 goto exit;
 721                 }
 722                 ret = memif_msg_enq_connect(cc->dev);
 723                 if (ret < 0)
 724                         goto exit;
 725                 break;
 726         case MEMIF_MSG_TYPE_INIT:
 727                 /*
 728                  * This cc does not have an interface associated with it.
 729                  * If suitable interface is found it will be assigned here.
 730                  */
 731                 ret = memif_msg_receive_init(cc, &msg);
 732                 if (ret < 0)
 733                         goto exit;
 734                 ret = memif_msg_enq_ack(cc->dev);
 735                 if (ret < 0)
 736                         goto exit;
 737                 break;
 738         case MEMIF_MSG_TYPE_ADD_REGION:
 739                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
 740                 if (ret < 0)
 741                         goto exit;
 742                 ret = memif_msg_enq_ack(cc->dev);
 743                 if (ret < 0)
 744                         goto exit;
 745                 break;
 746         case MEMIF_MSG_TYPE_ADD_RING:
 747                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
 748                 if (ret < 0)
 749                         goto exit;
 750                 ret = memif_msg_enq_ack(cc->dev);
 751                 if (ret < 0)
 752                         goto exit;
 753                 break;
 754         case MEMIF_MSG_TYPE_CONNECT:
 755                 ret = memif_msg_receive_connect(cc->dev, &msg);
 756                 if (ret < 0)
 757                         goto exit;
 758                 ret = memif_msg_enq_connected(cc->dev);
 759                 if (ret < 0)
 760                         goto exit;
 761                 break;
 762         case MEMIF_MSG_TYPE_CONNECTED:
 763                 ret = memif_msg_receive_connected(cc->dev, &msg);
 764                 break;
 765         case MEMIF_MSG_TYPE_DISCONNECT:
 766                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
 767                 if (ret < 0)
 768                         goto exit;
 769                 break;
 770         default:
 771                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
 772                 ret = -1;
 773                 goto exit;
 774         }
 775
 776  exit:
 777         return ret;
 778 }
 779
 780 static void
 781 memif_intr_handler(void *arg)
 782 {
 783         struct memif_control_channel *cc = arg;
 784         int ret;
 785
 786         ret = memif_msg_receive(cc);
 787         /* if driver failed to assign device */
 788         if (cc->dev == NULL) {
 789                 memif_msg_send_from_queue(cc);
 790                 ret = rte_intr_callback_unregister_pending(cc->intr_handle,
 791                                                            memif_intr_handler,
 792                                                            cc,
 793                                                            memif_intr_unregister_handler);
 794                 if (ret < 0)
 795                         MIF_LOG(WARNING,
 796                                 "Failed to unregister control channel callback.");
 797                 return;
 798         }
 799         /* if memif_msg_receive failed */
 800         if (ret < 0)
 801                 goto disconnect;
 802
 803         ret = memif_msg_send_from_queue(cc);
 804         if (ret < 0)
 805                 goto disconnect;
 806
 807         return;
 808
 809  disconnect:
 810         if (cc->dev == NULL) {
 811                 MIF_LOG(WARNING, "eth dev not allocated");
 812                 return;
 813         }
 814         memif_disconnect(cc->dev);
 815 }
 816
 817 static void
 818 memif_listener_handler(void *arg)
 819 {
 820         struct memif_socket *socket = arg;
 821         int sockfd;
 822         int addr_len;
 823         struct sockaddr_un client;
 824         struct memif_control_channel *cc;
 825         int ret;
 826
 827         addr_len = sizeof(client);
 828         sockfd = accept(rte_intr_fd_get(socket->intr_handle),
 829                         (struct sockaddr *)&client, (socklen_t *)&addr_len);
 830         if (sockfd < 0) {
 831                 MIF_LOG(ERR,
 832                         "Failed to accept connection request on socket fd %d",
 833                         rte_intr_fd_get(socket->intr_handle));
 834                 return;
 835         }
 836
 837         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
 838
 839         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
 840         if (cc == NULL) {
 841                 MIF_LOG(ERR, "Failed to allocate control channel.");
 842                 goto error;
 843         }
 844
 845         /* Allocate interrupt instance */
 846         cc->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
 847         if (cc->intr_handle == NULL) {
 848                 MIF_LOG(ERR, "Failed to allocate intr handle");
 849                 goto error;
 850         }
 851
 852         if (rte_intr_fd_set(cc->intr_handle, sockfd))
 853                 goto error;
 854
 855         if (rte_intr_type_set(cc->intr_handle, RTE_INTR_HANDLE_EXT))
 856                 goto error;
 857
 858         cc->socket = socket;
 859         cc->dev = NULL;
 860         TAILQ_INIT(&cc->msg_queue);
 861
 862         ret = rte_intr_callback_register(cc->intr_handle, memif_intr_handler,
 863                                          cc);
 864         if (ret < 0) {
 865                 MIF_LOG(ERR, "Failed to register control channel callback.");
 866                 goto error;
 867         }
 868
 869         ret = memif_msg_enq_hello(cc);
 870         if (ret < 0) {
 871                 MIF_LOG(ERR, "Failed to enqueue hello message.");
 872                 goto error;
 873         }
 874         ret = memif_msg_send_from_queue(cc);
 875         if (ret < 0)
 876                 goto error;
 877
 878         return;
 879
 880  error:
 881         if (sockfd >= 0) {
 882                 close(sockfd);
 883                 sockfd = -1;
 884         }
 885         if (cc != NULL) {
 886                 rte_intr_instance_free(cc->intr_handle);
 887                 rte_free(cc);
 888         }
 889 }
 890
 891 static struct memif_socket *
 892 memif_socket_create(char *key, uint8_t listener, bool is_abstract)
 893 {
 894         struct memif_socket *sock;
 895         struct sockaddr_un un = { 0 };
 896         uint32_t sunlen;
 897         int sockfd;
 898         int ret;
 899         int on = 1;
 900
 901         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
 902         if (sock == NULL) {
 903                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
 904                 return NULL;
 905         }
 906
 907         sock->listener = listener;
 908         strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
 909         TAILQ_INIT(&sock->dev_queue);
 910
 911         if (listener != 0) {
 912                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
 913                 if (sockfd < 0)
 914                         goto error;
 915
 916                 un.sun_family = AF_UNIX;
 917                 if (is_abstract) {
 918                         /* abstract address */
 919                         un.sun_path[0] = '\0';
 920                         strlcpy(un.sun_path + 1, sock->filename, MEMIF_SOCKET_UN_SIZE - 1);
 921                         sunlen = RTE_MIN(1 + strlen(sock->filename),
 922                                          MEMIF_SOCKET_UN_SIZE) +
 923                                  sizeof(un) - sizeof(un.sun_path);
 924                 } else {
 925                         sunlen = sizeof(un);
 926                         strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
 927                 }
 928
 929                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
 930                                  sizeof(on));
 931                 if (ret < 0)
 932                         goto error;
 933
 934                 ret = bind(sockfd, (struct sockaddr *)&un, sunlen);
 935                 if (ret < 0)
 936                         goto error;
 937
 938                 ret = listen(sockfd, 1);
 939                 if (ret < 0)
 940                         goto error;
 941
 942                 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename);
 943
 944                 /* Allocate interrupt instance */
 945                 sock->intr_handle =
 946                         rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
 947                 if (sock->intr_handle == NULL) {
 948                         MIF_LOG(ERR, "Failed to allocate intr handle");
 949                         goto error;
 950                 }
 951
 952                 if (rte_intr_fd_set(sock->intr_handle, sockfd))
 953                         goto error;
 954
 955                 if (rte_intr_type_set(sock->intr_handle, RTE_INTR_HANDLE_EXT))
 956                         goto error;
 957
 958                 ret = rte_intr_callback_register(sock->intr_handle,
 959                                                  memif_listener_handler, sock);
 960                 if (ret < 0) {
 961                         MIF_LOG(ERR, "Failed to register interrupt "
 962                                 "callback for listener socket");
 963                         return NULL;
 964                 }
 965         }
 966
 967         return sock;
 968
 969  error:
 970         MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno));
 971         if (sock != NULL) {
 972                 rte_intr_instance_free(sock->intr_handle);
 973                 rte_free(sock);
 974         }
 975         if (sockfd >= 0)
 976                 close(sockfd);
 977         return NULL;
 978 }
 979
 980 static struct rte_hash *
 981 memif_create_socket_hash(void)
 982 {
 983         struct rte_hash_parameters params = { 0 };
 984
 985         params.name = MEMIF_SOCKET_HASH_NAME;
 986         params.entries = 256;
 987         params.key_len = MEMIF_SOCKET_UN_SIZE;
 988         params.hash_func = rte_jhash;
 989         params.hash_func_init_val = 0;
 990         params.socket_id = SOCKET_ID_ANY;
 991         return rte_hash_create(&params);
 992 }
 993
 994 int
 995 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
 996 {
 997         struct pmd_internals *pmd = dev->data->dev_private;
 998         struct memif_socket *socket = NULL;
 999         struct memif_socket_dev_list_elt *elt;
1000         struct pmd_internals *tmp_pmd;
1001         struct rte_hash *hash;
1002         int ret;
1003         char key[MEMIF_SOCKET_UN_SIZE];
1004
1005         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1006         if (hash == NULL) {
1007                 hash = memif_create_socket_hash();
1008                 if (hash == NULL) {
1009                         MIF_LOG(ERR, "Failed to create memif socket hash.");
1010                         return -1;
1011                 }
1012         }
1013
1014         memset(key, 0, MEMIF_SOCKET_UN_SIZE);
1015         strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
1016         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
1017         if (ret < 0) {
1018                 socket = memif_socket_create(key,
1019                         (pmd->role == MEMIF_ROLE_CLIENT) ? 0 : 1,
1020                         pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT);
1021                 if (socket == NULL)
1022                         return -1;
1023                 ret = rte_hash_add_key_data(hash, key, socket);
1024                 if (ret < 0) {
1025                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
1026                         return ret;
1027                 }
1028         }
1029         pmd->socket_filename = socket->filename;
1030
1031         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
1032                 tmp_pmd = elt->dev->data->dev_private;
1033                 if (tmp_pmd->id == pmd->id && tmp_pmd->role == pmd->role) {
1034                         MIF_LOG(ERR, "Two interfaces with the same id (%d) can "
1035                                 "not have the same role.", pmd->id);
1036                         return -1;
1037                 }
1038         }
1039
1040         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
1041         if (elt == NULL) {
1042                 MIF_LOG(ERR, "Failed to add device to socket device list.");
1043                 return -1;
1044         }
1045         elt->dev = dev;
1046         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1047
1048         return 0;
1049 }
1050
1051 void
1052 memif_socket_remove_device(struct rte_eth_dev *dev)
1053 {
1054         struct pmd_internals *pmd = dev->data->dev_private;
1055         struct memif_socket *socket = NULL;
1056         struct memif_socket_dev_list_elt *elt, *next;
1057         struct rte_hash *hash;
1058         int ret;
1059
1060         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1061         if (hash == NULL)
1062                 return;
1063
1064         if (pmd->socket_filename == NULL)
1065                 return;
1066
1067         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1068                 return;
1069
1070         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1071                 next = TAILQ_NEXT(elt, next);
1072                 if (elt->dev == dev) {
1073                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1074                         rte_free(elt);
1075                         pmd->socket_filename = NULL;
1076                 }
1077         }
1078
1079         /* remove socket, if this was the last device using it */
1080         if (TAILQ_EMPTY(&socket->dev_queue)) {
1081                 rte_hash_del_key(hash, socket->filename);
1082                 if (socket->listener && !(pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT)) {
1083                         /* remove listener socket file,
1084                          * so we can create new one later.
1085                          */
1086                         ret = remove(socket->filename);
1087                         if (ret < 0)
1088                                 MIF_LOG(ERR, "Failed to remove socket file: %s",
1089                                         socket->filename);
1090                 }
1091                 if (pmd->role != MEMIF_ROLE_CLIENT)
1092                         rte_intr_instance_free(socket->intr_handle);
1093                 rte_free(socket);
1094         }
1095 }
1096
1097 int
1098 memif_connect_server(struct rte_eth_dev *dev)
1099 {
1100         struct pmd_internals *pmd = dev->data->dev_private;
1101
1102         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1103         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1104         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1105         return 0;
1106 }
1107
1108 int
1109 memif_connect_client(struct rte_eth_dev *dev)
1110 {
1111         int sockfd;
1112         int ret;
1113         uint32_t sunlen;
1114         struct sockaddr_un sun = { 0 };
1115         struct pmd_internals *pmd = dev->data->dev_private;
1116
1117         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1118         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1119         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1120
1121         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1122         if (sockfd < 0) {
1123                 MIF_LOG(ERR, "Failed to open socket.");
1124                 return -1;
1125         }
1126
1127         sun.sun_family = AF_UNIX;
1128         sunlen = sizeof(struct sockaddr_un);
1129         if (pmd->flags & ETH_MEMIF_FLAG_SOCKET_ABSTRACT) {
1130                 /* abstract address */
1131                 sun.sun_path[0] = '\0';
1132                 strlcpy(sun.sun_path + 1,  pmd->socket_filename, MEMIF_SOCKET_UN_SIZE - 1);
1133                 sunlen = RTE_MIN(strlen(pmd->socket_filename) + 1,
1134                                  MEMIF_SOCKET_UN_SIZE) +
1135                          sizeof(sun) - sizeof(sun.sun_path);
1136         } else {
1137                 strlcpy(sun.sun_path,  pmd->socket_filename, MEMIF_SOCKET_UN_SIZE);
1138         }
1139
1140         ret = connect(sockfd, (struct sockaddr *)&sun, sunlen);
1141         if (ret < 0) {
1142                 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename);
1143                 goto error;
1144         }
1145
1146         MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename);
1147
1148         pmd->cc = rte_zmalloc("memif-cc",
1149                               sizeof(struct memif_control_channel), 0);
1150         if (pmd->cc == NULL) {
1151                 MIF_LOG(ERR, "Failed to allocate control channel.");
1152                 goto error;
1153         }
1154
1155         /* Allocate interrupt instance */
1156         pmd->cc->intr_handle =
1157                 rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
1158         if (pmd->cc->intr_handle == NULL) {
1159                 MIF_LOG(ERR, "Failed to allocate intr handle");
1160                 goto error;
1161         }
1162
1163         if (rte_intr_fd_set(pmd->cc->intr_handle, sockfd))
1164                 goto error;
1165
1166         if (rte_intr_type_set(pmd->cc->intr_handle, RTE_INTR_HANDLE_EXT))
1167                 goto error;
1168
1169         pmd->cc->socket = NULL;
1170         pmd->cc->dev = dev;
1171         TAILQ_INIT(&pmd->cc->msg_queue);
1172
1173         ret = rte_intr_callback_register(pmd->cc->intr_handle,
1174                                          memif_intr_handler, pmd->cc);
1175         if (ret < 0) {
1176                 MIF_LOG(ERR, "Failed to register interrupt callback for control fd");
1177                 goto error;
1178         }
1179
1180         return 0;
1181
1182  error:
1183         if (sockfd >= 0) {
1184                 close(sockfd);
1185                 sockfd = -1;
1186         }
1187         if (pmd->cc != NULL) {
1188                 rte_intr_instance_free(pmd->cc->intr_handle);
1189                 rte_free(pmd->cc);
1190                 pmd->cc = NULL;
1191         }
1192         return -1;
1193 }