drivers/net/memif/memif_socket.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
   3  */
   4
   5 #include <stdlib.h>
   6 #include <fcntl.h>
   7 #include <unistd.h>
   8 #include <sys/types.h>
   9 #include <sys/socket.h>
  10 #include <sys/ioctl.h>
  11 #include <errno.h>
  12
  13 #include <rte_version.h>
  14 #include <rte_mbuf.h>
  15 #include <rte_ether.h>
  16 #include <rte_ethdev_driver.h>
  17 #include <rte_ethdev_vdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_bus_vdev.h>
  21 #include <rte_hash.h>
  22 #include <rte_jhash.h>
  23 #include <rte_string_fns.h>
  24
  25 #include "rte_eth_memif.h"
  26 #include "memif_socket.h"
  27
  28 static void memif_intr_handler(void *arg);
  29
  30 static ssize_t
  31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
  32 {
  33         struct msghdr mh = { 0 };
  34         struct iovec iov[1];
  35         struct cmsghdr *cmsg;
  36         char ctl[CMSG_SPACE(sizeof(int))];
  37
  38         iov[0].iov_base = msg;
  39         iov[0].iov_len = sizeof(memif_msg_t);
  40         mh.msg_iov = iov;
  41         mh.msg_iovlen = 1;
  42
  43         if (afd > 0) {
  44                 memset(&ctl, 0, sizeof(ctl));
  45                 mh.msg_control = ctl;
  46                 mh.msg_controllen = sizeof(ctl);
  47                 cmsg = CMSG_FIRSTHDR(&mh);
  48                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  49                 cmsg->cmsg_level = SOL_SOCKET;
  50                 cmsg->cmsg_type = SCM_RIGHTS;
  51                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
  52         }
  53
  54         return sendmsg(fd, &mh, 0);
  55 }
  56
  57 static int
  58 memif_msg_send_from_queue(struct memif_control_channel *cc)
  59 {
  60         ssize_t size;
  61         int ret = 0;
  62         struct memif_msg_queue_elt *e;
  63
  64         e = TAILQ_FIRST(&cc->msg_queue);
  65         if (e == NULL)
  66                 return 0;
  67
  68         size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
  69         if (size != sizeof(memif_msg_t)) {
  70                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
  71                 ret = -1;
  72         } else {
  73                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
  74         }
  75         TAILQ_REMOVE(&cc->msg_queue, e, next);
  76         rte_free(e);
  77
  78         return ret;
  79 }
  80
  81 static struct memif_msg_queue_elt *
  82 memif_msg_enq(struct memif_control_channel *cc)
  83 {
  84         struct memif_msg_queue_elt *e;
  85
  86         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
  87         if (e == NULL) {
  88                 MIF_LOG(ERR, "Failed to allocate control message.");
  89                 return NULL;
  90         }
  91
  92         e->fd = -1;
  93         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
  94
  95         return e;
  96 }
  97
  98 void
  99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
 100                          int err_code)
 101 {
 102         struct memif_msg_queue_elt *e;
 103         struct pmd_internals *pmd;
 104         memif_msg_disconnect_t *d;
 105
 106         if (cc == NULL) {
 107                 MIF_LOG(DEBUG, "Missing control channel.");
 108                 return;
 109         }
 110
 111         e = memif_msg_enq(cc);
 112         if (e == NULL) {
 113                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
 114                 return;
 115         }
 116
 117         d = &e->msg.disconnect;
 118
 119         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
 120         d->code = err_code;
 121
 122         if (reason != NULL) {
 123                 strlcpy((char *)d->string, reason, sizeof(d->string));
 124                 if (cc->dev != NULL) {
 125                         pmd = cc->dev->data->dev_private;
 126                         strlcpy(pmd->local_disc_string, reason,
 127                                 sizeof(pmd->local_disc_string));
 128                 }
 129         }
 130 }
 131
 132 static int
 133 memif_msg_enq_hello(struct memif_control_channel *cc)
 134 {
 135         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
 136         memif_msg_hello_t *h;
 137
 138         if (e == NULL)
 139                 return -1;
 140
 141         h = &e->msg.hello;
 142
 143         e->msg.type = MEMIF_MSG_TYPE_HELLO;
 144         h->min_version = MEMIF_VERSION;
 145         h->max_version = MEMIF_VERSION;
 146         h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 147         h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
 148         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
 149         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
 150
 151         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
 152
 153         return 0;
 154 }
 155
 156 static int
 157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
 158 {
 159         struct pmd_internals *pmd = dev->data->dev_private;
 160         memif_msg_hello_t *h = &msg->hello;
 161
 162         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
 163                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
 164                 return -1;
 165         }
 166
 167         /* Set parameters for active connection */
 168         pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1,
 169                                            pmd->cfg.num_s2m_rings);
 170         pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1,
 171                                            pmd->cfg.num_m2s_rings);
 172         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
 173                                             pmd->cfg.log2_ring_size);
 174         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
 175
 176         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
 177
 178         MIF_LOG(DEBUG, "Connecting to %s.", pmd->remote_name);
 179
 180         return 0;
 181 }
 182
 183 static int
 184 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
 185 {
 186         memif_msg_init_t *i = &msg->init;
 187         struct memif_socket_dev_list_elt *elt;
 188         struct pmd_internals *pmd;
 189         struct rte_eth_dev *dev;
 190
 191         if (i->version != MEMIF_VERSION) {
 192                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
 193                 return -1;
 194         }
 195
 196         if (cc->socket == NULL) {
 197                 memif_msg_enq_disconnect(cc, "Device error", 0);
 198                 return -1;
 199         }
 200
 201         /* Find device with requested ID */
 202         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
 203                 dev = elt->dev;
 204                 pmd = dev->data->dev_private;
 205                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
 206                     pmd->id == i->id) {
 207                         /* assign control channel to device */
 208                         cc->dev = dev;
 209                         pmd->cc = cc;
 210
 211                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
 212                                 memif_msg_enq_disconnect(pmd->cc,
 213                                                          "Only ethernet mode supported",
 214                                                          0);
 215                                 return -1;
 216                         }
 217
 218                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
 219                                            ETH_MEMIF_FLAG_CONNECTED)) {
 220                                 memif_msg_enq_disconnect(pmd->cc,
 221                                                          "Already connected", 0);
 222                                 return -1;
 223                         }
 224                         strlcpy(pmd->remote_name, (char *)i->name,
 225                                 sizeof(pmd->remote_name));
 226
 227                         if (*pmd->secret != '\0') {
 228                                 if (*i->secret == '\0') {
 229                                         memif_msg_enq_disconnect(pmd->cc,
 230                                                                  "Secret required", 0);
 231                                         return -1;
 232                                 }
 233                                 if (strncmp(pmd->secret, (char *)i->secret,
 234                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
 235                                         memif_msg_enq_disconnect(pmd->cc,
 236                                                                  "Incorrect secret", 0);
 237                                         return -1;
 238                                 }
 239                         }
 240
 241                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
 242                         return 0;
 243                 }
 244         }
 245
 246         /* ID not found on this socket */
 247         MIF_LOG(DEBUG, "ID %u not found.", i->id);
 248         memif_msg_enq_disconnect(cc, "ID not found", 0);
 249         return -1;
 250 }
 251
 252 static int
 253 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
 254                              int fd)
 255 {
 256         struct pmd_internals *pmd = dev->data->dev_private;
 257         struct pmd_process_private *proc_private = dev->process_private;
 258         memif_msg_add_region_t *ar = &msg->add_region;
 259         struct memif_region *r;
 260
 261         if (fd < 0) {
 262                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
 263                 return -1;
 264         }
 265
 266         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
 267                         ar->index != proc_private->regions_num ||
 268                         proc_private->regions[ar->index] != NULL) {
 269                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
 270                 return -1;
 271         }
 272
 273         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
 274         if (r == NULL) {
 275                 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
 276                 return -ENOMEM;
 277         }
 278
 279         r->fd = fd;
 280         r->region_size = ar->size;
 281         r->addr = NULL;
 282
 283         proc_private->regions[ar->index] = r;
 284         proc_private->regions_num++;
 285
 286         return 0;
 287 }
 288
 289 static int
 290 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
 291 {
 292         struct pmd_internals *pmd = dev->data->dev_private;
 293         memif_msg_add_ring_t *ar = &msg->add_ring;
 294         struct memif_queue *mq;
 295
 296         if (fd < 0) {
 297                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
 298                 return -1;
 299         }
 300
 301         /* check if we have enough queues */
 302         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) {
 303                 if (ar->index >= pmd->cfg.num_s2m_rings) {
 304                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 305                         return -1;
 306                 }
 307                 pmd->run.num_s2m_rings++;
 308         } else {
 309                 if (ar->index >= pmd->cfg.num_m2s_rings) {
 310                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
 311                         return -1;
 312                 }
 313                 pmd->run.num_m2s_rings++;
 314         }
 315
 316         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ?
 317             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
 318
 319         mq->intr_handle.fd = fd;
 320         mq->log2_ring_size = ar->log2_ring_size;
 321         mq->region = ar->region;
 322         mq->ring_offset = ar->offset;
 323
 324         return 0;
 325 }
 326
 327 static int
 328 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
 329 {
 330         struct pmd_internals *pmd = dev->data->dev_private;
 331         memif_msg_connect_t *c = &msg->connect;
 332         int ret;
 333
 334         ret = memif_connect(dev);
 335         if (ret < 0)
 336                 return ret;
 337
 338         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 339                 sizeof(pmd->remote_if_name));
 340         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 341
 342         return 0;
 343 }
 344
 345 static int
 346 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
 347 {
 348         struct pmd_internals *pmd = dev->data->dev_private;
 349         memif_msg_connected_t *c = &msg->connected;
 350         int ret;
 351
 352         ret = memif_connect(dev);
 353         if (ret < 0)
 354                 return ret;
 355
 356         strlcpy(pmd->remote_if_name, (char *)c->if_name,
 357                 sizeof(pmd->remote_if_name));
 358         MIF_LOG(INFO, "Remote interface %s connected.", pmd->remote_if_name);
 359
 360         return 0;
 361 }
 362
 363 static int
 364 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
 365 {
 366         struct pmd_internals *pmd = dev->data->dev_private;
 367         memif_msg_disconnect_t *d = &msg->disconnect;
 368
 369         memset(pmd->remote_disc_string, 0, sizeof(pmd->remote_disc_string));
 370         strlcpy(pmd->remote_disc_string, (char *)d->string,
 371                 sizeof(pmd->remote_disc_string));
 372
 373         MIF_LOG(INFO, "Disconnect received: %s", pmd->remote_disc_string);
 374
 375         memset(pmd->local_disc_string, 0, 96);
 376         memif_disconnect(dev);
 377         return 0;
 378 }
 379
 380 static int
 381 memif_msg_enq_ack(struct rte_eth_dev *dev)
 382 {
 383         struct pmd_internals *pmd = dev->data->dev_private;
 384         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 385         if (e == NULL)
 386                 return -1;
 387
 388         e->msg.type = MEMIF_MSG_TYPE_ACK;
 389
 390         return 0;
 391 }
 392
 393 static int
 394 memif_msg_enq_init(struct rte_eth_dev *dev)
 395 {
 396         struct pmd_internals *pmd = dev->data->dev_private;
 397         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 398         memif_msg_init_t *i = &e->msg.init;
 399
 400         if (e == NULL)
 401                 return -1;
 402
 403         i = &e->msg.init;
 404         e->msg.type = MEMIF_MSG_TYPE_INIT;
 405         i->version = MEMIF_VERSION;
 406         i->id = pmd->id;
 407         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
 408
 409         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
 410
 411         if (*pmd->secret != '\0')
 412                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
 413
 414         return 0;
 415 }
 416
 417 static int
 418 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
 419 {
 420         struct pmd_internals *pmd = dev->data->dev_private;
 421         struct pmd_process_private *proc_private = dev->process_private;
 422         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 423         memif_msg_add_region_t *ar;
 424         struct memif_region *mr = proc_private->regions[idx];
 425
 426         if (e == NULL)
 427                 return -1;
 428
 429         ar = &e->msg.add_region;
 430         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
 431         e->fd = mr->fd;
 432         ar->index = idx;
 433         ar->size = mr->region_size;
 434
 435         return 0;
 436 }
 437
 438 static int
 439 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
 440                        memif_ring_type_t type)
 441 {
 442         struct pmd_internals *pmd = dev->data->dev_private;
 443         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 444         struct memif_queue *mq;
 445         memif_msg_add_ring_t *ar;
 446
 447         if (e == NULL)
 448                 return -1;
 449
 450         ar = &e->msg.add_ring;
 451         mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] :
 452             dev->data->rx_queues[idx];
 453
 454         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
 455         e->fd = mq->intr_handle.fd;
 456         ar->index = idx;
 457         ar->offset = mq->ring_offset;
 458         ar->region = mq->region;
 459         ar->log2_ring_size = mq->log2_ring_size;
 460         ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
 461         ar->private_hdr_size = 0;
 462
 463         return 0;
 464 }
 465
 466 static int
 467 memif_msg_enq_connect(struct rte_eth_dev *dev)
 468 {
 469         struct pmd_internals *pmd = dev->data->dev_private;
 470         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 471         memif_msg_connect_t *c;
 472
 473         if (e == NULL)
 474                 return -1;
 475
 476         c = &e->msg.connect;
 477         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
 478         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 479
 480         return 0;
 481 }
 482
 483 static int
 484 memif_msg_enq_connected(struct rte_eth_dev *dev)
 485 {
 486         struct pmd_internals *pmd = dev->data->dev_private;
 487         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
 488         memif_msg_connected_t *c;
 489
 490         if (e == NULL)
 491                 return -1;
 492
 493         c = &e->msg.connected;
 494         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
 495         strlcpy((char *)c->if_name, dev->data->name, sizeof(c->if_name));
 496
 497         return 0;
 498 }
 499
 500 static void
 501 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
 502 {
 503         struct memif_msg_queue_elt *elt;
 504         struct memif_control_channel *cc = arg;
 505
 506         /* close control channel fd */
 507         close(intr_handle->fd);
 508         /* clear message queue */
 509         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
 510                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
 511                 rte_free(elt);
 512         }
 513         /* free control channel */
 514         rte_free(cc);
 515 }
 516
 517 void
 518 memif_disconnect(struct rte_eth_dev *dev)
 519 {
 520         struct pmd_internals *pmd = dev->data->dev_private;
 521         struct memif_msg_queue_elt *elt, *next;
 522         struct memif_queue *mq;
 523         struct rte_intr_handle *ih;
 524         int i;
 525         int ret;
 526
 527         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 528         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
 529         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
 530
 531         if (pmd->cc != NULL) {
 532                 /* Clear control message queue (except disconnect message if any). */
 533                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
 534                         next = TAILQ_NEXT(elt, next);
 535                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
 536                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
 537                                 rte_free(elt);
 538                         }
 539                 }
 540                 /* send disconnect message (if there is any in queue) */
 541                 memif_msg_send_from_queue(pmd->cc);
 542
 543                 /* at this point, there should be no more messages in queue */
 544                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
 545                         MIF_LOG(WARNING,
 546                                 "Unexpected message(s) in message queue.");
 547                 }
 548
 549                 ih = &pmd->cc->intr_handle;
 550                 if (ih->fd > 0) {
 551                         ret = rte_intr_callback_unregister(ih,
 552                                                         memif_intr_handler,
 553                                                         pmd->cc);
 554                         /*
 555                          * If callback is active (disconnecting based on
 556                          * received control message).
 557                          */
 558                         if (ret == -EAGAIN) {
 559                                 ret = rte_intr_callback_unregister_pending(ih,
 560                                                         memif_intr_handler,
 561                                                         pmd->cc,
 562                                                         memif_intr_unregister_handler);
 563                         } else if (ret > 0) {
 564                                 close(ih->fd);
 565                                 rte_free(pmd->cc);
 566                         }
 567                         pmd->cc = NULL;
 568                         if (ret <= 0)
 569                                 MIF_LOG(WARNING,
 570                                         "Failed to unregister control channel callback.");
 571                 }
 572         }
 573
 574         /* unconfig interrupts */
 575         for (i = 0; i < pmd->cfg.num_s2m_rings; i++) {
 576                 if (pmd->role == MEMIF_ROLE_SLAVE) {
 577                         if (dev->data->tx_queues != NULL)
 578                                 mq = dev->data->tx_queues[i];
 579                         else
 580                                 continue;
 581                 } else {
 582                         if (dev->data->rx_queues != NULL)
 583                                 mq = dev->data->rx_queues[i];
 584                         else
 585                                 continue;
 586                 }
 587                 if (mq->intr_handle.fd > 0) {
 588                         close(mq->intr_handle.fd);
 589                         mq->intr_handle.fd = -1;
 590                 }
 591         }
 592         for (i = 0; i < pmd->cfg.num_m2s_rings; i++) {
 593                 if (pmd->role == MEMIF_ROLE_MASTER) {
 594                         if (dev->data->tx_queues != NULL)
 595                                 mq = dev->data->tx_queues[i];
 596                         else
 597                                 continue;
 598                 } else {
 599                         if (dev->data->rx_queues != NULL)
 600                                 mq = dev->data->rx_queues[i];
 601                         else
 602                                 continue;
 603                 }
 604                 if (mq->intr_handle.fd > 0) {
 605                         close(mq->intr_handle.fd);
 606                         mq->intr_handle.fd = -1;
 607                 }
 608         }
 609
 610         memif_free_regions(dev);
 611
 612         /* reset connection configuration */
 613         memset(&pmd->run, 0, sizeof(pmd->run));
 614
 615         MIF_LOG(DEBUG, "Disconnected.");
 616 }
 617
 618 static int
 619 memif_msg_receive(struct memif_control_channel *cc)
 620 {
 621         char ctl[CMSG_SPACE(sizeof(int)) +
 622                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
 623         struct msghdr mh = { 0 };
 624         struct iovec iov[1];
 625         memif_msg_t msg = { 0 };
 626         ssize_t size;
 627         int ret = 0;
 628         struct ucred *cr __rte_unused;
 629         cr = 0;
 630         struct cmsghdr *cmsg;
 631         int afd = -1;
 632         int i;
 633         struct pmd_internals *pmd;
 634         struct pmd_process_private *proc_private;
 635
 636         iov[0].iov_base = (void *)&msg;
 637         iov[0].iov_len = sizeof(memif_msg_t);
 638         mh.msg_iov = iov;
 639         mh.msg_iovlen = 1;
 640         mh.msg_control = ctl;
 641         mh.msg_controllen = sizeof(ctl);
 642
 643         size = recvmsg(cc->intr_handle.fd, &mh, 0);
 644         if (size != sizeof(memif_msg_t)) {
 645                 MIF_LOG(DEBUG, "Invalid message size.");
 646                 memif_msg_enq_disconnect(cc, "Invalid message size", 0);
 647                 return -1;
 648         }
 649         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
 650
 651         cmsg = CMSG_FIRSTHDR(&mh);
 652         while (cmsg) {
 653                 if (cmsg->cmsg_level == SOL_SOCKET) {
 654                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
 655                                 cr = (struct ucred *)CMSG_DATA(cmsg);
 656                         else if (cmsg->cmsg_type == SCM_RIGHTS)
 657                                 rte_memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
 658                 }
 659                 cmsg = CMSG_NXTHDR(&mh, cmsg);
 660         }
 661
 662         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
 663                 MIF_LOG(DEBUG, "Unexpected message.");
 664                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
 665                 return -1;
 666         }
 667
 668         /* get device from hash data */
 669         switch (msg.type) {
 670         case MEMIF_MSG_TYPE_ACK:
 671                 break;
 672         case MEMIF_MSG_TYPE_HELLO:
 673                 ret = memif_msg_receive_hello(cc->dev, &msg);
 674                 if (ret < 0)
 675                         goto exit;
 676                 ret = memif_init_regions_and_queues(cc->dev);
 677                 if (ret < 0)
 678                         goto exit;
 679                 ret = memif_msg_enq_init(cc->dev);
 680                 if (ret < 0)
 681                         goto exit;
 682                 pmd = cc->dev->data->dev_private;
 683                 proc_private = cc->dev->process_private;
 684                 for (i = 0; i < proc_private->regions_num; i++) {
 685                         ret = memif_msg_enq_add_region(cc->dev, i);
 686                         if (ret < 0)
 687                                 goto exit;
 688                 }
 689                 for (i = 0; i < pmd->run.num_s2m_rings; i++) {
 690                         ret = memif_msg_enq_add_ring(cc->dev, i,
 691                                                      MEMIF_RING_S2M);
 692                         if (ret < 0)
 693                                 goto exit;
 694                 }
 695                 for (i = 0; i < pmd->run.num_m2s_rings; i++) {
 696                         ret = memif_msg_enq_add_ring(cc->dev, i,
 697                                                      MEMIF_RING_M2S);
 698                         if (ret < 0)
 699                                 goto exit;
 700                 }
 701                 ret = memif_msg_enq_connect(cc->dev);
 702                 if (ret < 0)
 703                         goto exit;
 704                 break;
 705         case MEMIF_MSG_TYPE_INIT:
 706                 /*
 707                  * This cc does not have an interface asociated with it.
 708                  * If suitable interface is found it will be assigned here.
 709                  */
 710                 ret = memif_msg_receive_init(cc, &msg);
 711                 if (ret < 0)
 712                         goto exit;
 713                 ret = memif_msg_enq_ack(cc->dev);
 714                 if (ret < 0)
 715                         goto exit;
 716                 break;
 717         case MEMIF_MSG_TYPE_ADD_REGION:
 718                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
 719                 if (ret < 0)
 720                         goto exit;
 721                 ret = memif_msg_enq_ack(cc->dev);
 722                 if (ret < 0)
 723                         goto exit;
 724                 break;
 725         case MEMIF_MSG_TYPE_ADD_RING:
 726                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
 727                 if (ret < 0)
 728                         goto exit;
 729                 ret = memif_msg_enq_ack(cc->dev);
 730                 if (ret < 0)
 731                         goto exit;
 732                 break;
 733         case MEMIF_MSG_TYPE_CONNECT:
 734                 ret = memif_msg_receive_connect(cc->dev, &msg);
 735                 if (ret < 0)
 736                         goto exit;
 737                 ret = memif_msg_enq_connected(cc->dev);
 738                 if (ret < 0)
 739                         goto exit;
 740                 break;
 741         case MEMIF_MSG_TYPE_CONNECTED:
 742                 ret = memif_msg_receive_connected(cc->dev, &msg);
 743                 break;
 744         case MEMIF_MSG_TYPE_DISCONNECT:
 745                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
 746                 if (ret < 0)
 747                         goto exit;
 748                 break;
 749         default:
 750                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
 751                 ret = -1;
 752                 goto exit;
 753         }
 754
 755  exit:
 756         return ret;
 757 }
 758
 759 static void
 760 memif_intr_handler(void *arg)
 761 {
 762         struct memif_control_channel *cc = arg;
 763         int ret;
 764
 765         ret = memif_msg_receive(cc);
 766         /* if driver failed to assign device */
 767         if (cc->dev == NULL) {
 768                 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
 769                                                            memif_intr_handler,
 770                                                            cc,
 771                                                            memif_intr_unregister_handler);
 772                 if (ret < 0)
 773                         MIF_LOG(WARNING,
 774                                 "Failed to unregister control channel callback.");
 775                 return;
 776         }
 777         /* if memif_msg_receive failed */
 778         if (ret < 0)
 779                 goto disconnect;
 780
 781         ret = memif_msg_send_from_queue(cc);
 782         if (ret < 0)
 783                 goto disconnect;
 784
 785         return;
 786
 787  disconnect:
 788         if (cc->dev == NULL) {
 789                 MIF_LOG(WARNING, "eth dev not allocated");
 790                 return;
 791         }
 792         memif_disconnect(cc->dev);
 793 }
 794
 795 static void
 796 memif_listener_handler(void *arg)
 797 {
 798         struct memif_socket *socket = arg;
 799         int sockfd;
 800         int addr_len;
 801         struct sockaddr_un client;
 802         struct memif_control_channel *cc;
 803         int ret;
 804
 805         addr_len = sizeof(client);
 806         sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
 807                         (socklen_t *)&addr_len);
 808         if (sockfd < 0) {
 809                 MIF_LOG(ERR,
 810                         "Failed to accept connection request on socket fd %d",
 811                         socket->intr_handle.fd);
 812                 return;
 813         }
 814
 815         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
 816
 817         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
 818         if (cc == NULL) {
 819                 MIF_LOG(ERR, "Failed to allocate control channel.");
 820                 goto error;
 821         }
 822
 823         cc->intr_handle.fd = sockfd;
 824         cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
 825         cc->socket = socket;
 826         cc->dev = NULL;
 827         TAILQ_INIT(&cc->msg_queue);
 828
 829         ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
 830         if (ret < 0) {
 831                 MIF_LOG(ERR, "Failed to register control channel callback.");
 832                 goto error;
 833         }
 834
 835         ret = memif_msg_enq_hello(cc);
 836         if (ret < 0) {
 837                 MIF_LOG(ERR, "Failed to enqueue hello message.");
 838                 goto error;
 839         }
 840         ret = memif_msg_send_from_queue(cc);
 841         if (ret < 0)
 842                 goto error;
 843
 844         return;
 845
 846  error:
 847         if (sockfd >= 0) {
 848                 close(sockfd);
 849                 sockfd = -1;
 850         }
 851         if (cc != NULL)
 852                 rte_free(cc);
 853 }
 854
 855 static struct memif_socket *
 856 memif_socket_create(char *key, uint8_t listener)
 857 {
 858         struct memif_socket *sock;
 859         struct sockaddr_un un;
 860         int sockfd;
 861         int ret;
 862         int on = 1;
 863
 864         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
 865         if (sock == NULL) {
 866                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
 867                 return NULL;
 868         }
 869
 870         sock->listener = listener;
 871         strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
 872         TAILQ_INIT(&sock->dev_queue);
 873
 874         if (listener != 0) {
 875                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
 876                 if (sockfd < 0)
 877                         goto error;
 878
 879                 un.sun_family = AF_UNIX;
 880                 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
 881
 882                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
 883                                  sizeof(on));
 884                 if (ret < 0)
 885                         goto error;
 886
 887                 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un));
 888                 if (ret < 0)
 889                         goto error;
 890
 891                 ret = listen(sockfd, 1);
 892                 if (ret < 0)
 893                         goto error;
 894
 895                 MIF_LOG(DEBUG, "Memif listener socket %s created.", sock->filename);
 896
 897                 sock->intr_handle.fd = sockfd;
 898                 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
 899                 ret = rte_intr_callback_register(&sock->intr_handle,
 900                                                  memif_listener_handler, sock);
 901                 if (ret < 0) {
 902                         MIF_LOG(ERR, "Failed to register interrupt "
 903                                 "callback for listener socket");
 904                         return NULL;
 905                 }
 906         }
 907
 908         return sock;
 909
 910  error:
 911         MIF_LOG(ERR, "Failed to setup socket %s: %s", key, strerror(errno));
 912         if (sock != NULL)
 913                 rte_free(sock);
 914         if (sockfd >= 0)
 915                 close(sockfd);
 916         return NULL;
 917 }
 918
 919 static struct rte_hash *
 920 memif_create_socket_hash(void)
 921 {
 922         struct rte_hash_parameters params = { 0 };
 923
 924         params.name = MEMIF_SOCKET_HASH_NAME;
 925         params.entries = 256;
 926         params.key_len = MEMIF_SOCKET_UN_SIZE;
 927         params.hash_func = rte_jhash;
 928         params.hash_func_init_val = 0;
 929         return rte_hash_create(&params);
 930 }
 931
 932 int
 933 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
 934 {
 935         struct pmd_internals *pmd = dev->data->dev_private;
 936         struct memif_socket *socket = NULL;
 937         struct memif_socket_dev_list_elt *elt;
 938         struct pmd_internals *tmp_pmd;
 939         struct rte_hash *hash;
 940         int ret;
 941         char key[MEMIF_SOCKET_UN_SIZE];
 942
 943         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
 944         if (hash == NULL) {
 945                 hash = memif_create_socket_hash();
 946                 if (hash == NULL) {
 947                         MIF_LOG(ERR, "Failed to create memif socket hash.");
 948                         return -1;
 949                 }
 950         }
 951
 952         memset(key, 0, MEMIF_SOCKET_UN_SIZE);
 953         strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
 954         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
 955         if (ret < 0) {
 956                 socket = memif_socket_create(key,
 957                                              (pmd->role == MEMIF_ROLE_SLAVE) ? 0 : 1);
 958                 if (socket == NULL)
 959                         return -1;
 960                 ret = rte_hash_add_key_data(hash, key, socket);
 961                 if (ret < 0) {
 962                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
 963                         return ret;
 964                 }
 965         }
 966         pmd->socket_filename = socket->filename;
 967
 968         if (socket->listener != 0 && pmd->role == MEMIF_ROLE_SLAVE) {
 969                 MIF_LOG(ERR, "Socket is a listener.");
 970                 return -1;
 971         } else if ((socket->listener == 0) && (pmd->role == MEMIF_ROLE_MASTER)) {
 972                 MIF_LOG(ERR, "Socket is not a listener.");
 973                 return -1;
 974         }
 975
 976         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
 977                 tmp_pmd = elt->dev->data->dev_private;
 978                 if (tmp_pmd->id == pmd->id) {
 979                         MIF_LOG(ERR, "Memif device with id %d already "
 980                                 "exists on socket %s",
 981                                 pmd->id, socket->filename);
 982                         return -1;
 983                 }
 984         }
 985
 986         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
 987         if (elt == NULL) {
 988                 MIF_LOG(ERR, "Failed to add device to socket device list.");
 989                 return -1;
 990         }
 991         elt->dev = dev;
 992         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
 993
 994         return 0;
 995 }
 996
 997 void
 998 memif_socket_remove_device(struct rte_eth_dev *dev)
 999 {
1000         struct pmd_internals *pmd = dev->data->dev_private;
1001         struct memif_socket *socket = NULL;
1002         struct memif_socket_dev_list_elt *elt, *next;
1003         struct rte_hash *hash;
1004         int ret;
1005
1006         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1007         if (hash == NULL)
1008                 return;
1009
1010         if (pmd->socket_filename == NULL)
1011                 return;
1012
1013         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1014                 return;
1015
1016         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1017                 next = TAILQ_NEXT(elt, next);
1018                 if (elt->dev == dev) {
1019                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1020                         rte_free(elt);
1021                         pmd->socket_filename = NULL;
1022                 }
1023         }
1024
1025         /* remove socket, if this was the last device using it */
1026         if (TAILQ_EMPTY(&socket->dev_queue)) {
1027                 rte_hash_del_key(hash, socket->filename);
1028                 if (socket->listener) {
1029                         /* remove listener socket file,
1030                          * so we can create new one later.
1031                          */
1032                         ret = remove(socket->filename);
1033                         if (ret < 0)
1034                                 MIF_LOG(ERR, "Failed to remove socket file: %s",
1035                                         socket->filename);
1036                 }
1037                 rte_free(socket);
1038         }
1039 }
1040
1041 int
1042 memif_connect_master(struct rte_eth_dev *dev)
1043 {
1044         struct pmd_internals *pmd = dev->data->dev_private;
1045
1046         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1047         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1048         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1049         return 0;
1050 }
1051
1052 int
1053 memif_connect_slave(struct rte_eth_dev *dev)
1054 {
1055         int sockfd;
1056         int ret;
1057         struct sockaddr_un sun;
1058         struct pmd_internals *pmd = dev->data->dev_private;
1059
1060         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1061         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1062         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1063
1064         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1065         if (sockfd < 0) {
1066                 MIF_LOG(ERR, "Failed to open socket.");
1067                 return -1;
1068         }
1069
1070         sun.sun_family = AF_UNIX;
1071
1072         memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1);
1073
1074         ret = connect(sockfd, (struct sockaddr *)&sun,
1075                       sizeof(struct sockaddr_un));
1076         if (ret < 0) {
1077                 MIF_LOG(ERR, "Failed to connect socket: %s.", pmd->socket_filename);
1078                 goto error;
1079         }
1080
1081         MIF_LOG(DEBUG, "Memif socket: %s connected.", pmd->socket_filename);
1082
1083         pmd->cc = rte_zmalloc("memif-cc",
1084                               sizeof(struct memif_control_channel), 0);
1085         if (pmd->cc == NULL) {
1086                 MIF_LOG(ERR, "Failed to allocate control channel.");
1087                 goto error;
1088         }
1089
1090         pmd->cc->intr_handle.fd = sockfd;
1091         pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1092         pmd->cc->socket = NULL;
1093         pmd->cc->dev = dev;
1094         TAILQ_INIT(&pmd->cc->msg_queue);
1095
1096         ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1097                                          memif_intr_handler, pmd->cc);
1098         if (ret < 0) {
1099                 MIF_LOG(ERR, "Failed to register interrupt callback for control fd");
1100                 goto error;
1101         }
1102
1103         return 0;
1104
1105  error:
1106         if (sockfd >= 0) {
1107                 close(sockfd);
1108                 sockfd = -1;
1109         }
1110         if (pmd->cc != NULL) {
1111                 rte_free(pmd->cc);
1112                 pmd->cc = NULL;
1113         }
1114         return -1;
1115 }