net/mlx5: add Altivec Rx
[dpdk.git] / drivers / net / memif / memif_socket.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2018-2019 Cisco Systems, Inc.  All rights reserved.
3  */
4
5 #include <stdlib.h>
6 #include <fcntl.h>
7 #include <unistd.h>
8 #include <sys/types.h>
9 #include <sys/socket.h>
10 #include <sys/ioctl.h>
11 #include <errno.h>
12
13 #include <rte_version.h>
14 #include <rte_mbuf.h>
15 #include <rte_ether.h>
16 #include <rte_ethdev_driver.h>
17 #include <rte_ethdev_vdev.h>
18 #include <rte_malloc.h>
19 #include <rte_kvargs.h>
20 #include <rte_bus_vdev.h>
21 #include <rte_hash.h>
22 #include <rte_jhash.h>
23 #include <rte_string_fns.h>
24
25 #include "rte_eth_memif.h"
26 #include "memif_socket.h"
27
28 static void memif_intr_handler(void *arg);
29
30 static ssize_t
31 memif_msg_send(int fd, memif_msg_t *msg, int afd)
32 {
33         struct msghdr mh = { 0 };
34         struct iovec iov[1];
35         struct cmsghdr *cmsg;
36         char ctl[CMSG_SPACE(sizeof(int))];
37
38         iov[0].iov_base = msg;
39         iov[0].iov_len = sizeof(memif_msg_t);
40         mh.msg_iov = iov;
41         mh.msg_iovlen = 1;
42
43         if (afd > 0) {
44                 memset(&ctl, 0, sizeof(ctl));
45                 mh.msg_control = ctl;
46                 mh.msg_controllen = sizeof(ctl);
47                 cmsg = CMSG_FIRSTHDR(&mh);
48                 cmsg->cmsg_len = CMSG_LEN(sizeof(int));
49                 cmsg->cmsg_level = SOL_SOCKET;
50                 cmsg->cmsg_type = SCM_RIGHTS;
51                 rte_memcpy(CMSG_DATA(cmsg), &afd, sizeof(int));
52         }
53
54         return sendmsg(fd, &mh, 0);
55 }
56
57 static int
58 memif_msg_send_from_queue(struct memif_control_channel *cc)
59 {
60         ssize_t size;
61         int ret = 0;
62         struct memif_msg_queue_elt *e;
63
64         e = TAILQ_FIRST(&cc->msg_queue);
65         if (e == NULL)
66                 return 0;
67
68         size = memif_msg_send(cc->intr_handle.fd, &e->msg, e->fd);
69         if (size != sizeof(memif_msg_t)) {
70                 MIF_LOG(ERR, "sendmsg fail: %s.", strerror(errno));
71                 ret = -1;
72         } else {
73                 MIF_LOG(DEBUG, "Sent msg type %u.", e->msg.type);
74         }
75         TAILQ_REMOVE(&cc->msg_queue, e, next);
76         rte_free(e);
77
78         return ret;
79 }
80
81 static struct memif_msg_queue_elt *
82 memif_msg_enq(struct memif_control_channel *cc)
83 {
84         struct memif_msg_queue_elt *e;
85
86         e = rte_zmalloc("memif_msg", sizeof(struct memif_msg_queue_elt), 0);
87         if (e == NULL) {
88                 MIF_LOG(ERR, "Failed to allocate control message.");
89                 return NULL;
90         }
91
92         e->fd = -1;
93         TAILQ_INSERT_TAIL(&cc->msg_queue, e, next);
94
95         return e;
96 }
97
98 void
99 memif_msg_enq_disconnect(struct memif_control_channel *cc, const char *reason,
100                          int err_code)
101 {
102         struct memif_msg_queue_elt *e;
103         struct pmd_internals *pmd;
104         memif_msg_disconnect_t *d;
105
106         if (cc == NULL) {
107                 MIF_LOG(DEBUG, "Missing control channel.");
108                 return;
109         }
110
111         e = memif_msg_enq(cc);
112         if (e == NULL) {
113                 MIF_LOG(WARNING, "Failed to enqueue disconnect message.");
114                 return;
115         }
116
117         d = &e->msg.disconnect;
118
119         e->msg.type = MEMIF_MSG_TYPE_DISCONNECT;
120         d->code = err_code;
121
122         if (reason != NULL) {
123                 strlcpy((char *)d->string, reason, sizeof(d->string));
124                 if (cc->dev != NULL) {
125                         pmd = cc->dev->data->dev_private;
126                         strlcpy(pmd->local_disc_string, reason,
127                                 sizeof(pmd->local_disc_string));
128                 }
129         }
130 }
131
132 static int
133 memif_msg_enq_hello(struct memif_control_channel *cc)
134 {
135         struct memif_msg_queue_elt *e = memif_msg_enq(cc);
136         memif_msg_hello_t *h;
137
138         if (e == NULL)
139                 return -1;
140
141         h = &e->msg.hello;
142
143         e->msg.type = MEMIF_MSG_TYPE_HELLO;
144         h->min_version = MEMIF_VERSION;
145         h->max_version = MEMIF_VERSION;
146         h->max_s2m_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
147         h->max_m2s_ring = ETH_MEMIF_MAX_NUM_Q_PAIRS;
148         h->max_region = ETH_MEMIF_MAX_REGION_NUM - 1;
149         h->max_log2_ring_size = ETH_MEMIF_MAX_LOG2_RING_SIZE;
150
151         strlcpy((char *)h->name, rte_version(), sizeof(h->name));
152
153         return 0;
154 }
155
156 static int
157 memif_msg_receive_hello(struct rte_eth_dev *dev, memif_msg_t *msg)
158 {
159         struct pmd_internals *pmd = dev->data->dev_private;
160         memif_msg_hello_t *h = &msg->hello;
161
162         if (h->min_version > MEMIF_VERSION || h->max_version < MEMIF_VERSION) {
163                 memif_msg_enq_disconnect(pmd->cc, "Incompatible memif version", 0);
164                 return -1;
165         }
166
167         /* Set parameters for active connection */
168         pmd->run.num_s2m_rings = RTE_MIN(h->max_s2m_ring + 1,
169                                            pmd->cfg.num_s2m_rings);
170         pmd->run.num_m2s_rings = RTE_MIN(h->max_m2s_ring + 1,
171                                            pmd->cfg.num_m2s_rings);
172         pmd->run.log2_ring_size = RTE_MIN(h->max_log2_ring_size,
173                                             pmd->cfg.log2_ring_size);
174         pmd->run.pkt_buffer_size = pmd->cfg.pkt_buffer_size;
175
176         strlcpy(pmd->remote_name, (char *)h->name, sizeof(pmd->remote_name));
177
178         MIF_LOG(DEBUG, "%s: Connecting to %s.",
179                 rte_vdev_device_name(pmd->vdev), pmd->remote_name);
180
181         return 0;
182 }
183
184 static int
185 memif_msg_receive_init(struct memif_control_channel *cc, memif_msg_t *msg)
186 {
187         memif_msg_init_t *i = &msg->init;
188         struct memif_socket_dev_list_elt *elt;
189         struct pmd_internals *pmd;
190         struct rte_eth_dev *dev;
191
192         if (i->version != MEMIF_VERSION) {
193                 memif_msg_enq_disconnect(cc, "Incompatible memif version", 0);
194                 return -1;
195         }
196
197         if (cc->socket == NULL) {
198                 memif_msg_enq_disconnect(cc, "Device error", 0);
199                 return -1;
200         }
201
202         /* Find device with requested ID */
203         TAILQ_FOREACH(elt, &cc->socket->dev_queue, next) {
204                 dev = elt->dev;
205                 pmd = dev->data->dev_private;
206                 if (((pmd->flags & ETH_MEMIF_FLAG_DISABLED) == 0) &&
207                     pmd->id == i->id) {
208                         /* assign control channel to device */
209                         cc->dev = dev;
210                         pmd->cc = cc;
211
212                         if (i->mode != MEMIF_INTERFACE_MODE_ETHERNET) {
213                                 memif_msg_enq_disconnect(pmd->cc,
214                                                          "Only ethernet mode supported",
215                                                          0);
216                                 return -1;
217                         }
218
219                         if (pmd->flags & (ETH_MEMIF_FLAG_CONNECTING |
220                                            ETH_MEMIF_FLAG_CONNECTED)) {
221                                 memif_msg_enq_disconnect(pmd->cc,
222                                                          "Already connected", 0);
223                                 return -1;
224                         }
225                         strlcpy(pmd->remote_name, (char *)i->name,
226                                 sizeof(pmd->remote_name));
227
228                         if (*pmd->secret != '\0') {
229                                 if (*i->secret == '\0') {
230                                         memif_msg_enq_disconnect(pmd->cc,
231                                                                  "Secret required", 0);
232                                         return -1;
233                                 }
234                                 if (strncmp(pmd->secret, (char *)i->secret,
235                                                 ETH_MEMIF_SECRET_SIZE) != 0) {
236                                         memif_msg_enq_disconnect(pmd->cc,
237                                                                  "Incorrect secret", 0);
238                                         return -1;
239                                 }
240                         }
241
242                         pmd->flags |= ETH_MEMIF_FLAG_CONNECTING;
243                         return 0;
244                 }
245         }
246
247         /* ID not found on this socket */
248         MIF_LOG(DEBUG, "ID %u not found.", i->id);
249         memif_msg_enq_disconnect(cc, "ID not found", 0);
250         return -1;
251 }
252
253 static int
254 memif_msg_receive_add_region(struct rte_eth_dev *dev, memif_msg_t *msg,
255                              int fd)
256 {
257         struct pmd_internals *pmd = dev->data->dev_private;
258         struct pmd_process_private *proc_private = dev->process_private;
259         memif_msg_add_region_t *ar = &msg->add_region;
260         struct memif_region *r;
261
262         if (fd < 0) {
263                 memif_msg_enq_disconnect(pmd->cc, "Missing region fd", 0);
264                 return -1;
265         }
266
267         if (ar->index >= ETH_MEMIF_MAX_REGION_NUM ||
268                         ar->index != proc_private->regions_num ||
269                         proc_private->regions[ar->index] != NULL) {
270                 memif_msg_enq_disconnect(pmd->cc, "Invalid region index", 0);
271                 return -1;
272         }
273
274         r = rte_zmalloc("region", sizeof(struct memif_region), 0);
275         if (r == NULL) {
276                 memif_msg_enq_disconnect(pmd->cc, "Failed to alloc memif region.", 0);
277                 return -ENOMEM;
278         }
279
280         r->fd = fd;
281         r->region_size = ar->size;
282         r->addr = NULL;
283
284         proc_private->regions[ar->index] = r;
285         proc_private->regions_num++;
286
287         return 0;
288 }
289
290 static int
291 memif_msg_receive_add_ring(struct rte_eth_dev *dev, memif_msg_t *msg, int fd)
292 {
293         struct pmd_internals *pmd = dev->data->dev_private;
294         memif_msg_add_ring_t *ar = &msg->add_ring;
295         struct memif_queue *mq;
296
297         if (fd < 0) {
298                 memif_msg_enq_disconnect(pmd->cc, "Missing interrupt fd", 0);
299                 return -1;
300         }
301
302         /* check if we have enough queues */
303         if (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) {
304                 if (ar->index >= pmd->cfg.num_s2m_rings) {
305                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
306                         return -1;
307                 }
308                 pmd->run.num_s2m_rings++;
309         } else {
310                 if (ar->index >= pmd->cfg.num_m2s_rings) {
311                         memif_msg_enq_disconnect(pmd->cc, "Invalid ring index", 0);
312                         return -1;
313                 }
314                 pmd->run.num_m2s_rings++;
315         }
316
317         mq = (ar->flags & MEMIF_MSG_ADD_RING_FLAG_S2M) ?
318             dev->data->rx_queues[ar->index] : dev->data->tx_queues[ar->index];
319
320         mq->intr_handle.fd = fd;
321         mq->log2_ring_size = ar->log2_ring_size;
322         mq->region = ar->region;
323         mq->ring_offset = ar->offset;
324
325         return 0;
326 }
327
328 static int
329 memif_msg_receive_connect(struct rte_eth_dev *dev, memif_msg_t *msg)
330 {
331         struct pmd_internals *pmd = dev->data->dev_private;
332         memif_msg_connect_t *c = &msg->connect;
333         int ret;
334
335         ret = memif_connect(dev);
336         if (ret < 0)
337                 return ret;
338
339         strlcpy(pmd->remote_if_name, (char *)c->if_name,
340                 sizeof(pmd->remote_if_name));
341         MIF_LOG(INFO, "%s: Remote interface %s connected.",
342                 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name);
343
344         return 0;
345 }
346
347 static int
348 memif_msg_receive_connected(struct rte_eth_dev *dev, memif_msg_t *msg)
349 {
350         struct pmd_internals *pmd = dev->data->dev_private;
351         memif_msg_connected_t *c = &msg->connected;
352         int ret;
353
354         ret = memif_connect(dev);
355         if (ret < 0)
356                 return ret;
357
358         strlcpy(pmd->remote_if_name, (char *)c->if_name,
359                 sizeof(pmd->remote_if_name));
360         MIF_LOG(INFO, "%s: Remote interface %s connected.",
361                 rte_vdev_device_name(pmd->vdev), pmd->remote_if_name);
362
363         return 0;
364 }
365
366 static int
367 memif_msg_receive_disconnect(struct rte_eth_dev *dev, memif_msg_t *msg)
368 {
369         struct pmd_internals *pmd = dev->data->dev_private;
370         memif_msg_disconnect_t *d = &msg->disconnect;
371
372         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
373         strlcpy(pmd->remote_disc_string, (char *)d->string,
374                 sizeof(pmd->remote_disc_string));
375
376         MIF_LOG(INFO, "%s: Disconnect received: %s",
377                 rte_vdev_device_name(pmd->vdev), pmd->remote_disc_string);
378
379         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
380         memif_disconnect(dev);
381         return 0;
382 }
383
384 static int
385 memif_msg_enq_ack(struct rte_eth_dev *dev)
386 {
387         struct pmd_internals *pmd = dev->data->dev_private;
388         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
389         if (e == NULL)
390                 return -1;
391
392         e->msg.type = MEMIF_MSG_TYPE_ACK;
393
394         return 0;
395 }
396
397 static int
398 memif_msg_enq_init(struct rte_eth_dev *dev)
399 {
400         struct pmd_internals *pmd = dev->data->dev_private;
401         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
402         memif_msg_init_t *i = &e->msg.init;
403
404         if (e == NULL)
405                 return -1;
406
407         i = &e->msg.init;
408         e->msg.type = MEMIF_MSG_TYPE_INIT;
409         i->version = MEMIF_VERSION;
410         i->id = pmd->id;
411         i->mode = MEMIF_INTERFACE_MODE_ETHERNET;
412
413         strlcpy((char *)i->name, rte_version(), sizeof(i->name));
414
415         if (*pmd->secret != '\0')
416                 strlcpy((char *)i->secret, pmd->secret, sizeof(i->secret));
417
418         return 0;
419 }
420
421 static int
422 memif_msg_enq_add_region(struct rte_eth_dev *dev, uint8_t idx)
423 {
424         struct pmd_internals *pmd = dev->data->dev_private;
425         struct pmd_process_private *proc_private = dev->process_private;
426         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
427         memif_msg_add_region_t *ar;
428         struct memif_region *mr = proc_private->regions[idx];
429
430         if (e == NULL)
431                 return -1;
432
433         ar = &e->msg.add_region;
434         e->msg.type = MEMIF_MSG_TYPE_ADD_REGION;
435         e->fd = mr->fd;
436         ar->index = idx;
437         ar->size = mr->region_size;
438
439         return 0;
440 }
441
442 static int
443 memif_msg_enq_add_ring(struct rte_eth_dev *dev, uint8_t idx,
444                        memif_ring_type_t type)
445 {
446         struct pmd_internals *pmd = dev->data->dev_private;
447         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
448         struct memif_queue *mq;
449         memif_msg_add_ring_t *ar;
450
451         if (e == NULL)
452                 return -1;
453
454         ar = &e->msg.add_ring;
455         mq = (type == MEMIF_RING_S2M) ? dev->data->tx_queues[idx] :
456             dev->data->rx_queues[idx];
457
458         e->msg.type = MEMIF_MSG_TYPE_ADD_RING;
459         e->fd = mq->intr_handle.fd;
460         ar->index = idx;
461         ar->offset = mq->ring_offset;
462         ar->region = mq->region;
463         ar->log2_ring_size = mq->log2_ring_size;
464         ar->flags = (type == MEMIF_RING_S2M) ? MEMIF_MSG_ADD_RING_FLAG_S2M : 0;
465         ar->private_hdr_size = 0;
466
467         return 0;
468 }
469
470 static int
471 memif_msg_enq_connect(struct rte_eth_dev *dev)
472 {
473         struct pmd_internals *pmd = dev->data->dev_private;
474         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
475         const char *name = rte_vdev_device_name(pmd->vdev);
476         memif_msg_connect_t *c;
477
478         if (e == NULL)
479                 return -1;
480
481         c = &e->msg.connect;
482         e->msg.type = MEMIF_MSG_TYPE_CONNECT;
483         strlcpy((char *)c->if_name, name, sizeof(c->if_name));
484
485         return 0;
486 }
487
488 static int
489 memif_msg_enq_connected(struct rte_eth_dev *dev)
490 {
491         struct pmd_internals *pmd = dev->data->dev_private;
492         struct memif_msg_queue_elt *e = memif_msg_enq(pmd->cc);
493         const char *name = rte_vdev_device_name(pmd->vdev);
494         memif_msg_connected_t *c;
495
496         if (e == NULL)
497                 return -1;
498
499         c = &e->msg.connected;
500         e->msg.type = MEMIF_MSG_TYPE_CONNECTED;
501         strlcpy((char *)c->if_name, name, sizeof(c->if_name));
502
503         return 0;
504 }
505
506 static void
507 memif_intr_unregister_handler(struct rte_intr_handle *intr_handle, void *arg)
508 {
509         struct memif_msg_queue_elt *elt;
510         struct memif_control_channel *cc = arg;
511
512         /* close control channel fd */
513         close(intr_handle->fd);
514         /* clear message queue */
515         while ((elt = TAILQ_FIRST(&cc->msg_queue)) != NULL) {
516                 TAILQ_REMOVE(&cc->msg_queue, elt, next);
517                 rte_free(elt);
518         }
519         /* free control channel */
520         rte_free(cc);
521 }
522
523 void
524 memif_disconnect(struct rte_eth_dev *dev)
525 {
526         struct pmd_internals *pmd = dev->data->dev_private;
527         struct pmd_process_private *proc_private = dev->process_private;
528         struct memif_msg_queue_elt *elt, *next;
529         struct memif_queue *mq;
530         struct rte_intr_handle *ih;
531         int i;
532         int ret;
533
534         dev->data->dev_link.link_status = ETH_LINK_DOWN;
535         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTING;
536         pmd->flags &= ~ETH_MEMIF_FLAG_CONNECTED;
537
538         if (pmd->cc != NULL) {
539                 /* Clear control message queue (except disconnect message if any). */
540                 for (elt = TAILQ_FIRST(&pmd->cc->msg_queue); elt != NULL; elt = next) {
541                         next = TAILQ_NEXT(elt, next);
542                         if (elt->msg.type != MEMIF_MSG_TYPE_DISCONNECT) {
543                                 TAILQ_REMOVE(&pmd->cc->msg_queue, elt, next);
544                                 rte_free(elt);
545                         }
546                 }
547                 /* send disconnect message (if there is any in queue) */
548                 memif_msg_send_from_queue(pmd->cc);
549
550                 /* at this point, there should be no more messages in queue */
551                 if (TAILQ_FIRST(&pmd->cc->msg_queue) != NULL) {
552                         MIF_LOG(WARNING,
553                                 "Unexpected message(s) in message queue.");
554                 }
555
556                 ih = &pmd->cc->intr_handle;
557                 if (ih->fd > 0) {
558                         ret = rte_intr_callback_unregister(ih,
559                                                         memif_intr_handler,
560                                                         pmd->cc);
561                         /*
562                          * If callback is active (disconnecting based on
563                          * received control message).
564                          */
565                         if (ret == -EAGAIN) {
566                                 ret = rte_intr_callback_unregister_pending(ih,
567                                                         memif_intr_handler,
568                                                         pmd->cc,
569                                                         memif_intr_unregister_handler);
570                         } else if (ret > 0) {
571                                 close(ih->fd);
572                                 rte_free(pmd->cc);
573                         }
574                         pmd->cc = NULL;
575                         if (ret <= 0)
576                                 MIF_LOG(WARNING,
577                                         "Failed to unregister control channel callback.");
578                 }
579         }
580
581         /* unconfig interrupts */
582         for (i = 0; i < pmd->cfg.num_s2m_rings; i++) {
583                 if (pmd->role == MEMIF_ROLE_SLAVE) {
584                         if (dev->data->tx_queues != NULL)
585                                 mq = dev->data->tx_queues[i];
586                         else
587                                 continue;
588                 } else {
589                         if (dev->data->rx_queues != NULL)
590                                 mq = dev->data->rx_queues[i];
591                         else
592                                 continue;
593                 }
594                 if (mq->intr_handle.fd > 0) {
595                         close(mq->intr_handle.fd);
596                         mq->intr_handle.fd = -1;
597                 }
598         }
599         for (i = 0; i < pmd->cfg.num_m2s_rings; i++) {
600                 if (pmd->role == MEMIF_ROLE_MASTER) {
601                         if (dev->data->tx_queues != NULL)
602                                 mq = dev->data->tx_queues[i];
603                         else
604                                 continue;
605                 } else {
606                         if (dev->data->rx_queues != NULL)
607                                 mq = dev->data->rx_queues[i];
608                         else
609                                 continue;
610                 }
611                 if (mq->intr_handle.fd > 0) {
612                         close(mq->intr_handle.fd);
613                         mq->intr_handle.fd = -1;
614                 }
615         }
616
617         memif_free_regions(proc_private);
618
619         /* reset connection configuration */
620         memset(&pmd->run, 0, sizeof(pmd->run));
621
622         MIF_LOG(DEBUG, "Disconnected.");
623 }
624
625 static int
626 memif_msg_receive(struct memif_control_channel *cc)
627 {
628         char ctl[CMSG_SPACE(sizeof(int)) +
629                  CMSG_SPACE(sizeof(struct ucred))] = { 0 };
630         struct msghdr mh = { 0 };
631         struct iovec iov[1];
632         memif_msg_t msg = { 0 };
633         ssize_t size;
634         int ret = 0;
635         struct ucred *cr __rte_unused;
636         cr = 0;
637         struct cmsghdr *cmsg;
638         int afd = -1;
639         int i;
640         struct pmd_internals *pmd;
641         struct pmd_process_private *proc_private;
642
643         iov[0].iov_base = (void *)&msg;
644         iov[0].iov_len = sizeof(memif_msg_t);
645         mh.msg_iov = iov;
646         mh.msg_iovlen = 1;
647         mh.msg_control = ctl;
648         mh.msg_controllen = sizeof(ctl);
649
650         size = recvmsg(cc->intr_handle.fd, &mh, 0);
651         if (size != sizeof(memif_msg_t)) {
652                 MIF_LOG(DEBUG, "Invalid message size.");
653                 memif_msg_enq_disconnect(cc, "Invalid message size", 0);
654                 return -1;
655         }
656         MIF_LOG(DEBUG, "Received msg type: %u.", msg.type);
657
658         cmsg = CMSG_FIRSTHDR(&mh);
659         while (cmsg) {
660                 if (cmsg->cmsg_level == SOL_SOCKET) {
661                         if (cmsg->cmsg_type == SCM_CREDENTIALS)
662                                 cr = (struct ucred *)CMSG_DATA(cmsg);
663                         else if (cmsg->cmsg_type == SCM_RIGHTS)
664                                 memcpy(&afd, CMSG_DATA(cmsg), sizeof(int));
665                 }
666                 cmsg = CMSG_NXTHDR(&mh, cmsg);
667         }
668
669         if (cc->dev == NULL && msg.type != MEMIF_MSG_TYPE_INIT) {
670                 MIF_LOG(DEBUG, "Unexpected message.");
671                 memif_msg_enq_disconnect(cc, "Unexpected message", 0);
672                 return -1;
673         }
674
675         /* get device from hash data */
676         switch (msg.type) {
677         case MEMIF_MSG_TYPE_ACK:
678                 break;
679         case MEMIF_MSG_TYPE_HELLO:
680                 ret = memif_msg_receive_hello(cc->dev, &msg);
681                 if (ret < 0)
682                         goto exit;
683                 ret = memif_init_regions_and_queues(cc->dev);
684                 if (ret < 0)
685                         goto exit;
686                 ret = memif_msg_enq_init(cc->dev);
687                 if (ret < 0)
688                         goto exit;
689                 pmd = cc->dev->data->dev_private;
690                 proc_private = cc->dev->process_private;
691                 for (i = 0; i < proc_private->regions_num; i++) {
692                         ret = memif_msg_enq_add_region(cc->dev, i);
693                         if (ret < 0)
694                                 goto exit;
695                 }
696                 for (i = 0; i < pmd->run.num_s2m_rings; i++) {
697                         ret = memif_msg_enq_add_ring(cc->dev, i,
698                                                      MEMIF_RING_S2M);
699                         if (ret < 0)
700                                 goto exit;
701                 }
702                 for (i = 0; i < pmd->run.num_m2s_rings; i++) {
703                         ret = memif_msg_enq_add_ring(cc->dev, i,
704                                                      MEMIF_RING_M2S);
705                         if (ret < 0)
706                                 goto exit;
707                 }
708                 ret = memif_msg_enq_connect(cc->dev);
709                 if (ret < 0)
710                         goto exit;
711                 break;
712         case MEMIF_MSG_TYPE_INIT:
713                 /*
714                  * This cc does not have an interface asociated with it.
715                  * If suitable interface is found it will be assigned here.
716                  */
717                 ret = memif_msg_receive_init(cc, &msg);
718                 if (ret < 0)
719                         goto exit;
720                 ret = memif_msg_enq_ack(cc->dev);
721                 if (ret < 0)
722                         goto exit;
723                 break;
724         case MEMIF_MSG_TYPE_ADD_REGION:
725                 ret = memif_msg_receive_add_region(cc->dev, &msg, afd);
726                 if (ret < 0)
727                         goto exit;
728                 ret = memif_msg_enq_ack(cc->dev);
729                 if (ret < 0)
730                         goto exit;
731                 break;
732         case MEMIF_MSG_TYPE_ADD_RING:
733                 ret = memif_msg_receive_add_ring(cc->dev, &msg, afd);
734                 if (ret < 0)
735                         goto exit;
736                 ret = memif_msg_enq_ack(cc->dev);
737                 if (ret < 0)
738                         goto exit;
739                 break;
740         case MEMIF_MSG_TYPE_CONNECT:
741                 ret = memif_msg_receive_connect(cc->dev, &msg);
742                 if (ret < 0)
743                         goto exit;
744                 ret = memif_msg_enq_connected(cc->dev);
745                 if (ret < 0)
746                         goto exit;
747                 break;
748         case MEMIF_MSG_TYPE_CONNECTED:
749                 ret = memif_msg_receive_connected(cc->dev, &msg);
750                 break;
751         case MEMIF_MSG_TYPE_DISCONNECT:
752                 ret = memif_msg_receive_disconnect(cc->dev, &msg);
753                 if (ret < 0)
754                         goto exit;
755                 break;
756         default:
757                 memif_msg_enq_disconnect(cc, "Unknown message type", 0);
758                 ret = -1;
759                 goto exit;
760         }
761
762  exit:
763         return ret;
764 }
765
766 static void
767 memif_intr_handler(void *arg)
768 {
769         struct memif_control_channel *cc = arg;
770         int ret;
771
772         ret = memif_msg_receive(cc);
773         /* if driver failed to assign device */
774         if (cc->dev == NULL) {
775                 ret = rte_intr_callback_unregister_pending(&cc->intr_handle,
776                                                            memif_intr_handler,
777                                                            cc,
778                                                            memif_intr_unregister_handler);
779                 if (ret < 0)
780                         MIF_LOG(WARNING,
781                                 "Failed to unregister control channel callback.");
782                 return;
783         }
784         /* if memif_msg_receive failed */
785         if (ret < 0)
786                 goto disconnect;
787
788         ret = memif_msg_send_from_queue(cc);
789         if (ret < 0)
790                 goto disconnect;
791
792         return;
793
794  disconnect:
795         if (cc->dev == NULL) {
796                 MIF_LOG(WARNING, "eth dev not allocated");
797                 return;
798         }
799         memif_disconnect(cc->dev);
800 }
801
802 static void
803 memif_listener_handler(void *arg)
804 {
805         struct memif_socket *socket = arg;
806         int sockfd;
807         int addr_len;
808         struct sockaddr_un client;
809         struct memif_control_channel *cc;
810         int ret;
811
812         addr_len = sizeof(client);
813         sockfd = accept(socket->intr_handle.fd, (struct sockaddr *)&client,
814                         (socklen_t *)&addr_len);
815         if (sockfd < 0) {
816                 MIF_LOG(ERR,
817                         "Failed to accept connection request on socket fd %d",
818                         socket->intr_handle.fd);
819                 return;
820         }
821
822         MIF_LOG(DEBUG, "%s: Connection request accepted.", socket->filename);
823
824         cc = rte_zmalloc("memif-cc", sizeof(struct memif_control_channel), 0);
825         if (cc == NULL) {
826                 MIF_LOG(ERR, "Failed to allocate control channel.");
827                 goto error;
828         }
829
830         cc->intr_handle.fd = sockfd;
831         cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
832         cc->socket = socket;
833         cc->dev = NULL;
834         TAILQ_INIT(&cc->msg_queue);
835
836         ret = rte_intr_callback_register(&cc->intr_handle, memif_intr_handler, cc);
837         if (ret < 0) {
838                 MIF_LOG(ERR, "Failed to register control channel callback.");
839                 goto error;
840         }
841
842         ret = memif_msg_enq_hello(cc);
843         if (ret < 0) {
844                 MIF_LOG(ERR, "Failed to enqueue hello message.");
845                 goto error;
846         }
847         ret = memif_msg_send_from_queue(cc);
848         if (ret < 0)
849                 goto error;
850
851         return;
852
853  error:
854         if (sockfd >= 0) {
855                 close(sockfd);
856                 sockfd = -1;
857         }
858         if (cc != NULL)
859                 rte_free(cc);
860 }
861
862 static struct memif_socket *
863 memif_socket_create(struct pmd_internals *pmd,
864                     const char *key, uint8_t listener)
865 {
866         struct memif_socket *sock;
867         struct sockaddr_un un;
868         int sockfd;
869         int ret;
870         int on = 1;
871
872         sock = rte_zmalloc("memif-socket", sizeof(struct memif_socket), 0);
873         if (sock == NULL) {
874                 MIF_LOG(ERR, "Failed to allocate memory for memif socket");
875                 return NULL;
876         }
877
878         sock->listener = listener;
879         strlcpy(sock->filename, key, MEMIF_SOCKET_UN_SIZE);
880         TAILQ_INIT(&sock->dev_queue);
881
882         if (listener != 0) {
883                 sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
884                 if (sockfd < 0)
885                         goto error;
886
887                 un.sun_family = AF_UNIX;
888                 strlcpy(un.sun_path, sock->filename, MEMIF_SOCKET_UN_SIZE);
889
890                 ret = setsockopt(sockfd, SOL_SOCKET, SO_PASSCRED, &on,
891                                  sizeof(on));
892                 if (ret < 0)
893                         goto error;
894
895                 ret = bind(sockfd, (struct sockaddr *)&un, sizeof(un));
896                 if (ret < 0)
897                         goto error;
898
899                 ret = listen(sockfd, 1);
900                 if (ret < 0)
901                         goto error;
902
903                 MIF_LOG(DEBUG, "%s: Memif listener socket %s created.",
904                         rte_vdev_device_name(pmd->vdev), sock->filename);
905
906                 sock->intr_handle.fd = sockfd;
907                 sock->intr_handle.type = RTE_INTR_HANDLE_EXT;
908                 ret = rte_intr_callback_register(&sock->intr_handle,
909                                                  memif_listener_handler, sock);
910                 if (ret < 0) {
911                         MIF_LOG(ERR, "%s: Failed to register interrupt "
912                                 "callback for listener socket",
913                                 rte_vdev_device_name(pmd->vdev));
914                         return NULL;
915                 }
916         }
917
918         return sock;
919
920  error:
921         MIF_LOG(ERR, "%s: Failed to setup socket %s: %s",
922                 rte_vdev_device_name(pmd->vdev) ?
923                 rte_vdev_device_name(pmd->vdev) : "NULL", key, strerror(errno));
924         if (sock != NULL)
925                 rte_free(sock);
926         if (sockfd >= 0)
927                 close(sockfd);
928         return NULL;
929 }
930
931 static struct rte_hash *
932 memif_create_socket_hash(void)
933 {
934         struct rte_hash_parameters params = { 0 };
935
936         params.name = MEMIF_SOCKET_HASH_NAME;
937         params.entries = 256;
938         params.key_len = MEMIF_SOCKET_UN_SIZE;
939         params.hash_func = rte_jhash;
940         params.hash_func_init_val = 0;
941         return rte_hash_create(&params);
942 }
943
944 int
945 memif_socket_init(struct rte_eth_dev *dev, const char *socket_filename)
946 {
947         struct pmd_internals *pmd = dev->data->dev_private;
948         struct memif_socket *socket = NULL;
949         struct memif_socket_dev_list_elt *elt;
950         struct pmd_internals *tmp_pmd;
951         struct rte_hash *hash;
952         int ret;
953         char key[MEMIF_SOCKET_UN_SIZE];
954
955         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
956         if (hash == NULL) {
957                 hash = memif_create_socket_hash();
958                 if (hash == NULL) {
959                         MIF_LOG(ERR, "Failed to create memif socket hash.");
960                         return -1;
961                 }
962         }
963
964         memset(key, 0, MEMIF_SOCKET_UN_SIZE);
965         strlcpy(key, socket_filename, MEMIF_SOCKET_UN_SIZE);
966         ret = rte_hash_lookup_data(hash, key, (void **)&socket);
967         if (ret < 0) {
968                 socket = memif_socket_create(pmd, key,
969                                              (pmd->role ==
970                                               MEMIF_ROLE_SLAVE) ? 0 : 1);
971                 if (socket == NULL)
972                         return -1;
973                 ret = rte_hash_add_key_data(hash, key, socket);
974                 if (ret < 0) {
975                         MIF_LOG(ERR, "Failed to add socket to socket hash.");
976                         return ret;
977                 }
978         }
979         pmd->socket_filename = socket->filename;
980
981         if (socket->listener != 0 && pmd->role == MEMIF_ROLE_SLAVE) {
982                 MIF_LOG(ERR, "Socket is a listener.");
983                 return -1;
984         } else if ((socket->listener == 0) && (pmd->role == MEMIF_ROLE_MASTER)) {
985                 MIF_LOG(ERR, "Socket is not a listener.");
986                 return -1;
987         }
988
989         TAILQ_FOREACH(elt, &socket->dev_queue, next) {
990                 tmp_pmd = elt->dev->data->dev_private;
991                 if (tmp_pmd->id == pmd->id) {
992                         MIF_LOG(ERR, "Memif device with id %d already "
993                                 "exists on socket %s",
994                                 pmd->id, socket->filename);
995                         return -1;
996                 }
997         }
998
999         elt = rte_malloc("pmd-queue", sizeof(struct memif_socket_dev_list_elt), 0);
1000         if (elt == NULL) {
1001                 MIF_LOG(ERR, "%s: Failed to add device to socket device list.",
1002                         rte_vdev_device_name(pmd->vdev));
1003                 return -1;
1004         }
1005         elt->dev = dev;
1006         TAILQ_INSERT_TAIL(&socket->dev_queue, elt, next);
1007
1008         return 0;
1009 }
1010
1011 void
1012 memif_socket_remove_device(struct rte_eth_dev *dev)
1013 {
1014         struct pmd_internals *pmd = dev->data->dev_private;
1015         struct memif_socket *socket = NULL;
1016         struct memif_socket_dev_list_elt *elt, *next;
1017         struct rte_hash *hash;
1018         int ret;
1019
1020         hash = rte_hash_find_existing(MEMIF_SOCKET_HASH_NAME);
1021         if (hash == NULL)
1022                 return;
1023
1024         if (pmd->socket_filename == NULL)
1025                 return;
1026
1027         if (rte_hash_lookup_data(hash, pmd->socket_filename, (void **)&socket) < 0)
1028                 return;
1029
1030         for (elt = TAILQ_FIRST(&socket->dev_queue); elt != NULL; elt = next) {
1031                 next = TAILQ_NEXT(elt, next);
1032                 if (elt->dev == dev) {
1033                         TAILQ_REMOVE(&socket->dev_queue, elt, next);
1034                         rte_free(elt);
1035                         pmd->socket_filename = NULL;
1036                 }
1037         }
1038
1039         /* remove socket, if this was the last device using it */
1040         if (TAILQ_EMPTY(&socket->dev_queue)) {
1041                 rte_hash_del_key(hash, socket->filename);
1042                 if (socket->listener) {
1043                         /* remove listener socket file,
1044                          * so we can create new one later.
1045                          */
1046                         ret = remove(socket->filename);
1047                         if (ret < 0)
1048                                 MIF_LOG(ERR, "Failed to remove socket file: %s",
1049                                         socket->filename);
1050                 }
1051                 rte_free(socket);
1052         }
1053 }
1054
1055 int
1056 memif_connect_master(struct rte_eth_dev *dev)
1057 {
1058         struct pmd_internals *pmd = dev->data->dev_private;
1059
1060         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1061         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1062         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1063         return 0;
1064 }
1065
1066 int
1067 memif_connect_slave(struct rte_eth_dev *dev)
1068 {
1069         int sockfd;
1070         int ret;
1071         struct sockaddr_un sun;
1072         struct pmd_internals *pmd = dev->data->dev_private;
1073
1074         memset(pmd->local_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1075         memset(pmd->remote_disc_string, 0, ETH_MEMIF_DISC_STRING_SIZE);
1076         pmd->flags &= ~ETH_MEMIF_FLAG_DISABLED;
1077
1078         sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
1079         if (sockfd < 0) {
1080                 MIF_LOG(ERR, "%s: Failed to open socket.",
1081                         rte_vdev_device_name(pmd->vdev));
1082                 return -1;
1083         }
1084
1085         sun.sun_family = AF_UNIX;
1086
1087         memcpy(sun.sun_path, pmd->socket_filename, sizeof(sun.sun_path) - 1);
1088
1089         ret = connect(sockfd, (struct sockaddr *)&sun,
1090                       sizeof(struct sockaddr_un));
1091         if (ret < 0) {
1092                 MIF_LOG(ERR, "%s: Failed to connect socket: %s.",
1093                         rte_vdev_device_name(pmd->vdev), pmd->socket_filename);
1094                 goto error;
1095         }
1096
1097         MIF_LOG(DEBUG, "%s: Memif socket: %s connected.",
1098                 rte_vdev_device_name(pmd->vdev), pmd->socket_filename);
1099
1100         pmd->cc = rte_zmalloc("memif-cc",
1101                               sizeof(struct memif_control_channel), 0);
1102         if (pmd->cc == NULL) {
1103                 MIF_LOG(ERR, "%s: Failed to allocate control channel.",
1104                         rte_vdev_device_name(pmd->vdev));
1105                 goto error;
1106         }
1107
1108         pmd->cc->intr_handle.fd = sockfd;
1109         pmd->cc->intr_handle.type = RTE_INTR_HANDLE_EXT;
1110         pmd->cc->socket = NULL;
1111         pmd->cc->dev = dev;
1112         TAILQ_INIT(&pmd->cc->msg_queue);
1113
1114         ret = rte_intr_callback_register(&pmd->cc->intr_handle,
1115                                          memif_intr_handler, pmd->cc);
1116         if (ret < 0) {
1117                 MIF_LOG(ERR, "%s: Failed to register interrupt callback "
1118                         "for control fd", rte_vdev_device_name(pmd->vdev));
1119                 goto error;
1120         }
1121
1122         return 0;
1123
1124  error:
1125         if (sockfd >= 0) {
1126                 close(sockfd);
1127                 sockfd = -1;
1128         }
1129         if (pmd->cc != NULL) {
1130                 rte_free(pmd->cc);
1131                 pmd->cc = NULL;
1132         }
1133         return -1;
1134 }