1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(c) 2010-2014 Intel Corporation.
6 #include <linux/version.h>
7 #include <linux/module.h>
8 #include <linux/miscdevice.h>
9 #include <linux/netdevice.h>
10 #include <linux/etherdevice.h>
11 #include <linux/pci.h>
12 #include <linux/kthread.h>
13 #include <linux/rwsem.h>
14 #include <linux/mutex.h>
15 #include <linux/nsproxy.h>
16 #include <net/net_namespace.h>
17 #include <net/netns/generic.h>
19 #include <rte_kni_common.h>
24 MODULE_VERSION(KNI_VERSION);
25 MODULE_LICENSE("Dual BSD/GPL");
26 MODULE_AUTHOR("Intel Corporation");
27 MODULE_DESCRIPTION("Kernel Module for managing kni devices");
29 #define KNI_RX_LOOP_NUM 1000
31 #define KNI_MAX_DEVICES 32
36 /* Kernel thread mode */
37 static char *kthread_mode;
38 static uint32_t multiple_kthread_on;
40 /* Default carrier state for created KNI network interfaces */
42 uint32_t kni_dflt_carrier;
44 /* Request processing support for bifurcated drivers. */
45 static char *enable_bifurcated;
46 uint32_t bifurcated_support;
48 /* KNI thread scheduling interval */
49 static long min_scheduling_interval = 100; /* us */
50 static long max_scheduling_interval = 200; /* us */
52 #define KNI_DEV_IN_USE_BIT_NUM 0 /* Bit number for device in use */
54 static int kni_net_id;
57 unsigned long device_in_use; /* device in use flag */
58 struct mutex kni_kthread_lock;
59 struct task_struct *kni_kthread;
60 struct rw_semaphore kni_list_lock;
61 struct list_head kni_list_head;
65 kni_init_net(struct net *net)
67 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
68 struct kni_net *knet = net_generic(net, kni_net_id);
70 memset(knet, 0, sizeof(*knet));
75 knet = kzalloc(sizeof(struct kni_net), GFP_KERNEL);
82 /* Clear the bit of device in use */
83 clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use);
85 mutex_init(&knet->kni_kthread_lock);
87 init_rwsem(&knet->kni_list_lock);
88 INIT_LIST_HEAD(&knet->kni_list_head);
90 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
93 ret = net_assign_generic(net, kni_net_id, knet);
101 static void __net_exit
102 kni_exit_net(struct net *net)
104 struct kni_net *knet __maybe_unused;
106 knet = net_generic(net, kni_net_id);
107 mutex_destroy(&knet->kni_kthread_lock);
109 #ifndef HAVE_SIMPLIFIED_PERNET_OPERATIONS
114 static struct pernet_operations kni_net_ops = {
115 .init = kni_init_net,
116 .exit = kni_exit_net,
117 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
119 .size = sizeof(struct kni_net),
124 kni_thread_single(void *data)
126 struct kni_net *knet = data;
130 while (!kthread_should_stop()) {
131 down_read(&knet->kni_list_lock);
132 for (j = 0; j < KNI_RX_LOOP_NUM; j++) {
133 list_for_each_entry(dev, &knet->kni_list_head, list) {
135 kni_net_poll_resp(dev);
138 up_read(&knet->kni_list_lock);
139 /* reschedule out for a while */
140 usleep_range(min_scheduling_interval, max_scheduling_interval);
147 kni_thread_multiple(void *param)
150 struct kni_dev *dev = param;
152 while (!kthread_should_stop()) {
153 for (j = 0; j < KNI_RX_LOOP_NUM; j++) {
155 kni_net_poll_resp(dev);
157 usleep_range(min_scheduling_interval, max_scheduling_interval);
164 kni_open(struct inode *inode, struct file *file)
166 struct net *net = current->nsproxy->net_ns;
167 struct kni_net *knet = net_generic(net, kni_net_id);
169 /* kni device can be opened by one user only per netns */
170 if (test_and_set_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use))
173 file->private_data = get_net(net);
174 pr_debug("/dev/kni opened\n");
180 kni_dev_remove(struct kni_dev *dev)
186 unregister_netdev(dev->net_dev);
187 free_netdev(dev->net_dev);
190 kni_net_release_fifo_phy(dev);
196 kni_release(struct inode *inode, struct file *file)
198 struct net *net = file->private_data;
199 struct kni_net *knet = net_generic(net, kni_net_id);
200 struct kni_dev *dev, *n;
202 /* Stop kernel thread for single mode */
203 if (multiple_kthread_on == 0) {
204 mutex_lock(&knet->kni_kthread_lock);
205 /* Stop kernel thread */
206 if (knet->kni_kthread != NULL) {
207 kthread_stop(knet->kni_kthread);
208 knet->kni_kthread = NULL;
210 mutex_unlock(&knet->kni_kthread_lock);
213 down_write(&knet->kni_list_lock);
214 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) {
215 /* Stop kernel thread for multiple mode */
216 if (multiple_kthread_on && dev->pthread != NULL) {
217 kthread_stop(dev->pthread);
222 list_del(&dev->list);
224 up_write(&knet->kni_list_lock);
226 /* Clear the bit of device in use */
227 clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use);
230 pr_debug("/dev/kni closed\n");
236 kni_check_param(struct kni_dev *kni, struct rte_kni_device_info *dev)
241 /* Check if network name has been used */
242 if (!strncmp(kni->name, dev->name, RTE_KNI_NAMESIZE)) {
243 pr_err("KNI name %s duplicated\n", dev->name);
251 kni_run_thread(struct kni_net *knet, struct kni_dev *kni, uint8_t force_bind)
254 * Create a new kernel thread for multiple mode, set its core affinity,
255 * and finally wake it up.
257 if (multiple_kthread_on) {
258 kni->pthread = kthread_create(kni_thread_multiple,
259 (void *)kni, "kni_%s", kni->name);
260 if (IS_ERR(kni->pthread)) {
266 kthread_bind(kni->pthread, kni->core_id);
267 wake_up_process(kni->pthread);
269 mutex_lock(&knet->kni_kthread_lock);
271 if (knet->kni_kthread == NULL) {
272 knet->kni_kthread = kthread_create(kni_thread_single,
273 (void *)knet, "kni_single");
274 if (IS_ERR(knet->kni_kthread)) {
275 mutex_unlock(&knet->kni_kthread_lock);
281 kthread_bind(knet->kni_kthread, kni->core_id);
282 wake_up_process(knet->kni_kthread);
285 mutex_unlock(&knet->kni_kthread_lock);
292 kni_ioctl_create(struct net *net, uint32_t ioctl_num,
293 unsigned long ioctl_param)
295 struct kni_net *knet = net_generic(net, kni_net_id);
297 struct rte_kni_device_info dev_info;
298 struct net_device *net_dev = NULL;
299 struct kni_dev *kni, *dev, *n;
301 pr_info("Creating kni...\n");
302 /* Check the buffer size, to avoid warning */
303 if (_IOC_SIZE(ioctl_num) > sizeof(dev_info))
306 /* Copy kni info from user space */
307 if (copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)))
310 /* Check if name is zero-ended */
311 if (strnlen(dev_info.name, sizeof(dev_info.name)) == sizeof(dev_info.name)) {
312 pr_err("kni.name not zero-terminated");
317 * Check if the cpu core id is valid for binding.
319 if (dev_info.force_bind && !cpu_online(dev_info.core_id)) {
320 pr_err("cpu %u is not online\n", dev_info.core_id);
324 /* Check if it has been created */
325 down_read(&knet->kni_list_lock);
326 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) {
327 if (kni_check_param(dev, &dev_info) < 0) {
328 up_read(&knet->kni_list_lock);
332 up_read(&knet->kni_list_lock);
334 net_dev = alloc_netdev(sizeof(struct kni_dev), dev_info.name,
339 if (net_dev == NULL) {
340 pr_err("error allocating device \"%s\"\n", dev_info.name);
344 dev_net_set(net_dev, net);
346 kni = netdev_priv(net_dev);
348 kni->net_dev = net_dev;
349 kni->core_id = dev_info.core_id;
350 strncpy(kni->name, dev_info.name, RTE_KNI_NAMESIZE);
352 /* Translate user space info into kernel space info */
353 if (dev_info.iova_mode) {
354 #ifdef HAVE_IOVA_TO_KVA_MAPPING_SUPPORT
355 kni->tx_q = iova_to_kva(current, dev_info.tx_phys);
356 kni->rx_q = iova_to_kva(current, dev_info.rx_phys);
357 kni->alloc_q = iova_to_kva(current, dev_info.alloc_phys);
358 kni->free_q = iova_to_kva(current, dev_info.free_phys);
360 kni->req_q = iova_to_kva(current, dev_info.req_phys);
361 kni->resp_q = iova_to_kva(current, dev_info.resp_phys);
362 kni->sync_va = dev_info.sync_va;
363 kni->sync_kva = iova_to_kva(current, dev_info.sync_phys);
364 kni->usr_tsk = current;
367 pr_err("KNI module does not support IOVA to VA translation\n");
372 kni->tx_q = phys_to_virt(dev_info.tx_phys);
373 kni->rx_q = phys_to_virt(dev_info.rx_phys);
374 kni->alloc_q = phys_to_virt(dev_info.alloc_phys);
375 kni->free_q = phys_to_virt(dev_info.free_phys);
377 kni->req_q = phys_to_virt(dev_info.req_phys);
378 kni->resp_q = phys_to_virt(dev_info.resp_phys);
379 kni->sync_va = dev_info.sync_va;
380 kni->sync_kva = phys_to_virt(dev_info.sync_phys);
384 kni->mbuf_size = dev_info.mbuf_size;
386 pr_debug("tx_phys: 0x%016llx, tx_q addr: 0x%p\n",
387 (unsigned long long) dev_info.tx_phys, kni->tx_q);
388 pr_debug("rx_phys: 0x%016llx, rx_q addr: 0x%p\n",
389 (unsigned long long) dev_info.rx_phys, kni->rx_q);
390 pr_debug("alloc_phys: 0x%016llx, alloc_q addr: 0x%p\n",
391 (unsigned long long) dev_info.alloc_phys, kni->alloc_q);
392 pr_debug("free_phys: 0x%016llx, free_q addr: 0x%p\n",
393 (unsigned long long) dev_info.free_phys, kni->free_q);
394 pr_debug("req_phys: 0x%016llx, req_q addr: 0x%p\n",
395 (unsigned long long) dev_info.req_phys, kni->req_q);
396 pr_debug("resp_phys: 0x%016llx, resp_q addr: 0x%p\n",
397 (unsigned long long) dev_info.resp_phys, kni->resp_q);
398 pr_debug("mbuf_size: %u\n", kni->mbuf_size);
400 /* if user has provided a valid mac address */
401 if (is_valid_ether_addr(dev_info.mac_addr))
402 memcpy(net_dev->dev_addr, dev_info.mac_addr, ETH_ALEN);
404 /* Generate random MAC address. */
405 eth_random_addr(net_dev->dev_addr);
408 net_dev->mtu = dev_info.mtu;
409 #ifdef HAVE_MAX_MTU_PARAM
410 net_dev->max_mtu = net_dev->mtu;
412 if (dev_info.min_mtu)
413 net_dev->min_mtu = dev_info.min_mtu;
415 if (dev_info.max_mtu)
416 net_dev->max_mtu = dev_info.max_mtu;
419 ret = register_netdev(net_dev);
421 pr_err("error %i registering device \"%s\"\n",
425 free_netdev(net_dev);
429 netif_carrier_off(net_dev);
431 ret = kni_run_thread(knet, kni, dev_info.force_bind);
435 down_write(&knet->kni_list_lock);
436 list_add(&kni->list, &knet->kni_list_head);
437 up_write(&knet->kni_list_lock);
443 kni_ioctl_release(struct net *net, uint32_t ioctl_num,
444 unsigned long ioctl_param)
446 struct kni_net *knet = net_generic(net, kni_net_id);
448 struct kni_dev *dev, *n;
449 struct rte_kni_device_info dev_info;
451 if (_IOC_SIZE(ioctl_num) > sizeof(dev_info))
454 if (copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)))
457 /* Release the network device according to its name */
458 if (strlen(dev_info.name) == 0)
461 down_write(&knet->kni_list_lock);
462 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) {
463 if (strncmp(dev->name, dev_info.name, RTE_KNI_NAMESIZE) != 0)
466 if (multiple_kthread_on && dev->pthread != NULL) {
467 kthread_stop(dev->pthread);
472 list_del(&dev->list);
476 up_write(&knet->kni_list_lock);
477 pr_info("%s release kni named %s\n",
478 (ret == 0 ? "Successfully" : "Unsuccessfully"), dev_info.name);
484 kni_ioctl(struct inode *inode, uint32_t ioctl_num, unsigned long ioctl_param)
487 struct net *net = current->nsproxy->net_ns;
489 pr_debug("IOCTL num=0x%0x param=0x%0lx\n", ioctl_num, ioctl_param);
492 * Switch according to the ioctl called
494 switch (_IOC_NR(ioctl_num)) {
495 case _IOC_NR(RTE_KNI_IOCTL_TEST):
496 /* For test only, not used */
498 case _IOC_NR(RTE_KNI_IOCTL_CREATE):
499 ret = kni_ioctl_create(net, ioctl_num, ioctl_param);
501 case _IOC_NR(RTE_KNI_IOCTL_RELEASE):
502 ret = kni_ioctl_release(net, ioctl_num, ioctl_param);
505 pr_debug("IOCTL default\n");
513 kni_compat_ioctl(struct inode *inode, uint32_t ioctl_num,
514 unsigned long ioctl_param)
516 /* 32 bits app on 64 bits OS to be supported later */
517 pr_debug("Not implemented.\n");
522 static const struct file_operations kni_fops = {
523 .owner = THIS_MODULE,
525 .release = kni_release,
526 .unlocked_ioctl = (void *)kni_ioctl,
527 .compat_ioctl = (void *)kni_compat_ioctl,
530 static struct miscdevice kni_misc = {
531 .minor = MISC_DYNAMIC_MINOR,
537 kni_parse_kthread_mode(void)
542 if (strcmp(kthread_mode, "single") == 0)
544 else if (strcmp(kthread_mode, "multiple") == 0)
545 multiple_kthread_on = 1;
553 kni_parse_carrier_state(void)
556 kni_dflt_carrier = 0;
560 if (strcmp(carrier, "off") == 0)
561 kni_dflt_carrier = 0;
562 else if (strcmp(carrier, "on") == 0)
563 kni_dflt_carrier = 1;
571 kni_parse_bifurcated_support(void)
573 if (!enable_bifurcated) {
574 bifurcated_support = 0;
578 if (strcmp(enable_bifurcated, "on") == 0)
579 bifurcated_support = 1;
591 if (kni_parse_kthread_mode() < 0) {
592 pr_err("Invalid parameter for kthread_mode\n");
596 if (multiple_kthread_on == 0)
597 pr_debug("Single kernel thread for all KNI devices\n");
599 pr_debug("Multiple kernel thread mode enabled\n");
601 if (kni_parse_carrier_state() < 0) {
602 pr_err("Invalid parameter for carrier\n");
606 if (kni_dflt_carrier == 0)
607 pr_debug("Default carrier state set to off.\n");
609 pr_debug("Default carrier state set to on.\n");
611 if (kni_parse_bifurcated_support() < 0) {
612 pr_err("Invalid parameter for bifurcated support\n");
615 if (bifurcated_support == 1)
616 pr_debug("bifurcated support is enabled.\n");
618 if (min_scheduling_interval < 0 || max_scheduling_interval < 0 ||
619 min_scheduling_interval > KNI_KTHREAD_MAX_RESCHEDULE_INTERVAL ||
620 max_scheduling_interval > KNI_KTHREAD_MAX_RESCHEDULE_INTERVAL ||
621 min_scheduling_interval >= max_scheduling_interval) {
622 pr_err("Invalid parameters for scheduling interval\n");
626 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
627 rc = register_pernet_subsys(&kni_net_ops);
629 rc = register_pernet_gen_subsys(&kni_net_id, &kni_net_ops);
634 rc = misc_register(&kni_misc);
636 pr_err("Misc registration failed\n");
640 /* Configure the lo mode according to the input parameter */
641 kni_net_config_lo_mode(lo_mode);
646 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
647 unregister_pernet_subsys(&kni_net_ops);
649 unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops);
657 misc_deregister(&kni_misc);
658 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
659 unregister_pernet_subsys(&kni_net_ops);
661 unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops);
665 module_init(kni_init);
666 module_exit(kni_exit);
668 module_param(lo_mode, charp, 0644);
669 MODULE_PARM_DESC(lo_mode,
670 "KNI loopback mode (default=lo_mode_none):\n"
671 "\t\tlo_mode_none Kernel loopback disabled\n"
672 "\t\tlo_mode_fifo Enable kernel loopback with fifo\n"
673 "\t\tlo_mode_fifo_skb Enable kernel loopback with fifo and skb buffer\n"
677 module_param(kthread_mode, charp, 0644);
678 MODULE_PARM_DESC(kthread_mode,
679 "Kernel thread mode (default=single):\n"
680 "\t\tsingle Single kernel thread mode enabled.\n"
681 "\t\tmultiple Multiple kernel thread mode enabled.\n"
685 module_param(carrier, charp, 0644);
686 MODULE_PARM_DESC(carrier,
687 "Default carrier state for KNI interface (default=off):\n"
688 "\t\toff Interfaces will be created with carrier state set to off.\n"
689 "\t\ton Interfaces will be created with carrier state set to on.\n"
693 module_param(enable_bifurcated, charp, 0644);
694 MODULE_PARM_DESC(enable_bifurcated,
695 "Enable request processing support for bifurcated drivers, "
696 "which means releasing rtnl_lock before calling userspace callback and "
697 "supporting async requests (default=off):\n"
698 "\t\ton Enable request processing support for bifurcated drivers.\n"
702 module_param(min_scheduling_interval, long, 0644);
703 MODULE_PARM_DESC(min_scheduling_interval,
704 "KNI thread min scheduling interval (default=100 microseconds)"
707 module_param(max_scheduling_interval, long, 0644);
708 MODULE_PARM_DESC(max_scheduling_interval,
709 "KNI thread max scheduling interval (default=200 microseconds)"