1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright(c) 2010-2014 Intel Corporation.
6 #include <linux/version.h>
7 #include <linux/module.h>
8 #include <linux/miscdevice.h>
9 #include <linux/netdevice.h>
10 #include <linux/etherdevice.h>
11 #include <linux/pci.h>
12 #include <linux/kthread.h>
13 #include <linux/rwsem.h>
14 #include <linux/mutex.h>
15 #include <linux/nsproxy.h>
16 #include <net/net_namespace.h>
17 #include <net/netns/generic.h>
19 #include <rte_kni_common.h>
24 MODULE_VERSION(KNI_VERSION);
25 MODULE_LICENSE("Dual BSD/GPL");
26 MODULE_AUTHOR("Intel Corporation");
27 MODULE_DESCRIPTION("Kernel Module for managing kni devices");
29 #define KNI_RX_LOOP_NUM 1000
31 #define KNI_MAX_DEVICES 32
36 /* Kernel thread mode */
37 static char *kthread_mode;
38 static uint32_t multiple_kthread_on;
40 /* Default carrier state for created KNI network interfaces */
42 uint32_t kni_dflt_carrier;
44 /* Request processing support for bifurcated drivers. */
45 static char *enable_bifurcated;
46 uint32_t bifurcated_support;
48 /* KNI thread scheduling interval */
49 static long min_scheduling_interval = 100; /* us */
50 static long max_scheduling_interval = 200; /* us */
52 #define KNI_DEV_IN_USE_BIT_NUM 0 /* Bit number for device in use */
54 static int kni_net_id;
57 unsigned long device_in_use; /* device in use flag */
58 struct mutex kni_kthread_lock;
59 struct task_struct *kni_kthread;
60 struct rw_semaphore kni_list_lock;
61 struct list_head kni_list_head;
65 kni_init_net(struct net *net)
67 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
68 struct kni_net *knet = net_generic(net, kni_net_id);
70 memset(knet, 0, sizeof(*knet));
75 knet = kzalloc(sizeof(struct kni_net), GFP_KERNEL);
82 /* Clear the bit of device in use */
83 clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use);
85 mutex_init(&knet->kni_kthread_lock);
87 init_rwsem(&knet->kni_list_lock);
88 INIT_LIST_HEAD(&knet->kni_list_head);
90 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
93 ret = net_assign_generic(net, kni_net_id, knet);
101 static void __net_exit
102 kni_exit_net(struct net *net)
104 struct kni_net *knet __maybe_unused;
106 knet = net_generic(net, kni_net_id);
107 mutex_destroy(&knet->kni_kthread_lock);
109 #ifndef HAVE_SIMPLIFIED_PERNET_OPERATIONS
114 static struct pernet_operations kni_net_ops = {
115 .init = kni_init_net,
116 .exit = kni_exit_net,
117 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
119 .size = sizeof(struct kni_net),
124 kni_thread_single(void *data)
126 struct kni_net *knet = data;
130 while (!kthread_should_stop()) {
131 down_read(&knet->kni_list_lock);
132 for (j = 0; j < KNI_RX_LOOP_NUM; j++) {
133 list_for_each_entry(dev, &knet->kni_list_head, list) {
135 kni_net_poll_resp(dev);
138 up_read(&knet->kni_list_lock);
139 /* reschedule out for a while */
140 usleep_range(min_scheduling_interval, max_scheduling_interval);
147 kni_thread_multiple(void *param)
150 struct kni_dev *dev = param;
152 while (!kthread_should_stop()) {
153 for (j = 0; j < KNI_RX_LOOP_NUM; j++) {
155 kni_net_poll_resp(dev);
157 usleep_range(min_scheduling_interval, max_scheduling_interval);
164 kni_open(struct inode *inode, struct file *file)
166 struct net *net = current->nsproxy->net_ns;
167 struct kni_net *knet = net_generic(net, kni_net_id);
169 /* kni device can be opened by one user only per netns */
170 if (test_and_set_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use))
173 file->private_data = get_net(net);
174 pr_debug("/dev/kni opened\n");
180 kni_dev_remove(struct kni_dev *dev)
186 * The memory of kni device is allocated and released together
187 * with net device. Release mbuf before freeing net device.
189 kni_net_release_fifo_phy(dev);
192 unregister_netdev(dev->net_dev);
193 free_netdev(dev->net_dev);
200 kni_release(struct inode *inode, struct file *file)
202 struct net *net = file->private_data;
203 struct kni_net *knet = net_generic(net, kni_net_id);
204 struct kni_dev *dev, *n;
206 /* Stop kernel thread for single mode */
207 if (multiple_kthread_on == 0) {
208 mutex_lock(&knet->kni_kthread_lock);
209 /* Stop kernel thread */
210 if (knet->kni_kthread != NULL) {
211 kthread_stop(knet->kni_kthread);
212 knet->kni_kthread = NULL;
214 mutex_unlock(&knet->kni_kthread_lock);
217 down_write(&knet->kni_list_lock);
218 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) {
219 /* Stop kernel thread for multiple mode */
220 if (multiple_kthread_on && dev->pthread != NULL) {
221 kthread_stop(dev->pthread);
225 list_del(&dev->list);
228 up_write(&knet->kni_list_lock);
230 /* Clear the bit of device in use */
231 clear_bit(KNI_DEV_IN_USE_BIT_NUM, &knet->device_in_use);
234 pr_debug("/dev/kni closed\n");
240 kni_check_param(struct kni_dev *kni, struct rte_kni_device_info *dev)
245 /* Check if network name has been used */
246 if (!strncmp(kni->name, dev->name, RTE_KNI_NAMESIZE)) {
247 pr_err("KNI name %s duplicated\n", dev->name);
255 kni_run_thread(struct kni_net *knet, struct kni_dev *kni, uint8_t force_bind)
258 * Create a new kernel thread for multiple mode, set its core affinity,
259 * and finally wake it up.
261 if (multiple_kthread_on) {
262 kni->pthread = kthread_create(kni_thread_multiple,
263 (void *)kni, "kni_%s", kni->name);
264 if (IS_ERR(kni->pthread)) {
270 kthread_bind(kni->pthread, kni->core_id);
271 wake_up_process(kni->pthread);
273 mutex_lock(&knet->kni_kthread_lock);
275 if (knet->kni_kthread == NULL) {
276 knet->kni_kthread = kthread_create(kni_thread_single,
277 (void *)knet, "kni_single");
278 if (IS_ERR(knet->kni_kthread)) {
279 mutex_unlock(&knet->kni_kthread_lock);
285 kthread_bind(knet->kni_kthread, kni->core_id);
286 wake_up_process(knet->kni_kthread);
289 mutex_unlock(&knet->kni_kthread_lock);
296 kni_ioctl_create(struct net *net, uint32_t ioctl_num,
297 unsigned long ioctl_param)
299 struct kni_net *knet = net_generic(net, kni_net_id);
301 struct rte_kni_device_info dev_info;
302 struct net_device *net_dev = NULL;
303 struct kni_dev *kni, *dev, *n;
305 pr_info("Creating kni...\n");
306 /* Check the buffer size, to avoid warning */
307 if (_IOC_SIZE(ioctl_num) > sizeof(dev_info))
310 /* Copy kni info from user space */
311 if (copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)))
314 /* Check if name is zero-ended */
315 if (strnlen(dev_info.name, sizeof(dev_info.name)) == sizeof(dev_info.name)) {
316 pr_err("kni.name not zero-terminated");
321 * Check if the cpu core id is valid for binding.
323 if (dev_info.force_bind && !cpu_online(dev_info.core_id)) {
324 pr_err("cpu %u is not online\n", dev_info.core_id);
328 /* Check if it has been created */
329 down_read(&knet->kni_list_lock);
330 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) {
331 if (kni_check_param(dev, &dev_info) < 0) {
332 up_read(&knet->kni_list_lock);
336 up_read(&knet->kni_list_lock);
338 net_dev = alloc_netdev(sizeof(struct kni_dev), dev_info.name,
343 if (net_dev == NULL) {
344 pr_err("error allocating device \"%s\"\n", dev_info.name);
348 dev_net_set(net_dev, net);
350 kni = netdev_priv(net_dev);
352 kni->net_dev = net_dev;
353 kni->core_id = dev_info.core_id;
354 strncpy(kni->name, dev_info.name, RTE_KNI_NAMESIZE);
356 /* Translate user space info into kernel space info */
357 if (dev_info.iova_mode) {
358 #ifdef HAVE_IOVA_TO_KVA_MAPPING_SUPPORT
359 kni->tx_q = iova_to_kva(current, dev_info.tx_phys);
360 kni->rx_q = iova_to_kva(current, dev_info.rx_phys);
361 kni->alloc_q = iova_to_kva(current, dev_info.alloc_phys);
362 kni->free_q = iova_to_kva(current, dev_info.free_phys);
364 kni->req_q = iova_to_kva(current, dev_info.req_phys);
365 kni->resp_q = iova_to_kva(current, dev_info.resp_phys);
366 kni->sync_va = dev_info.sync_va;
367 kni->sync_kva = iova_to_kva(current, dev_info.sync_phys);
368 kni->usr_tsk = current;
371 pr_err("KNI module does not support IOVA to VA translation\n");
376 kni->tx_q = phys_to_virt(dev_info.tx_phys);
377 kni->rx_q = phys_to_virt(dev_info.rx_phys);
378 kni->alloc_q = phys_to_virt(dev_info.alloc_phys);
379 kni->free_q = phys_to_virt(dev_info.free_phys);
381 kni->req_q = phys_to_virt(dev_info.req_phys);
382 kni->resp_q = phys_to_virt(dev_info.resp_phys);
383 kni->sync_va = dev_info.sync_va;
384 kni->sync_kva = phys_to_virt(dev_info.sync_phys);
388 kni->mbuf_size = dev_info.mbuf_size;
390 pr_debug("tx_phys: 0x%016llx, tx_q addr: 0x%p\n",
391 (unsigned long long) dev_info.tx_phys, kni->tx_q);
392 pr_debug("rx_phys: 0x%016llx, rx_q addr: 0x%p\n",
393 (unsigned long long) dev_info.rx_phys, kni->rx_q);
394 pr_debug("alloc_phys: 0x%016llx, alloc_q addr: 0x%p\n",
395 (unsigned long long) dev_info.alloc_phys, kni->alloc_q);
396 pr_debug("free_phys: 0x%016llx, free_q addr: 0x%p\n",
397 (unsigned long long) dev_info.free_phys, kni->free_q);
398 pr_debug("req_phys: 0x%016llx, req_q addr: 0x%p\n",
399 (unsigned long long) dev_info.req_phys, kni->req_q);
400 pr_debug("resp_phys: 0x%016llx, resp_q addr: 0x%p\n",
401 (unsigned long long) dev_info.resp_phys, kni->resp_q);
402 pr_debug("mbuf_size: %u\n", kni->mbuf_size);
404 /* if user has provided a valid mac address */
405 if (is_valid_ether_addr(dev_info.mac_addr)) {
406 #ifdef HAVE_ETH_HW_ADDR_SET
407 eth_hw_addr_set(net_dev, dev_info.mac_addr);
409 memcpy(net_dev->dev_addr, dev_info.mac_addr, ETH_ALEN);
412 /* Assign random MAC address. */
413 eth_hw_addr_random(net_dev);
417 net_dev->mtu = dev_info.mtu;
418 #ifdef HAVE_MAX_MTU_PARAM
419 net_dev->max_mtu = net_dev->mtu;
421 if (dev_info.min_mtu)
422 net_dev->min_mtu = dev_info.min_mtu;
424 if (dev_info.max_mtu)
425 net_dev->max_mtu = dev_info.max_mtu;
428 ret = register_netdev(net_dev);
430 pr_err("error %i registering device \"%s\"\n",
434 free_netdev(net_dev);
438 netif_carrier_off(net_dev);
440 ret = kni_run_thread(knet, kni, dev_info.force_bind);
444 down_write(&knet->kni_list_lock);
445 list_add(&kni->list, &knet->kni_list_head);
446 up_write(&knet->kni_list_lock);
452 kni_ioctl_release(struct net *net, uint32_t ioctl_num,
453 unsigned long ioctl_param)
455 struct kni_net *knet = net_generic(net, kni_net_id);
457 struct kni_dev *dev, *n;
458 struct rte_kni_device_info dev_info;
460 if (_IOC_SIZE(ioctl_num) > sizeof(dev_info))
463 if (copy_from_user(&dev_info, (void *)ioctl_param, sizeof(dev_info)))
466 /* Release the network device according to its name */
467 if (strlen(dev_info.name) == 0)
470 down_write(&knet->kni_list_lock);
471 list_for_each_entry_safe(dev, n, &knet->kni_list_head, list) {
472 if (strncmp(dev->name, dev_info.name, RTE_KNI_NAMESIZE) != 0)
475 if (multiple_kthread_on && dev->pthread != NULL) {
476 kthread_stop(dev->pthread);
480 list_del(&dev->list);
485 up_write(&knet->kni_list_lock);
486 pr_info("%s release kni named %s\n",
487 (ret == 0 ? "Successfully" : "Unsuccessfully"), dev_info.name);
493 kni_ioctl(struct file *file, unsigned int ioctl_num, unsigned long ioctl_param)
496 struct net *net = current->nsproxy->net_ns;
498 pr_debug("IOCTL num=0x%0x param=0x%0lx\n", ioctl_num, ioctl_param);
501 * Switch according to the ioctl called
503 switch (_IOC_NR(ioctl_num)) {
504 case _IOC_NR(RTE_KNI_IOCTL_TEST):
505 /* For test only, not used */
507 case _IOC_NR(RTE_KNI_IOCTL_CREATE):
508 ret = kni_ioctl_create(net, ioctl_num, ioctl_param);
510 case _IOC_NR(RTE_KNI_IOCTL_RELEASE):
511 ret = kni_ioctl_release(net, ioctl_num, ioctl_param);
514 pr_debug("IOCTL default\n");
522 kni_compat_ioctl(struct file *file, unsigned int ioctl_num,
523 unsigned long ioctl_param)
525 /* 32 bits app on 64 bits OS to be supported later */
526 pr_debug("Not implemented.\n");
531 static const struct file_operations kni_fops = {
532 .owner = THIS_MODULE,
534 .release = kni_release,
535 .unlocked_ioctl = kni_ioctl,
536 .compat_ioctl = kni_compat_ioctl,
539 static struct miscdevice kni_misc = {
540 .minor = MISC_DYNAMIC_MINOR,
546 kni_parse_kthread_mode(void)
551 if (strcmp(kthread_mode, "single") == 0)
553 else if (strcmp(kthread_mode, "multiple") == 0)
554 multiple_kthread_on = 1;
562 kni_parse_carrier_state(void)
565 kni_dflt_carrier = 0;
569 if (strcmp(carrier, "off") == 0)
570 kni_dflt_carrier = 0;
571 else if (strcmp(carrier, "on") == 0)
572 kni_dflt_carrier = 1;
580 kni_parse_bifurcated_support(void)
582 if (!enable_bifurcated) {
583 bifurcated_support = 0;
587 if (strcmp(enable_bifurcated, "on") == 0)
588 bifurcated_support = 1;
600 if (kni_parse_kthread_mode() < 0) {
601 pr_err("Invalid parameter for kthread_mode\n");
605 if (multiple_kthread_on == 0)
606 pr_debug("Single kernel thread for all KNI devices\n");
608 pr_debug("Multiple kernel thread mode enabled\n");
610 if (kni_parse_carrier_state() < 0) {
611 pr_err("Invalid parameter for carrier\n");
615 if (kni_dflt_carrier == 0)
616 pr_debug("Default carrier state set to off.\n");
618 pr_debug("Default carrier state set to on.\n");
620 if (kni_parse_bifurcated_support() < 0) {
621 pr_err("Invalid parameter for bifurcated support\n");
624 if (bifurcated_support == 1)
625 pr_debug("bifurcated support is enabled.\n");
627 if (min_scheduling_interval < 0 || max_scheduling_interval < 0 ||
628 min_scheduling_interval > KNI_KTHREAD_MAX_RESCHEDULE_INTERVAL ||
629 max_scheduling_interval > KNI_KTHREAD_MAX_RESCHEDULE_INTERVAL ||
630 min_scheduling_interval >= max_scheduling_interval) {
631 pr_err("Invalid parameters for scheduling interval\n");
635 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
636 rc = register_pernet_subsys(&kni_net_ops);
638 rc = register_pernet_gen_subsys(&kni_net_id, &kni_net_ops);
643 rc = misc_register(&kni_misc);
645 pr_err("Misc registration failed\n");
649 /* Configure the lo mode according to the input parameter */
650 kni_net_config_lo_mode(lo_mode);
655 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
656 unregister_pernet_subsys(&kni_net_ops);
658 unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops);
666 misc_deregister(&kni_misc);
667 #ifdef HAVE_SIMPLIFIED_PERNET_OPERATIONS
668 unregister_pernet_subsys(&kni_net_ops);
670 unregister_pernet_gen_subsys(kni_net_id, &kni_net_ops);
674 module_init(kni_init);
675 module_exit(kni_exit);
677 module_param(lo_mode, charp, 0644);
678 MODULE_PARM_DESC(lo_mode,
679 "KNI loopback mode (default=lo_mode_none):\n"
680 "\t\tlo_mode_none Kernel loopback disabled\n"
681 "\t\tlo_mode_fifo Enable kernel loopback with fifo\n"
682 "\t\tlo_mode_fifo_skb Enable kernel loopback with fifo and skb buffer\n"
686 module_param(kthread_mode, charp, 0644);
687 MODULE_PARM_DESC(kthread_mode,
688 "Kernel thread mode (default=single):\n"
689 "\t\tsingle Single kernel thread mode enabled.\n"
690 "\t\tmultiple Multiple kernel thread mode enabled.\n"
694 module_param(carrier, charp, 0644);
695 MODULE_PARM_DESC(carrier,
696 "Default carrier state for KNI interface (default=off):\n"
697 "\t\toff Interfaces will be created with carrier state set to off.\n"
698 "\t\ton Interfaces will be created with carrier state set to on.\n"
702 module_param(enable_bifurcated, charp, 0644);
703 MODULE_PARM_DESC(enable_bifurcated,
704 "Enable request processing support for bifurcated drivers, "
705 "which means releasing rtnl_lock before calling userspace callback and "
706 "supporting async requests (default=off):\n"
707 "\t\ton Enable request processing support for bifurcated drivers.\n"
711 module_param(min_scheduling_interval, long, 0644);
712 MODULE_PARM_DESC(min_scheduling_interval,
713 "KNI thread min scheduling interval (default=100 microseconds)"
716 module_param(max_scheduling_interval, long, 0644);
717 MODULE_PARM_DESC(max_scheduling_interval,
718 "KNI thread max scheduling interval (default=200 microseconds)"