4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of version 2 of the GNU General Public License as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18 * The full GNU General Public License is included in this distribution
19 * in the file called LICENSE.GPL.
21 * Contact Information:
26 * This code is inspired from the book "Linux Device Drivers" by
27 * Alessandro Rubini and Jonathan Corbet, published by O'Reilly & Associates
30 #include <linux/device.h>
31 #include <linux/module.h>
32 #include <linux/version.h>
33 #include <linux/netdevice.h>
34 #include <linux/etherdevice.h> /* eth_type_trans */
35 #include <linux/skbuff.h>
36 #include <linux/kthread.h>
37 #include <linux/delay.h>
39 #include <exec-env/rte_kni_common.h>
45 #define WD_TIMEOUT 5 /*jiffies */
47 #define MBUF_BURST_SZ 32
49 #define KNI_WAIT_RESPONSE_TIMEOUT 300 /* 3 seconds */
51 /* typedef for rx function */
52 typedef void (*kni_net_rx_t)(struct kni_dev *kni);
54 static int kni_net_tx(struct sk_buff *skb, struct net_device *dev);
55 static void kni_net_rx_normal(struct kni_dev *kni);
56 static void kni_net_rx_lo_fifo(struct kni_dev *kni);
57 static void kni_net_rx_lo_fifo_skb(struct kni_dev *kni);
58 static int kni_net_process_request(struct kni_dev *kni,
59 struct rte_kni_request *req);
61 /* kni rx function pointer, with default to normal rx */
62 static kni_net_rx_t kni_net_rx_func = kni_net_rx_normal;
68 kni_net_open(struct net_device *dev)
71 struct rte_kni_request req;
72 struct kni_dev *kni = netdev_priv(dev);
74 netif_start_queue(dev);
76 memset(&req, 0, sizeof(req));
77 req.req_id = RTE_KNI_REQ_CFG_NETWORK_IF;
79 /* Setting if_up to non-zero means up */
81 ret = kni_net_process_request(kni, &req);
83 return (ret == 0) ? req.result : ret;
87 kni_net_release(struct net_device *dev)
90 struct rte_kni_request req;
91 struct kni_dev *kni = netdev_priv(dev);
93 netif_stop_queue(dev); /* can't transmit any more */
95 memset(&req, 0, sizeof(req));
96 req.req_id = RTE_KNI_REQ_CFG_NETWORK_IF;
98 /* Setting if_up to 0 means down */
100 ret = kni_net_process_request(kni, &req);
102 return (ret == 0) ? req.result : ret;
106 * Configuration changes (passed on by ifconfig)
109 kni_net_config(struct net_device *dev, struct ifmap *map)
111 if (dev->flags & IFF_UP) /* can't act on a running interface */
114 /* ignore other fields */
119 * RX: normal working mode
122 kni_net_rx_normal(struct kni_dev *kni)
126 unsigned i, num_rx, num_fq;
127 struct rte_kni_mbuf *kva;
128 struct rte_kni_mbuf *va[MBUF_BURST_SZ];
132 struct net_device *dev = kni->net_dev;
134 /* Get the number of free entries in free_q */
135 num_fq = kni_fifo_free_count(kni->free_q);
137 /* No room on the free_q, bail out */
141 /* Calculate the number of entries to dequeue from rx_q */
142 num_rx = min(num_fq, (unsigned)MBUF_BURST_SZ);
144 /* Burst dequeue from rx_q */
145 num_rx = kni_fifo_get(kni->rx_q, (void **)va, num_rx);
149 /* Transfer received packets to netif */
150 for (i = 0; i < num_rx; i++) {
151 kva = (void *)va[i] - kni->mbuf_va + kni->mbuf_kva;
154 data_kva = kva->buf_addr + kva->data_off - kni->mbuf_va
157 skb = dev_alloc_skb(len + 2);
159 KNI_ERR("Out of mem, dropping pkts\n");
160 /* Update statistics */
161 kni->stats.rx_dropped++;
165 /* Align IP on 16B boundary */
168 if (kva->nb_segs == 1) {
169 memcpy(skb_put(skb, len), data_kva, len);
172 int kva_nb_segs = kva->nb_segs;
174 for (nb_segs = 0; nb_segs < kva_nb_segs; nb_segs++) {
175 memcpy(skb_put(skb, kva->data_len),
176 data_kva, kva->data_len);
181 kva = kva->next - kni->mbuf_va + kni->mbuf_kva;
182 data_kva = kva->buf_addr + kva->data_off
183 - kni->mbuf_va + kni->mbuf_kva;
188 skb->protocol = eth_type_trans(skb, dev);
189 skb->ip_summed = CHECKSUM_UNNECESSARY;
191 /* Call netif interface */
194 /* Update statistics */
195 kni->stats.rx_bytes += len;
196 kni->stats.rx_packets++;
199 /* Burst enqueue mbufs into free_q */
200 ret = kni_fifo_put(kni->free_q, (void **)va, num_rx);
202 /* Failing should not happen */
203 KNI_ERR("Fail to enqueue entries into free_q\n");
207 * RX: loopback with enqueue/dequeue fifos.
210 kni_net_rx_lo_fifo(struct kni_dev *kni)
214 unsigned i, num, num_rq, num_tq, num_aq, num_fq;
215 struct rte_kni_mbuf *kva;
216 struct rte_kni_mbuf *va[MBUF_BURST_SZ];
219 struct rte_kni_mbuf *alloc_kva;
220 struct rte_kni_mbuf *alloc_va[MBUF_BURST_SZ];
221 void *alloc_data_kva;
223 /* Get the number of entries in rx_q */
224 num_rq = kni_fifo_count(kni->rx_q);
226 /* Get the number of free entrie in tx_q */
227 num_tq = kni_fifo_free_count(kni->tx_q);
229 /* Get the number of entries in alloc_q */
230 num_aq = kni_fifo_count(kni->alloc_q);
232 /* Get the number of free entries in free_q */
233 num_fq = kni_fifo_free_count(kni->free_q);
235 /* Calculate the number of entries to be dequeued from rx_q */
236 num = min(num_rq, num_tq);
237 num = min(num, num_aq);
238 num = min(num, num_fq);
239 num = min(num, (unsigned)MBUF_BURST_SZ);
241 /* Return if no entry to dequeue from rx_q */
245 /* Burst dequeue from rx_q */
246 ret = kni_fifo_get(kni->rx_q, (void **)va, num);
248 return; /* Failing should not happen */
250 /* Dequeue entries from alloc_q */
251 ret = kni_fifo_get(kni->alloc_q, (void **)alloc_va, num);
255 for (i = 0; i < num; i++) {
256 kva = (void *)va[i] - kni->mbuf_va + kni->mbuf_kva;
258 data_kva = kva->buf_addr + kva->data_off -
259 kni->mbuf_va + kni->mbuf_kva;
261 alloc_kva = (void *)alloc_va[i] - kni->mbuf_va +
263 alloc_data_kva = alloc_kva->buf_addr +
264 alloc_kva->data_off - kni->mbuf_va +
266 memcpy(alloc_data_kva, data_kva, len);
267 alloc_kva->pkt_len = len;
268 alloc_kva->data_len = len;
270 kni->stats.tx_bytes += len;
271 kni->stats.rx_bytes += len;
274 /* Burst enqueue mbufs into tx_q */
275 ret = kni_fifo_put(kni->tx_q, (void **)alloc_va, num);
277 /* Failing should not happen */
278 KNI_ERR("Fail to enqueue mbufs into tx_q\n");
281 /* Burst enqueue mbufs into free_q */
282 ret = kni_fifo_put(kni->free_q, (void **)va, num);
284 /* Failing should not happen */
285 KNI_ERR("Fail to enqueue mbufs into free_q\n");
288 * Update statistic, and enqueue/dequeue failure is impossible,
289 * as all queues are checked at first.
291 kni->stats.tx_packets += num;
292 kni->stats.rx_packets += num;
296 * RX: loopback with enqueue/dequeue fifos and sk buffer copies.
299 kni_net_rx_lo_fifo_skb(struct kni_dev *kni)
303 unsigned i, num_rq, num_fq, num;
304 struct rte_kni_mbuf *kva;
305 struct rte_kni_mbuf *va[MBUF_BURST_SZ];
309 struct net_device *dev = kni->net_dev;
311 /* Get the number of entries in rx_q */
312 num_rq = kni_fifo_count(kni->rx_q);
314 /* Get the number of free entries in free_q */
315 num_fq = kni_fifo_free_count(kni->free_q);
317 /* Calculate the number of entries to dequeue from rx_q */
318 num = min(num_rq, num_fq);
319 num = min(num, (unsigned)MBUF_BURST_SZ);
321 /* Return if no entry to dequeue from rx_q */
325 /* Burst dequeue mbufs from rx_q */
326 ret = kni_fifo_get(kni->rx_q, (void **)va, num);
330 /* Copy mbufs to sk buffer and then call tx interface */
331 for (i = 0; i < num; i++) {
332 kva = (void *)va[i] - kni->mbuf_va + kni->mbuf_kva;
334 data_kva = kva->buf_addr + kva->data_off - kni->mbuf_va +
337 skb = dev_alloc_skb(len + 2);
339 KNI_ERR("Out of mem, dropping pkts\n");
341 /* Align IP on 16B boundary */
343 memcpy(skb_put(skb, len), data_kva, len);
345 skb->ip_summed = CHECKSUM_UNNECESSARY;
349 /* Simulate real usage, allocate/copy skb twice */
350 skb = dev_alloc_skb(len + 2);
352 KNI_ERR("Out of mem, dropping pkts\n");
353 kni->stats.rx_dropped++;
357 /* Align IP on 16B boundary */
360 if (kva->nb_segs == 1) {
361 memcpy(skb_put(skb, len), data_kva, len);
364 int kva_nb_segs = kva->nb_segs;
366 for (nb_segs = 0; nb_segs < kva_nb_segs; nb_segs++) {
367 memcpy(skb_put(skb, kva->data_len),
368 data_kva, kva->data_len);
373 kva = kva->next - kni->mbuf_va + kni->mbuf_kva;
374 data_kva = kva->buf_addr + kva->data_off
375 - kni->mbuf_va + kni->mbuf_kva;
380 skb->ip_summed = CHECKSUM_UNNECESSARY;
382 kni->stats.rx_bytes += len;
383 kni->stats.rx_packets++;
385 /* call tx interface */
386 kni_net_tx(skb, dev);
389 /* enqueue all the mbufs from rx_q into free_q */
390 ret = kni_fifo_put(kni->free_q, (void **)&va, num);
392 /* Failing should not happen */
393 KNI_ERR("Fail to enqueue mbufs into free_q\n");
398 kni_net_rx(struct kni_dev *kni)
401 * It doesn't need to check if it is NULL pointer,
402 * as it has a default value
404 (*kni_net_rx_func)(kni);
408 * Transmit a packet (called by the kernel)
412 kni_net_tx(struct sk_buff *skb, struct net_device *dev)
414 struct kni_dev *kni = netdev_priv(dev);
417 kni->stats.tx_dropped++;
423 kni_net_tx(struct sk_buff *skb, struct net_device *dev)
427 struct kni_dev *kni = netdev_priv(dev);
428 struct rte_kni_mbuf *pkt_kva = NULL;
429 struct rte_kni_mbuf *pkt_va = NULL;
431 /* save the timestamp */
432 #ifdef HAVE_TRANS_START_HELPER
433 netif_trans_update(dev);
435 dev->trans_start = jiffies;
438 /* Check if the length of skb is less than mbuf size */
439 if (skb->len > kni->mbuf_size)
443 * Check if it has at least one free entry in tx_q and
444 * one entry in alloc_q.
446 if (kni_fifo_free_count(kni->tx_q) == 0 ||
447 kni_fifo_count(kni->alloc_q) == 0) {
449 * If no free entry in tx_q or no entry in alloc_q,
450 * drops skb and goes out.
455 /* dequeue a mbuf from alloc_q */
456 ret = kni_fifo_get(kni->alloc_q, (void **)&pkt_va, 1);
457 if (likely(ret == 1)) {
460 pkt_kva = (void *)pkt_va - kni->mbuf_va + kni->mbuf_kva;
461 data_kva = pkt_kva->buf_addr + pkt_kva->data_off - kni->mbuf_va
465 memcpy(data_kva, skb->data, len);
466 if (unlikely(len < ETH_ZLEN)) {
467 memset(data_kva + len, 0, ETH_ZLEN - len);
470 pkt_kva->pkt_len = len;
471 pkt_kva->data_len = len;
473 /* enqueue mbuf into tx_q */
474 ret = kni_fifo_put(kni->tx_q, (void **)&pkt_va, 1);
475 if (unlikely(ret != 1)) {
476 /* Failing should not happen */
477 KNI_ERR("Fail to enqueue mbuf into tx_q\n");
481 /* Failing should not happen */
482 KNI_ERR("Fail to dequeue mbuf from alloc_q\n");
486 /* Free skb and update statistics */
488 kni->stats.tx_bytes += len;
489 kni->stats.tx_packets++;
494 /* Free skb and update statistics */
496 kni->stats.tx_dropped++;
503 * Deal with a transmit timeout.
506 kni_net_tx_timeout (struct net_device *dev)
508 struct kni_dev *kni = netdev_priv(dev);
510 KNI_DBG("Transmit timeout at %ld, latency %ld\n", jiffies,
511 jiffies - dev->trans_start);
513 kni->stats.tx_errors++;
514 netif_wake_queue(dev);
522 kni_net_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
524 KNI_DBG("kni_net_ioctl %d\n",
525 ((struct kni_dev *)netdev_priv(dev))->group_id);
531 kni_net_set_rx_mode(struct net_device *dev)
536 kni_net_change_mtu(struct net_device *dev, int new_mtu)
539 struct rte_kni_request req;
540 struct kni_dev *kni = netdev_priv(dev);
542 KNI_DBG("kni_net_change_mtu new mtu %d to be set\n", new_mtu);
544 memset(&req, 0, sizeof(req));
545 req.req_id = RTE_KNI_REQ_CHANGE_MTU;
546 req.new_mtu = new_mtu;
547 ret = kni_net_process_request(kni, &req);
548 if (ret == 0 && req.result == 0)
551 return (ret == 0) ? req.result : ret;
555 * Checks if the user space application provided the resp message
558 kni_net_poll_resp(struct kni_dev *kni)
560 if (kni_fifo_count(kni->resp_q))
561 wake_up_interruptible(&kni->wq);
565 * It can be called to process the request.
568 kni_net_process_request(struct kni_dev *kni, struct rte_kni_request *req)
576 KNI_ERR("No kni instance or request\n");
580 mutex_lock(&kni->sync_lock);
583 memcpy(kni->sync_kva, req, sizeof(struct rte_kni_request));
584 num = kni_fifo_put(kni->req_q, &kni->sync_va, 1);
586 KNI_ERR("Cannot send to req_q\n");
591 ret_val = wait_event_interruptible_timeout(kni->wq,
592 kni_fifo_count(kni->resp_q), 3 * HZ);
593 if (signal_pending(current) || ret_val <= 0) {
597 num = kni_fifo_get(kni->resp_q, (void **)&resp_va, 1);
598 if (num != 1 || resp_va != kni->sync_va) {
599 /* This should never happen */
600 KNI_ERR("No data in resp_q\n");
605 memcpy(req, kni->sync_kva, sizeof(struct rte_kni_request));
609 mutex_unlock(&kni->sync_lock);
614 * Return statistics to the caller
616 static struct net_device_stats *
617 kni_net_stats(struct net_device *dev)
619 struct kni_dev *kni = netdev_priv(dev);
624 * Fill the eth header
627 kni_net_header(struct sk_buff *skb, struct net_device *dev,
628 unsigned short type, const void *daddr,
629 const void *saddr, unsigned int len)
631 struct ethhdr *eth = (struct ethhdr *) skb_push(skb, ETH_HLEN);
633 memcpy(eth->h_source, saddr ? saddr : dev->dev_addr, dev->addr_len);
634 memcpy(eth->h_dest, daddr ? daddr : dev->dev_addr, dev->addr_len);
635 eth->h_proto = htons(type);
637 return dev->hard_header_len;
642 * Re-fill the eth header
644 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0))
646 kni_net_rebuild_header(struct sk_buff *skb)
648 struct net_device *dev = skb->dev;
649 struct ethhdr *eth = (struct ethhdr *) skb->data;
651 memcpy(eth->h_source, dev->dev_addr, dev->addr_len);
652 memcpy(eth->h_dest, dev->dev_addr, dev->addr_len);
659 * kni_net_set_mac - Change the Ethernet Address of the KNI NIC
660 * @netdev: network interface device structure
661 * @p: pointer to an address structure
663 * Returns 0 on success, negative on failure
665 static int kni_net_set_mac(struct net_device *netdev, void *p)
667 struct sockaddr *addr = p;
668 if (!is_valid_ether_addr((unsigned char *)(addr->sa_data)))
669 return -EADDRNOTAVAIL;
670 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
674 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
675 static int kni_net_change_carrier(struct net_device *dev, bool new_carrier)
678 netif_carrier_on(dev);
680 netif_carrier_off(dev);
685 static const struct header_ops kni_net_header_ops = {
686 .create = kni_net_header,
687 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 1, 0))
688 .rebuild = kni_net_rebuild_header,
690 .cache = NULL, /* disable caching */
693 static const struct net_device_ops kni_net_netdev_ops = {
694 .ndo_open = kni_net_open,
695 .ndo_stop = kni_net_release,
696 .ndo_set_config = kni_net_config,
697 .ndo_start_xmit = kni_net_tx,
698 .ndo_change_mtu = kni_net_change_mtu,
699 .ndo_do_ioctl = kni_net_ioctl,
700 .ndo_set_rx_mode = kni_net_set_rx_mode,
701 .ndo_get_stats = kni_net_stats,
702 .ndo_tx_timeout = kni_net_tx_timeout,
703 .ndo_set_mac_address = kni_net_set_mac,
704 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0))
705 .ndo_change_carrier = kni_net_change_carrier,
710 kni_net_init(struct net_device *dev)
712 struct kni_dev *kni = netdev_priv(dev);
714 KNI_DBG("kni_net_init\n");
716 init_waitqueue_head(&kni->wq);
717 mutex_init(&kni->sync_lock);
719 ether_setup(dev); /* assign some of the fields */
720 dev->netdev_ops = &kni_net_netdev_ops;
721 dev->header_ops = &kni_net_header_ops;
722 dev->watchdog_timeo = WD_TIMEOUT;
726 kni_net_config_lo_mode(char *lo_str)
729 KNI_PRINT("loopback disabled");
733 if (!strcmp(lo_str, "lo_mode_none"))
734 KNI_PRINT("loopback disabled");
735 else if (!strcmp(lo_str, "lo_mode_fifo")) {
736 KNI_PRINT("loopback mode=lo_mode_fifo enabled");
737 kni_net_rx_func = kni_net_rx_lo_fifo;
738 } else if (!strcmp(lo_str, "lo_mode_fifo_skb")) {
739 KNI_PRINT("loopback mode=lo_mode_fifo_skb enabled");
740 kni_net_rx_func = kni_net_rx_lo_fifo_skb;
742 KNI_PRINT("Incognizant parameter, loopback disabled");