4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #include <rte_malloc.h>
36 #include <rte_ethdev.h>
40 #include <rte_devargs.h>
41 #include <rte_kvargs.h>
43 #include <rte_alarm.h>
45 #include "rte_eth_bond.h"
46 #include "rte_eth_bond_private.h"
49 bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
51 struct bond_dev_private *internals;
53 uint16_t num_rx_slave = 0;
54 uint16_t num_rx_total = 0;
58 /* Cast to structure, containing bonded device's port id and queue id */
59 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
61 internals = bd_rx_q->dev_private;
64 for (i = 0; i < internals->active_slave_count && nb_pkts; i++) {
65 /* Offset of pointer to *bufs increases as packets are received
66 * from other slaves */
67 num_rx_slave = rte_eth_rx_burst(internals->active_slaves[i],
68 bd_rx_q->queue_id, bufs + num_rx_total, nb_pkts);
70 num_rx_total += num_rx_slave;
71 nb_pkts -= num_rx_slave;
79 bond_ethdev_rx_burst_active_backup(void *queue, struct rte_mbuf **bufs,
82 struct bond_dev_private *internals;
84 /* Cast to structure, containing bonded device's port id and queue id */
85 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
87 internals = bd_rx_q->dev_private;
89 return rte_eth_rx_burst(internals->current_primary_port,
90 bd_rx_q->queue_id, bufs, nb_pkts);
94 bond_ethdev_tx_burst_round_robin(void *queue, struct rte_mbuf **bufs,
97 struct bond_dev_private *internals;
98 struct bond_tx_queue *bd_tx_q;
100 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts];
101 uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
103 uint8_t num_of_slaves;
104 uint8_t slaves[RTE_MAX_ETHPORTS];
106 uint16_t num_tx_total = 0, num_tx_slave;
108 static int slave_idx = 0;
109 int i, cslave_idx = 0, tx_fail_total = 0;
111 bd_tx_q = (struct bond_tx_queue *)queue;
112 internals = bd_tx_q->dev_private;
114 /* Copy slave list to protect against slave up/down changes during tx
116 num_of_slaves = internals->active_slave_count;
117 memcpy(slaves, internals->active_slaves,
118 sizeof(internals->active_slaves[0]) * num_of_slaves);
120 if (num_of_slaves < 1)
123 /* Populate slaves mbuf with which packets are to be sent on it */
124 for (i = 0; i < nb_pkts; i++) {
125 cslave_idx = (slave_idx + i) % num_of_slaves;
126 slave_bufs[cslave_idx][(slave_nb_pkts[cslave_idx])++] = bufs[i];
129 /* increment current slave index so the next call to tx burst starts on the
131 slave_idx = ++cslave_idx;
133 /* Send packet burst on each slave device */
134 for (i = 0; i < num_of_slaves; i++) {
135 if (slave_nb_pkts[i] > 0) {
136 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
137 slave_bufs[i], slave_nb_pkts[i]);
139 /* if tx burst fails move packets to end of bufs */
140 if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
141 int tx_fail_slave = slave_nb_pkts[i] - num_tx_slave;
143 tx_fail_total += tx_fail_slave;
145 memcpy(&bufs[nb_pkts - tx_fail_total],
146 &slave_bufs[i][num_tx_slave], tx_fail_slave * sizeof(bufs[0]));
148 num_tx_total += num_tx_slave;
156 bond_ethdev_tx_burst_active_backup(void *queue,
157 struct rte_mbuf **bufs, uint16_t nb_pkts)
159 struct bond_dev_private *internals;
160 struct bond_tx_queue *bd_tx_q;
162 bd_tx_q = (struct bond_tx_queue *)queue;
163 internals = bd_tx_q->dev_private;
165 if (internals->active_slave_count < 1)
168 return rte_eth_tx_burst(internals->current_primary_port, bd_tx_q->queue_id,
172 static inline uint16_t
173 ether_hash(struct ether_hdr *eth_hdr)
175 uint16_t *word_src_addr = (uint16_t *)eth_hdr->s_addr.addr_bytes;
176 uint16_t *word_dst_addr = (uint16_t *)eth_hdr->d_addr.addr_bytes;
178 return (word_src_addr[0] ^ word_dst_addr[0]) ^
179 (word_src_addr[1] ^ word_dst_addr[1]) ^
180 (word_src_addr[2] ^ word_dst_addr[2]);
183 static inline uint32_t
184 ipv4_hash(struct ipv4_hdr *ipv4_hdr)
186 return (ipv4_hdr->src_addr ^ ipv4_hdr->dst_addr);
189 static inline uint32_t
190 ipv6_hash(struct ipv6_hdr *ipv6_hdr)
192 uint32_t *word_src_addr = (uint32_t *)&(ipv6_hdr->src_addr[0]);
193 uint32_t *word_dst_addr = (uint32_t *)&(ipv6_hdr->dst_addr[0]);
195 return (word_src_addr[0] ^ word_dst_addr[0]) ^
196 (word_src_addr[1] ^ word_dst_addr[1]) ^
197 (word_src_addr[2] ^ word_dst_addr[2]) ^
198 (word_src_addr[3] ^ word_dst_addr[3]);
202 udp_hash(struct udp_hdr *hdr)
204 return hdr->src_port ^ hdr->dst_port;
207 static inline uint16_t
208 xmit_slave_hash(const struct rte_mbuf *buf, uint8_t slave_count, uint8_t policy)
210 struct ether_hdr *eth_hdr;
211 struct udp_hdr *udp_hdr;
212 size_t eth_offset = 0;
215 if (slave_count == 1)
219 case BALANCE_XMIT_POLICY_LAYER2:
220 eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *);
222 hash = ether_hash(eth_hdr);
224 return hash % slave_count;
226 case BALANCE_XMIT_POLICY_LAYER23:
227 eth_hdr = rte_pktmbuf_mtod(buf, struct ether_hdr *);
229 if (buf->ol_flags & PKT_RX_VLAN_PKT)
230 eth_offset = sizeof(struct ether_hdr) + sizeof(struct vlan_hdr);
232 eth_offset = sizeof(struct ether_hdr);
234 if (buf->ol_flags & PKT_RX_IPV4_HDR) {
235 struct ipv4_hdr *ipv4_hdr;
236 ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(buf,
237 unsigned char *) + eth_offset);
239 hash = ether_hash(eth_hdr) ^ ipv4_hash(ipv4_hdr);
242 struct ipv6_hdr *ipv6_hdr;
244 ipv6_hdr = (struct ipv6_hdr *)(rte_pktmbuf_mtod(buf,
245 unsigned char *) + eth_offset);
247 hash = ether_hash(eth_hdr) ^ ipv6_hash(ipv6_hdr);
251 case BALANCE_XMIT_POLICY_LAYER34:
252 if (buf->ol_flags & PKT_RX_VLAN_PKT)
253 eth_offset = sizeof(struct ether_hdr) + sizeof(struct vlan_hdr);
255 eth_offset = sizeof(struct ether_hdr);
257 if (buf->ol_flags & PKT_RX_IPV4_HDR) {
258 struct ipv4_hdr *ipv4_hdr = (struct ipv4_hdr *)
259 (rte_pktmbuf_mtod(buf, unsigned char *) + eth_offset);
261 if (ipv4_hdr->next_proto_id == IPPROTO_UDP) {
262 udp_hdr = (struct udp_hdr *)
263 (rte_pktmbuf_mtod(buf, unsigned char *) + eth_offset +
264 sizeof(struct ipv4_hdr));
265 hash = ipv4_hash(ipv4_hdr) ^ udp_hash(udp_hdr);
267 hash = ipv4_hash(ipv4_hdr);
270 struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *)
271 (rte_pktmbuf_mtod(buf, unsigned char *) + eth_offset);
273 if (ipv6_hdr->proto == IPPROTO_UDP) {
274 udp_hdr = (struct udp_hdr *)
275 (rte_pktmbuf_mtod(buf, unsigned char *) + eth_offset +
276 sizeof(struct ipv6_hdr));
277 hash = ipv6_hash(ipv6_hdr) ^ udp_hash(udp_hdr);
279 hash = ipv6_hash(ipv6_hdr);
288 return hash % slave_count;
292 bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,
295 struct bond_dev_private *internals;
296 struct bond_tx_queue *bd_tx_q;
298 uint8_t num_of_slaves;
299 uint8_t slaves[RTE_MAX_ETHPORTS];
301 uint16_t num_tx_total = 0, num_tx_slave = 0, tx_fail_total = 0;
305 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts];
306 uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
308 bd_tx_q = (struct bond_tx_queue *)queue;
309 internals = bd_tx_q->dev_private;
311 /* Copy slave list to protect against slave up/down changes during tx
313 num_of_slaves = internals->active_slave_count;
314 memcpy(slaves, internals->active_slaves,
315 sizeof(internals->active_slaves[0]) * num_of_slaves);
317 if (num_of_slaves < 1)
320 /* Populate slaves mbuf with the packets which are to be sent on it */
321 for (i = 0; i < nb_pkts; i++) {
322 /* Select output slave using hash based on xmit policy */
323 op_slave_id = xmit_slave_hash(bufs[i], num_of_slaves,
324 internals->balance_xmit_policy);
326 /* Populate slave mbuf arrays with mbufs for that slave */
327 slave_bufs[op_slave_id][slave_nb_pkts[op_slave_id]++] = bufs[i];
330 /* Send packet burst on each slave device */
331 for (i = 0; i < num_of_slaves; i++) {
332 if (slave_nb_pkts[i] > 0) {
333 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
334 slave_bufs[i], slave_nb_pkts[i]);
336 /* if tx burst fails move packets to end of bufs */
337 if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
338 int slave_tx_fail_count = slave_nb_pkts[i] - num_tx_slave;
340 tx_fail_total += slave_tx_fail_count;
341 memcpy(bufs[nb_pkts - tx_fail_total],
342 slave_bufs[i][num_tx_slave], slave_tx_fail_count);
345 num_tx_total += num_tx_slave;
353 #ifdef RTE_MBUF_REFCNT
355 bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs,
358 struct bond_dev_private *internals;
359 struct bond_tx_queue *bd_tx_q;
361 uint8_t tx_failed_flag = 0, num_of_slaves;
362 uint8_t slaves[RTE_MAX_ETHPORTS];
364 uint16_t max_nb_of_tx_pkts = 0;
366 int slave_tx_total[RTE_MAX_ETHPORTS];
367 int i, most_successful_tx_slave = -1;
369 bd_tx_q = (struct bond_tx_queue *)queue;
370 internals = bd_tx_q->dev_private;
372 /* Copy slave list to protect against slave up/down changes during tx
374 num_of_slaves = internals->active_slave_count;
375 memcpy(slaves, internals->active_slaves,
376 sizeof(internals->active_slaves[0]) * num_of_slaves);
378 if (num_of_slaves < 1)
381 /* Increment reference count on mbufs */
382 for (i = 0; i < nb_pkts; i++)
383 rte_mbuf_refcnt_update(bufs[i], num_of_slaves - 1);
385 /* Transmit burst on each active slave */
386 for (i = 0; i < num_of_slaves; i++) {
387 slave_tx_total[i] = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
390 if (unlikely(slave_tx_total[i] < nb_pkts))
393 /* record the value and slave index for the slave which transmits the
394 * maximum number of packets */
395 if (slave_tx_total[i] > max_nb_of_tx_pkts) {
396 max_nb_of_tx_pkts = slave_tx_total[i];
397 most_successful_tx_slave = i;
401 /* if slaves fail to transmit packets from burst, the calling application
402 * is not expected to know about multiple references to packets so we must
403 * handle failures of all packets except those of the most successful slave
405 if (unlikely(tx_failed_flag))
406 for (i = 0; i < num_of_slaves; i++)
407 if (i != most_successful_tx_slave)
408 while (slave_tx_total[i] < nb_pkts)
409 rte_pktmbuf_free(bufs[slave_tx_total[i]++]);
411 return max_nb_of_tx_pkts;
416 link_properties_set(struct rte_eth_dev *bonded_eth_dev,
417 struct rte_eth_link *slave_dev_link)
419 struct rte_eth_link *bonded_dev_link = &bonded_eth_dev->data->dev_link;
420 struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
422 if (slave_dev_link->link_status &&
423 bonded_eth_dev->data->dev_started) {
424 bonded_dev_link->link_duplex = slave_dev_link->link_duplex;
425 bonded_dev_link->link_speed = slave_dev_link->link_speed;
427 internals->link_props_set = 1;
432 link_properties_reset(struct rte_eth_dev *bonded_eth_dev)
434 struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
436 memset(&(bonded_eth_dev->data->dev_link), 0,
437 sizeof(bonded_eth_dev->data->dev_link));
439 internals->link_props_set = 0;
443 link_properties_valid(struct rte_eth_link *bonded_dev_link,
444 struct rte_eth_link *slave_dev_link)
446 if (bonded_dev_link->link_duplex != slave_dev_link->link_duplex ||
447 bonded_dev_link->link_speed != slave_dev_link->link_speed)
454 mac_address_set(struct rte_eth_dev *eth_dev, struct ether_addr *new_mac_addr)
456 struct ether_addr *mac_addr;
458 mac_addr = eth_dev->data->mac_addrs;
460 if (eth_dev == NULL) {
461 RTE_BOND_LOG(ERR, "NULL pointer eth_dev specified");
465 if (new_mac_addr == NULL) {
466 RTE_BOND_LOG(ERR, "NULL pointer MAC specified");
470 /* If new MAC is different to current MAC then update */
471 if (memcmp(mac_addr, new_mac_addr, sizeof(*mac_addr)) != 0)
472 memcpy(mac_addr, new_mac_addr, sizeof(*mac_addr));
478 mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev)
480 struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
483 /* Update slave devices MAC addresses */
484 if (internals->slave_count < 1)
487 switch (internals->mode) {
488 case BONDING_MODE_ROUND_ROBIN:
489 case BONDING_MODE_BALANCE:
490 #ifdef RTE_MBUF_REFCNT
491 case BONDING_MODE_BROADCAST:
493 for (i = 0; i < internals->slave_count; i++) {
494 if (mac_address_set(&rte_eth_devices[internals->slaves[i].port_id],
495 bonded_eth_dev->data->mac_addrs)) {
496 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
497 internals->slaves[i].port_id);
502 case BONDING_MODE_ACTIVE_BACKUP:
504 for (i = 0; i < internals->slave_count; i++) {
505 if (internals->slaves[i].port_id ==
506 internals->current_primary_port) {
507 if (mac_address_set(&rte_eth_devices[internals->primary_port],
508 bonded_eth_dev->data->mac_addrs)) {
509 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
510 internals->current_primary_port);
515 &rte_eth_devices[internals->slaves[i].port_id],
516 &internals->slaves[i].persisted_mac_addr)) {
517 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
518 internals->slaves[i].port_id);
529 bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode)
531 struct bond_dev_private *internals;
533 internals = eth_dev->data->dev_private;
536 case BONDING_MODE_ROUND_ROBIN:
537 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_round_robin;
538 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
540 case BONDING_MODE_ACTIVE_BACKUP:
541 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_active_backup;
542 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
544 case BONDING_MODE_BALANCE:
545 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_balance;
546 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
548 #ifdef RTE_MBUF_REFCNT
549 case BONDING_MODE_BROADCAST:
550 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_broadcast;
551 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
558 internals->mode = mode;
564 slave_configure(struct rte_eth_dev *bonded_eth_dev,
565 struct rte_eth_dev *slave_eth_dev)
567 struct bond_rx_queue *bd_rx_q;
568 struct bond_tx_queue *bd_tx_q;
573 rte_eth_dev_stop(slave_eth_dev->data->port_id);
575 /* Enable interrupts on slave device if supported */
576 if (slave_eth_dev->driver->pci_drv.drv_flags & RTE_PCI_DRV_INTR_LSC)
577 slave_eth_dev->data->dev_conf.intr_conf.lsc = 1;
579 /* Configure device */
580 errval = rte_eth_dev_configure(slave_eth_dev->data->port_id,
581 bonded_eth_dev->data->nb_rx_queues,
582 bonded_eth_dev->data->nb_tx_queues,
583 &(slave_eth_dev->data->dev_conf));
585 RTE_BOND_LOG(ERR, "Cannot configure slave device: port %u , err (%d)",
586 slave_eth_dev->data->port_id, errval);
590 /* Setup Rx Queues */
591 for (q_id = 0; q_id < bonded_eth_dev->data->nb_rx_queues; q_id++) {
592 bd_rx_q = (struct bond_rx_queue *)bonded_eth_dev->data->rx_queues[q_id];
594 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, q_id,
596 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
597 &(bd_rx_q->rx_conf), bd_rx_q->mb_pool);
600 "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)",
601 slave_eth_dev->data->port_id, q_id, errval);
606 /* Setup Tx Queues */
607 for (q_id = 0; q_id < bonded_eth_dev->data->nb_tx_queues; q_id++) {
608 bd_tx_q = (struct bond_tx_queue *)bonded_eth_dev->data->tx_queues[q_id];
610 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, q_id,
612 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
616 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)",
617 slave_eth_dev->data->port_id, q_id, errval);
623 errval = rte_eth_dev_start(slave_eth_dev->data->port_id);
625 RTE_BOND_LOG(ERR, "rte_eth_dev_start: port=%u, err (%d)",
626 slave_eth_dev->data->port_id, errval);
634 slave_remove(struct bond_dev_private *internals,
635 struct rte_eth_dev *slave_eth_dev)
639 for (i = 0; i < internals->slave_count; i++) {
640 if (internals->slaves[i].port_id == slave_eth_dev->data->port_id)
643 if (found && i < (internals->slave_count - 1))
644 memcpy(&internals->slaves[i], &internals->slaves[i+1],
645 sizeof(internals->slaves[i]));
648 internals->slave_count--;
652 bond_ethdev_slave_link_status_change_monitor(void *cb_arg);
655 slave_add(struct bond_dev_private *internals,
656 struct rte_eth_dev *slave_eth_dev)
658 struct bond_slave_details *slave_details =
659 &internals->slaves[internals->slave_count];
661 slave_details->port_id = slave_eth_dev->data->port_id;
662 slave_details->last_link_status = 0;
664 /* If slave device doesn't support interrupts then we need to enabled
665 * polling to monitor link status */
666 if (!(slave_eth_dev->pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC)) {
667 slave_details->link_status_poll_enabled = 1;
669 if (!internals->link_status_polling_enabled) {
670 internals->link_status_polling_enabled = 1;
672 rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000,
673 bond_ethdev_slave_link_status_change_monitor,
674 (void *)&rte_eth_devices[internals->port_id]);
678 slave_details->link_status_wait_to_complete = 0;
680 memcpy(&(slave_details->persisted_mac_addr), slave_eth_dev->data->mac_addrs,
681 sizeof(struct ether_addr));
685 bond_ethdev_primary_set(struct bond_dev_private *internals,
686 uint8_t slave_port_id)
690 if (internals->active_slave_count < 1)
691 internals->current_primary_port = slave_port_id;
693 /* Search bonded device slave ports for new proposed primary port */
694 for (i = 0; i < internals->active_slave_count; i++) {
695 if (internals->active_slaves[i] == slave_port_id)
696 internals->current_primary_port = slave_port_id;
701 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev);
704 bond_ethdev_start(struct rte_eth_dev *eth_dev)
706 struct bond_dev_private *internals;
709 /* slave eth dev will be started by bonded device */
710 if (valid_bonded_ethdev(eth_dev)) {
711 RTE_BOND_LOG(ERR, "User tried to explicitly start a slave eth_dev (%d)",
712 eth_dev->data->port_id);
716 eth_dev->data->dev_link.link_status = 0;
717 eth_dev->data->dev_started = 1;
719 internals = eth_dev->data->dev_private;
721 if (internals->slave_count == 0) {
722 RTE_BOND_LOG(ERR, "Cannot start port since there are no slave devices");
726 if (internals->user_defined_mac == 0) {
727 struct ether_addr *new_mac_addr = NULL;
729 for (i = 0; i < internals->slave_count; i++)
730 if (internals->slaves[i].port_id == internals->primary_port)
731 new_mac_addr = &internals->slaves[i].persisted_mac_addr;
733 if (new_mac_addr == NULL)
736 if (mac_address_set(eth_dev, new_mac_addr) != 0) {
737 RTE_BOND_LOG(ERR, "bonded port (%d) failed to update MAC address",
738 eth_dev->data->port_id);
743 /* Update all slave devices MACs*/
744 if (mac_address_slaves_update(eth_dev) != 0)
747 /* If bonded device is configure in promiscuous mode then re-apply config */
748 if (internals->promiscuous_en)
749 bond_ethdev_promiscuous_enable(eth_dev);
751 /* Reconfigure each slave device if starting bonded device */
752 for (i = 0; i < internals->slave_count; i++) {
753 if (slave_configure(eth_dev,
754 &(rte_eth_devices[internals->slaves[i].port_id])) != 0) {
756 "bonded port (%d) failed to reconfigure slave device (%d)",
757 eth_dev->data->port_id, internals->slaves[i].port_id);
762 if (internals->user_defined_primary_port)
763 bond_ethdev_primary_set(internals, internals->primary_port);
769 bond_ethdev_stop(struct rte_eth_dev *eth_dev)
771 struct bond_dev_private *internals = eth_dev->data->dev_private;
773 internals->active_slave_count = 0;
774 internals->link_status_polling_enabled = 0;
776 eth_dev->data->dev_link.link_status = 0;
777 eth_dev->data->dev_started = 0;
781 bond_ethdev_close(struct rte_eth_dev *dev __rte_unused)
785 /* forward declaration */
786 static int bond_ethdev_configure(struct rte_eth_dev *dev);
789 bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
791 dev_info->driver_name = driver_name;
792 dev_info->max_mac_addrs = 1;
794 dev_info->max_rx_pktlen = (uint32_t)2048;
796 dev_info->max_rx_queues = (uint16_t)128;
797 dev_info->max_tx_queues = (uint16_t)512;
799 dev_info->min_rx_bufsize = 0;
800 dev_info->pci_dev = dev->pci_dev;
804 bond_ethdev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
805 uint16_t nb_rx_desc, unsigned int socket_id __rte_unused,
806 const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool)
808 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)
809 rte_zmalloc_socket(NULL, sizeof(struct bond_rx_queue),
810 0, dev->pci_dev->numa_node);
814 bd_rx_q->queue_id = rx_queue_id;
815 bd_rx_q->dev_private = dev->data->dev_private;
817 bd_rx_q->nb_rx_desc = nb_rx_desc;
819 memcpy(&(bd_rx_q->rx_conf), rx_conf, sizeof(struct rte_eth_rxconf));
820 bd_rx_q->mb_pool = mb_pool;
822 dev->data->rx_queues[rx_queue_id] = bd_rx_q;
828 bond_ethdev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
829 uint16_t nb_tx_desc, unsigned int socket_id __rte_unused,
830 const struct rte_eth_txconf *tx_conf)
832 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)
833 rte_zmalloc_socket(NULL, sizeof(struct bond_tx_queue),
834 0, dev->pci_dev->numa_node);
839 bd_tx_q->queue_id = tx_queue_id;
840 bd_tx_q->dev_private = dev->data->dev_private;
842 bd_tx_q->nb_tx_desc = nb_tx_desc;
843 memcpy(&(bd_tx_q->tx_conf), tx_conf, sizeof(bd_tx_q->tx_conf));
845 dev->data->tx_queues[tx_queue_id] = bd_tx_q;
851 bond_ethdev_rx_queue_release(void *queue)
860 bond_ethdev_tx_queue_release(void *queue)
870 bond_ethdev_slave_link_status_change_monitor(void *cb_arg)
872 struct rte_eth_dev *bonded_ethdev, *slave_ethdev;
873 struct bond_dev_private *internals;
875 /* Default value for polling slave found is true as we don't want to
876 * disable the polling thread if we cannot get the lock */
877 int i, polling_slave_found = 1;
882 bonded_ethdev = (struct rte_eth_dev *)cb_arg;
883 internals = (struct bond_dev_private *)bonded_ethdev->data->dev_private;
885 if (!bonded_ethdev->data->dev_started ||
886 !internals->link_status_polling_enabled)
889 /* If device is currently being configured then don't check slaves link
890 * status, wait until next period */
891 if (rte_spinlock_trylock(&internals->lock)) {
892 if (internals->slave_count > 0)
893 polling_slave_found = 0;
895 for (i = 0; i < internals->slave_count; i++) {
896 if (!internals->slaves[i].link_status_poll_enabled)
899 slave_ethdev = &rte_eth_devices[internals->slaves[i].port_id];
900 polling_slave_found = 1;
902 /* Update slave link status */
903 (*slave_ethdev->dev_ops->link_update)(slave_ethdev,
904 internals->slaves[i].link_status_wait_to_complete);
906 /* if link status has changed since last checked then call lsc
908 if (slave_ethdev->data->dev_link.link_status !=
909 internals->slaves[i].last_link_status) {
910 internals->slaves[i].last_link_status =
911 slave_ethdev->data->dev_link.link_status;
913 bond_ethdev_lsc_event_callback(internals->slaves[i].port_id,
914 RTE_ETH_EVENT_INTR_LSC,
915 &bonded_ethdev->data->port_id);
918 rte_spinlock_unlock(&internals->lock);
921 if (polling_slave_found)
922 /* Set alarm to continue monitoring link status of slave ethdev's */
923 rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000,
924 bond_ethdev_slave_link_status_change_monitor, cb_arg);
928 bond_ethdev_link_update(struct rte_eth_dev *bonded_eth_dev,
929 int wait_to_complete)
931 struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
933 if (!bonded_eth_dev->data->dev_started ||
934 internals->active_slave_count == 0) {
935 bonded_eth_dev->data->dev_link.link_status = 0;
938 struct rte_eth_dev *slave_eth_dev;
941 for (i = 0; i < internals->active_slave_count; i++) {
942 slave_eth_dev = &rte_eth_devices[internals->active_slaves[i]];
944 (*slave_eth_dev->dev_ops->link_update)(slave_eth_dev,
946 if (slave_eth_dev->data->dev_link.link_status == 1) {
952 bonded_eth_dev->data->dev_link.link_status = link_up;
959 bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
961 struct bond_dev_private *internals = dev->data->dev_private;
962 struct rte_eth_stats slave_stats;
966 /* clear bonded stats before populating from slaves */
967 memset(stats, 0, sizeof(*stats));
969 for (i = 0; i < internals->slave_count; i++) {
970 rte_eth_stats_get(internals->slaves[i].port_id, &slave_stats);
972 stats->ipackets += slave_stats.ipackets;
973 stats->opackets += slave_stats.opackets;
974 stats->ibytes += slave_stats.ibytes;
975 stats->obytes += slave_stats.obytes;
976 stats->ierrors += slave_stats.ierrors;
977 stats->oerrors += slave_stats.oerrors;
978 stats->imcasts += slave_stats.imcasts;
979 stats->rx_nombuf += slave_stats.rx_nombuf;
980 stats->fdirmatch += slave_stats.fdirmatch;
981 stats->fdirmiss += slave_stats.fdirmiss;
982 stats->tx_pause_xon += slave_stats.tx_pause_xon;
983 stats->rx_pause_xon += slave_stats.rx_pause_xon;
984 stats->tx_pause_xoff += slave_stats.tx_pause_xoff;
985 stats->rx_pause_xoff += slave_stats.rx_pause_xoff;
990 bond_ethdev_stats_reset(struct rte_eth_dev *dev)
992 struct bond_dev_private *internals = dev->data->dev_private;
995 for (i = 0; i < internals->slave_count; i++)
996 rte_eth_stats_reset(internals->slaves[i].port_id);
1000 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev)
1002 struct bond_dev_private *internals = eth_dev->data->dev_private;
1005 internals->promiscuous_en = 1;
1007 switch (internals->mode) {
1008 /* Promiscuous mode is propagated to all slaves */
1009 case BONDING_MODE_ROUND_ROBIN:
1010 case BONDING_MODE_BALANCE:
1011 #ifdef RTE_MBUF_REFCNT
1012 case BONDING_MODE_BROADCAST:
1014 for (i = 0; i < internals->slave_count; i++)
1015 rte_eth_promiscuous_enable(internals->slaves[i].port_id);
1017 /* Promiscuous mode is propagated only to primary slave */
1018 case BONDING_MODE_ACTIVE_BACKUP:
1020 rte_eth_promiscuous_enable(internals->current_primary_port);
1026 bond_ethdev_promiscuous_disable(struct rte_eth_dev *dev)
1028 struct bond_dev_private *internals = dev->data->dev_private;
1031 internals->promiscuous_en = 0;
1033 switch (internals->mode) {
1034 /* Promiscuous mode is propagated to all slaves */
1035 case BONDING_MODE_ROUND_ROBIN:
1036 case BONDING_MODE_BALANCE:
1037 #ifdef RTE_MBUF_REFCNT
1038 case BONDING_MODE_BROADCAST:
1040 for (i = 0; i < internals->slave_count; i++)
1041 rte_eth_promiscuous_disable(internals->slaves[i].port_id);
1043 /* Promiscuous mode is propagated only to primary slave */
1044 case BONDING_MODE_ACTIVE_BACKUP:
1046 rte_eth_promiscuous_disable(internals->current_primary_port);
1051 bond_ethdev_delayed_lsc_propagation(void *arg)
1056 _rte_eth_dev_callback_process((struct rte_eth_dev *)arg,
1057 RTE_ETH_EVENT_INTR_LSC);
1061 bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type,
1064 struct rte_eth_dev *bonded_eth_dev, *slave_eth_dev;
1065 struct bond_dev_private *internals;
1066 struct rte_eth_link link;
1068 int i, valid_slave = 0, active_pos = -1;
1069 uint8_t lsc_flag = 0;
1071 if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL)
1074 bonded_eth_dev = &rte_eth_devices[*(uint8_t *)param];
1075 slave_eth_dev = &rte_eth_devices[port_id];
1077 if (valid_bonded_ethdev(bonded_eth_dev))
1080 internals = bonded_eth_dev->data->dev_private;
1082 /* If the device isn't started don't handle interrupts */
1083 if (!bonded_eth_dev->data->dev_started)
1086 /* verify that port_id is a valid slave of bonded port */
1087 for (i = 0; i < internals->slave_count; i++) {
1088 if (internals->slaves[i].port_id == port_id) {
1097 /* Search for port in active port list */
1098 for (i = 0; i < internals->active_slave_count; i++) {
1099 if (port_id == internals->active_slaves[i]) {
1105 rte_eth_link_get_nowait(port_id, &link);
1106 if (link.link_status) {
1107 if (active_pos >= 0)
1110 /* if no active slave ports then set this port to be primary port */
1111 if (internals->active_slave_count < 1) {
1112 /* If first active slave, then change link status */
1113 bonded_eth_dev->data->dev_link.link_status = 1;
1114 internals->current_primary_port = port_id;
1117 /* Inherit eth dev link properties from first active slave */
1118 link_properties_set(bonded_eth_dev,
1119 &(slave_eth_dev->data->dev_link));
1121 internals->active_slaves[internals->active_slave_count++] = port_id;
1123 /* If user has defined the primary port then default to using it */
1124 if (internals->user_defined_primary_port &&
1125 internals->primary_port == port_id)
1126 bond_ethdev_primary_set(internals, port_id);
1131 /* Remove from active slave list */
1132 for (i = active_pos; i < (internals->active_slave_count - 1); i++)
1133 internals->active_slaves[i] = internals->active_slaves[i+1];
1135 internals->active_slave_count--;
1137 /* No active slaves, change link status to down and reset other
1138 * link properties */
1139 if (internals->active_slave_count < 1) {
1141 bonded_eth_dev->data->dev_link.link_status = 0;
1143 link_properties_reset(bonded_eth_dev);
1146 /* Update primary id, take first active slave from list or if none
1147 * available set to -1 */
1148 if (port_id == internals->current_primary_port) {
1149 if (internals->active_slave_count > 0)
1150 bond_ethdev_primary_set(internals,
1151 internals->active_slaves[0]);
1153 internals->current_primary_port = internals->primary_port;
1158 /* Cancel any possible outstanding interrupts if delays are enabled */
1159 if (internals->link_up_delay_ms > 0 ||
1160 internals->link_down_delay_ms > 0)
1161 rte_eal_alarm_cancel(bond_ethdev_delayed_lsc_propagation,
1164 if (bonded_eth_dev->data->dev_link.link_status) {
1165 if (internals->link_up_delay_ms > 0)
1166 rte_eal_alarm_set(internals->link_up_delay_ms * 1000,
1167 bond_ethdev_delayed_lsc_propagation,
1168 (void *)bonded_eth_dev);
1170 _rte_eth_dev_callback_process(bonded_eth_dev,
1171 RTE_ETH_EVENT_INTR_LSC);
1174 if (internals->link_down_delay_ms > 0)
1175 rte_eal_alarm_set(internals->link_down_delay_ms * 1000,
1176 bond_ethdev_delayed_lsc_propagation,
1177 (void *)bonded_eth_dev);
1179 _rte_eth_dev_callback_process(bonded_eth_dev,
1180 RTE_ETH_EVENT_INTR_LSC);
1185 struct eth_dev_ops default_dev_ops = {
1186 .dev_start = bond_ethdev_start,
1187 .dev_stop = bond_ethdev_stop,
1188 .dev_close = bond_ethdev_close,
1189 .dev_configure = bond_ethdev_configure,
1190 .dev_infos_get = bond_ethdev_info,
1191 .rx_queue_setup = bond_ethdev_rx_queue_setup,
1192 .tx_queue_setup = bond_ethdev_tx_queue_setup,
1193 .rx_queue_release = bond_ethdev_rx_queue_release,
1194 .tx_queue_release = bond_ethdev_tx_queue_release,
1195 .link_update = bond_ethdev_link_update,
1196 .stats_get = bond_ethdev_stats_get,
1197 .stats_reset = bond_ethdev_stats_reset,
1198 .promiscuous_enable = bond_ethdev_promiscuous_enable,
1199 .promiscuous_disable = bond_ethdev_promiscuous_disable
1203 bond_init(const char *name, const char *params)
1205 struct bond_dev_private *internals;
1206 struct rte_kvargs *kvlist;
1207 uint8_t bonding_mode, socket_id;
1208 int arg_count, port_id;
1210 RTE_LOG(INFO, EAL, "Initializing pmd_bond for %s\n", name);
1212 kvlist = rte_kvargs_parse(params, pmd_bond_init_valid_arguments);
1216 /* Parse link bonding mode */
1217 if (rte_kvargs_count(kvlist, PMD_BOND_MODE_KVARG) == 1) {
1218 if (rte_kvargs_process(kvlist, PMD_BOND_MODE_KVARG,
1219 &bond_ethdev_parse_slave_mode_kvarg, &bonding_mode) != 0) {
1220 RTE_LOG(ERR, EAL, "Invalid mode for bonded device %s\n", name);
1225 "Mode must be specified only once for bonded device %s\n",
1230 /* Parse socket id to create bonding device on */
1231 arg_count = rte_kvargs_count(kvlist, PMD_BOND_SOCKET_ID_KVARG);
1232 if (arg_count == 1) {
1233 if (rte_kvargs_process(kvlist, PMD_BOND_SOCKET_ID_KVARG,
1234 &bond_ethdev_parse_socket_id_kvarg, &socket_id) != 0) {
1236 "Invalid socket Id specified for bonded device %s\n",
1240 } else if (arg_count > 1) {
1242 "Socket Id can be specified only once for bonded device %s\n",
1246 socket_id = rte_socket_id();
1249 /* Create link bonding eth device */
1250 port_id = rte_eth_bond_create(name, bonding_mode, socket_id);
1253 "Failed to create socket %s in mode %u on socket %u.\n",
1254 name, bonding_mode, socket_id);
1257 internals = rte_eth_devices[port_id].data->dev_private;
1258 internals->kvlist = kvlist;
1261 "Create bonded device %s on port %d in mode %u on socket %u.\n",
1262 name, port_id, bonding_mode, socket_id);
1266 /* this part will resolve the slave portids after all the other pdev and vdev
1267 * have been allocated */
1269 bond_ethdev_configure(struct rte_eth_dev *dev)
1271 char *name = dev->data->name;
1272 struct bond_dev_private *internals = dev->data->dev_private;
1273 struct rte_kvargs *kvlist = internals->kvlist;
1274 int arg_count, port_id = dev - rte_eth_devices;
1277 * if no kvlist, it means that this bonded device has been created
1278 * through the bonding api.
1283 /* Parse MAC address for bonded device */
1284 arg_count = rte_kvargs_count(kvlist, PMD_BOND_MAC_ADDR_KVARG);
1285 if (arg_count == 1) {
1286 struct ether_addr bond_mac;
1288 if (rte_kvargs_process(kvlist, PMD_BOND_MAC_ADDR_KVARG,
1289 &bond_ethdev_parse_bond_mac_addr_kvarg, &bond_mac) < 0) {
1290 RTE_LOG(INFO, EAL, "Invalid mac address for bonded device %s\n",
1295 /* Set MAC address */
1296 if (rte_eth_bond_mac_address_set(port_id, &bond_mac) != 0) {
1298 "Failed to set mac address on bonded device %s\n",
1302 } else if (arg_count > 1) {
1304 "MAC address can be specified only once for bonded device %s\n",
1309 /* Parse/set balance mode transmit policy */
1310 arg_count = rte_kvargs_count(kvlist, PMD_BOND_XMIT_POLICY_KVARG);
1311 if (arg_count == 1) {
1312 uint8_t xmit_policy;
1314 if (rte_kvargs_process(kvlist, PMD_BOND_XMIT_POLICY_KVARG,
1315 &bond_ethdev_parse_balance_xmit_policy_kvarg, &xmit_policy) !=
1318 "Invalid xmit policy specified for bonded device %s\n",
1323 /* Set balance mode transmit policy*/
1324 if (rte_eth_bond_xmit_policy_set(port_id, xmit_policy) != 0) {
1326 "Failed to set balance xmit policy on bonded device %s\n",
1330 } else if (arg_count > 1) {
1332 "Transmit policy can be specified only once for bonded device"
1337 /* Parse/add slave ports to bonded device */
1338 if (rte_kvargs_count(kvlist, PMD_BOND_SLAVE_PORT_KVARG) > 0) {
1339 struct bond_ethdev_slave_ports slave_ports;
1342 memset(&slave_ports, 0, sizeof(slave_ports));
1344 if (rte_kvargs_process(kvlist, PMD_BOND_SLAVE_PORT_KVARG,
1345 &bond_ethdev_parse_slave_port_kvarg, &slave_ports) != 0) {
1347 "Failed to parse slave ports for bonded device %s\n",
1352 for (i = 0; i < slave_ports.slave_count; i++) {
1353 if (rte_eth_bond_slave_add(port_id, slave_ports.slaves[i]) != 0) {
1355 "Failed to add port %d as slave to bonded device %s\n",
1356 slave_ports.slaves[i], name);
1361 RTE_LOG(INFO, EAL, "No slaves specified for bonded device %s\n", name);
1365 /* Parse/set primary slave port id*/
1366 arg_count = rte_kvargs_count(kvlist, PMD_BOND_PRIMARY_SLAVE_KVARG);
1367 if (arg_count == 1) {
1368 uint8_t primary_slave_port_id;
1370 if (rte_kvargs_process(kvlist,
1371 PMD_BOND_PRIMARY_SLAVE_KVARG,
1372 &bond_ethdev_parse_primary_slave_port_id_kvarg,
1373 &primary_slave_port_id) < 0) {
1375 "Invalid primary slave port id specified for bonded device"
1380 /* Set balance mode transmit policy*/
1381 if (rte_eth_bond_primary_set(port_id, (uint8_t)primary_slave_port_id)
1384 "Failed to set primary slave port %d on bonded device %s\n",
1385 primary_slave_port_id, name);
1388 } else if (arg_count > 1) {
1390 "Primary slave can be specified only once for bonded device"
1395 /* Parse link status monitor polling interval */
1396 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LSC_POLL_PERIOD_KVARG);
1397 if (arg_count == 1) {
1398 uint32_t lsc_poll_interval_ms;
1400 if (rte_kvargs_process(kvlist,
1401 PMD_BOND_LSC_POLL_PERIOD_KVARG,
1402 &bond_ethdev_parse_time_ms_kvarg,
1403 &lsc_poll_interval_ms) < 0) {
1405 "Invalid lsc polling interval value specified for bonded"
1406 " device %s\n", name);
1410 if (rte_eth_bond_link_monitoring_set(port_id, lsc_poll_interval_ms)
1413 "Failed to set lsc monitor polling interval (%u ms) on"
1414 " bonded device %s\n", lsc_poll_interval_ms, name);
1417 } else if (arg_count > 1) {
1419 "LSC polling interval can be specified only once for bonded"
1420 " device %s\n", name);
1424 /* Parse link up interrupt propagation delay */
1425 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_UP_PROP_DELAY_KVARG);
1426 if (arg_count == 1) {
1427 uint32_t link_up_delay_ms;
1429 if (rte_kvargs_process(kvlist,
1430 PMD_BOND_LINK_UP_PROP_DELAY_KVARG,
1431 &bond_ethdev_parse_time_ms_kvarg,
1432 &link_up_delay_ms) < 0) {
1434 "Invalid link up propagation delay value specified for"
1435 " bonded device %s\n", name);
1439 /* Set balance mode transmit policy*/
1440 if (rte_eth_bond_link_up_prop_delay_set(port_id, link_up_delay_ms)
1443 "Failed to set link up propagation delay (%u ms) on bonded"
1444 " device %s\n", link_up_delay_ms, name);
1447 } else if (arg_count > 1) {
1449 "Link up propagation delay can be specified only once for"
1450 " bonded device %s\n", name);
1454 /* Parse link down interrupt propagation delay */
1455 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG);
1456 if (arg_count == 1) {
1457 uint32_t link_down_delay_ms;
1459 if (rte_kvargs_process(kvlist,
1460 PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG,
1461 &bond_ethdev_parse_time_ms_kvarg,
1462 &link_down_delay_ms) < 0) {
1464 "Invalid link down propagation delay value specified for"
1465 " bonded device %s\n", name);
1469 /* Set balance mode transmit policy*/
1470 if (rte_eth_bond_link_down_prop_delay_set(port_id, link_down_delay_ms)
1473 "Failed to set link down propagation delay (%u ms) on"
1474 " bonded device %s\n", link_down_delay_ms, name);
1477 } else if (arg_count > 1) {
1479 "Link down propagation delay can be specified only once for"
1480 " bonded device %s\n", name);
1487 static struct rte_driver bond_drv = {
1493 PMD_REGISTER_DRIVER(bond_drv);