1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2017 Intel Corporation
6 #include <netinet/in.h>
9 #include <rte_malloc.h>
10 #include <rte_ethdev_driver.h>
11 #include <rte_ethdev_vdev.h>
15 #include <rte_ip_frag.h>
16 #include <rte_devargs.h>
17 #include <rte_kvargs.h>
18 #include <rte_bus_vdev.h>
19 #include <rte_alarm.h>
20 #include <rte_cycles.h>
21 #include <rte_string_fns.h>
23 #include "rte_eth_bond.h"
24 #include "rte_eth_bond_private.h"
25 #include "rte_eth_bond_8023ad_private.h"
27 #define REORDER_PERIOD_MS 10
28 #define DEFAULT_POLLING_INTERVAL_10_MS (10)
29 #define BOND_MAX_MAC_ADDRS 16
31 #define HASH_L4_PORTS(h) ((h)->src_port ^ (h)->dst_port)
33 /* Table for statistics in mode 5 TLB */
34 static uint64_t tlb_last_obytets[RTE_MAX_ETHPORTS];
37 get_vlan_offset(struct rte_ether_hdr *eth_hdr, uint16_t *proto)
39 size_t vlan_offset = 0;
41 if (rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN) == *proto ||
42 rte_cpu_to_be_16(RTE_ETHER_TYPE_QINQ) == *proto) {
43 struct rte_vlan_hdr *vlan_hdr =
44 (struct rte_vlan_hdr *)(eth_hdr + 1);
46 vlan_offset = sizeof(struct rte_vlan_hdr);
47 *proto = vlan_hdr->eth_proto;
49 if (rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN) == *proto) {
50 vlan_hdr = vlan_hdr + 1;
51 *proto = vlan_hdr->eth_proto;
52 vlan_offset += sizeof(struct rte_vlan_hdr);
59 bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
61 struct bond_dev_private *internals;
63 uint16_t num_rx_total = 0;
65 uint16_t active_slave;
68 /* Cast to structure, containing bonded device's port id and queue id */
69 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
70 internals = bd_rx_q->dev_private;
71 slave_count = internals->active_slave_count;
72 active_slave = internals->active_slave;
74 for (i = 0; i < slave_count && nb_pkts; i++) {
75 uint16_t num_rx_slave;
77 /* Offset of pointer to *bufs increases as packets are received
78 * from other slaves */
80 rte_eth_rx_burst(internals->active_slaves[active_slave],
82 bufs + num_rx_total, nb_pkts);
83 num_rx_total += num_rx_slave;
84 nb_pkts -= num_rx_slave;
85 if (++active_slave == slave_count)
89 if (++internals->active_slave >= slave_count)
90 internals->active_slave = 0;
95 bond_ethdev_rx_burst_active_backup(void *queue, struct rte_mbuf **bufs,
98 struct bond_dev_private *internals;
100 /* Cast to structure, containing bonded device's port id and queue id */
101 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
103 internals = bd_rx_q->dev_private;
105 return rte_eth_rx_burst(internals->current_primary_port,
106 bd_rx_q->queue_id, bufs, nb_pkts);
109 static inline uint8_t
110 is_lacp_packets(uint16_t ethertype, uint8_t subtype, struct rte_mbuf *mbuf)
112 const uint16_t ether_type_slow_be =
113 rte_be_to_cpu_16(RTE_ETHER_TYPE_SLOW);
115 return !((mbuf->ol_flags & PKT_RX_VLAN) ? mbuf->vlan_tci : 0) &&
116 (ethertype == ether_type_slow_be &&
117 (subtype == SLOW_SUBTYPE_MARKER || subtype == SLOW_SUBTYPE_LACP));
120 /*****************************************************************************
121 * Flow director's setup for mode 4 optimization
124 static struct rte_flow_item_eth flow_item_eth_type_8023ad = {
125 .dst.addr_bytes = { 0 },
126 .src.addr_bytes = { 0 },
127 .type = RTE_BE16(RTE_ETHER_TYPE_SLOW),
130 static struct rte_flow_item_eth flow_item_eth_mask_type_8023ad = {
131 .dst.addr_bytes = { 0 },
132 .src.addr_bytes = { 0 },
136 static struct rte_flow_item flow_item_8023ad[] = {
138 .type = RTE_FLOW_ITEM_TYPE_ETH,
139 .spec = &flow_item_eth_type_8023ad,
141 .mask = &flow_item_eth_mask_type_8023ad,
144 .type = RTE_FLOW_ITEM_TYPE_END,
151 const struct rte_flow_attr flow_attr_8023ad = {
160 bond_ethdev_8023ad_flow_verify(struct rte_eth_dev *bond_dev,
161 uint16_t slave_port) {
162 struct rte_eth_dev_info slave_info;
163 struct rte_flow_error error;
164 struct bond_dev_private *internals = bond_dev->data->dev_private;
166 const struct rte_flow_action_queue lacp_queue_conf = {
170 const struct rte_flow_action actions[] = {
172 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
173 .conf = &lacp_queue_conf
176 .type = RTE_FLOW_ACTION_TYPE_END,
180 int ret = rte_flow_validate(slave_port, &flow_attr_8023ad,
181 flow_item_8023ad, actions, &error);
183 RTE_BOND_LOG(ERR, "%s: %s (slave_port=%d queue_id=%d)",
184 __func__, error.message, slave_port,
185 internals->mode4.dedicated_queues.rx_qid);
189 rte_eth_dev_info_get(slave_port, &slave_info);
190 if (slave_info.max_rx_queues < bond_dev->data->nb_rx_queues ||
191 slave_info.max_tx_queues < bond_dev->data->nb_tx_queues) {
193 "%s: Slave %d capabilities doesn't allow to allocate additional queues",
194 __func__, slave_port);
202 bond_8023ad_slow_pkt_hw_filter_supported(uint16_t port_id) {
203 struct rte_eth_dev *bond_dev = &rte_eth_devices[port_id];
204 struct bond_dev_private *internals = bond_dev->data->dev_private;
205 struct rte_eth_dev_info bond_info;
208 /* Verify if all slaves in bonding supports flow director and */
209 if (internals->slave_count > 0) {
210 rte_eth_dev_info_get(bond_dev->data->port_id, &bond_info);
212 internals->mode4.dedicated_queues.rx_qid = bond_info.nb_rx_queues;
213 internals->mode4.dedicated_queues.tx_qid = bond_info.nb_tx_queues;
215 for (idx = 0; idx < internals->slave_count; idx++) {
216 if (bond_ethdev_8023ad_flow_verify(bond_dev,
217 internals->slaves[idx].port_id) != 0)
226 bond_ethdev_8023ad_flow_set(struct rte_eth_dev *bond_dev, uint16_t slave_port) {
228 struct rte_flow_error error;
229 struct bond_dev_private *internals = bond_dev->data->dev_private;
230 struct rte_flow_action_queue lacp_queue_conf = {
231 .index = internals->mode4.dedicated_queues.rx_qid,
234 const struct rte_flow_action actions[] = {
236 .type = RTE_FLOW_ACTION_TYPE_QUEUE,
237 .conf = &lacp_queue_conf
240 .type = RTE_FLOW_ACTION_TYPE_END,
244 internals->mode4.dedicated_queues.flow[slave_port] = rte_flow_create(slave_port,
245 &flow_attr_8023ad, flow_item_8023ad, actions, &error);
246 if (internals->mode4.dedicated_queues.flow[slave_port] == NULL) {
247 RTE_BOND_LOG(ERR, "bond_ethdev_8023ad_flow_set: %s "
248 "(slave_port=%d queue_id=%d)",
249 error.message, slave_port,
250 internals->mode4.dedicated_queues.rx_qid);
257 static inline uint16_t
258 rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts,
261 /* Cast to structure, containing bonded device's port id and queue id */
262 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
263 struct bond_dev_private *internals = bd_rx_q->dev_private;
264 struct rte_eth_dev *bonded_eth_dev =
265 &rte_eth_devices[internals->port_id];
266 struct rte_ether_addr *bond_mac = bonded_eth_dev->data->mac_addrs;
267 struct rte_ether_hdr *hdr;
269 const uint16_t ether_type_slow_be =
270 rte_be_to_cpu_16(RTE_ETHER_TYPE_SLOW);
271 uint16_t num_rx_total = 0; /* Total number of received packets */
272 uint16_t slaves[RTE_MAX_ETHPORTS];
273 uint16_t slave_count, idx;
275 uint8_t collecting; /* current slave collecting status */
276 const uint8_t promisc = rte_eth_promiscuous_get(internals->port_id);
277 const uint8_t allmulti = rte_eth_allmulticast_get(internals->port_id);
283 /* Copy slave list to protect against slave up/down changes during tx
285 slave_count = internals->active_slave_count;
286 memcpy(slaves, internals->active_slaves,
287 sizeof(internals->active_slaves[0]) * slave_count);
289 idx = internals->active_slave;
290 if (idx >= slave_count) {
291 internals->active_slave = 0;
294 for (i = 0; i < slave_count && num_rx_total < nb_pkts; i++) {
296 collecting = ACTOR_STATE(&bond_mode_8023ad_ports[slaves[idx]],
299 /* Read packets from this slave */
300 num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id,
301 &bufs[num_rx_total], nb_pkts - num_rx_total);
303 for (k = j; k < 2 && k < num_rx_total; k++)
304 rte_prefetch0(rte_pktmbuf_mtod(bufs[k], void *));
306 /* Handle slow protocol packets. */
307 while (j < num_rx_total) {
308 if (j + 3 < num_rx_total)
309 rte_prefetch0(rte_pktmbuf_mtod(bufs[j + 3], void *));
311 hdr = rte_pktmbuf_mtod(bufs[j], struct rte_ether_hdr *);
312 subtype = ((struct slow_protocol_frame *)hdr)->slow_protocol.subtype;
314 /* Remove packet from array if:
315 * - it is slow packet but no dedicated rxq is present,
316 * - slave is not in collecting state,
317 * - bonding interface is not in promiscuous mode:
318 * - packet is unicast and address does not match,
319 * - packet is multicast and bonding interface
320 * is not in allmulti,
324 is_lacp_packets(hdr->ether_type, subtype,
328 ((rte_is_unicast_ether_addr(&hdr->d_addr) &&
329 !rte_is_same_ether_addr(bond_mac,
332 rte_is_multicast_ether_addr(&hdr->d_addr)))))) {
334 if (hdr->ether_type == ether_type_slow_be) {
335 bond_mode_8023ad_handle_slow_pkt(
336 internals, slaves[idx], bufs[j]);
338 rte_pktmbuf_free(bufs[j]);
340 /* Packet is managed by mode 4 or dropped, shift the array */
342 if (j < num_rx_total) {
343 memmove(&bufs[j], &bufs[j + 1], sizeof(bufs[0]) *
349 if (unlikely(++idx == slave_count))
353 if (++internals->active_slave >= slave_count)
354 internals->active_slave = 0;
360 bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
363 return rx_burst_8023ad(queue, bufs, nb_pkts, false);
367 bond_ethdev_rx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs,
370 return rx_burst_8023ad(queue, bufs, nb_pkts, true);
373 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
374 uint32_t burstnumberRX;
375 uint32_t burstnumberTX;
377 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
380 arp_op_name(uint16_t arp_op, char *buf, size_t buf_len)
383 case RTE_ARP_OP_REQUEST:
384 strlcpy(buf, "ARP Request", buf_len);
386 case RTE_ARP_OP_REPLY:
387 strlcpy(buf, "ARP Reply", buf_len);
389 case RTE_ARP_OP_REVREQUEST:
390 strlcpy(buf, "Reverse ARP Request", buf_len);
392 case RTE_ARP_OP_REVREPLY:
393 strlcpy(buf, "Reverse ARP Reply", buf_len);
395 case RTE_ARP_OP_INVREQUEST:
396 strlcpy(buf, "Peer Identify Request", buf_len);
398 case RTE_ARP_OP_INVREPLY:
399 strlcpy(buf, "Peer Identify Reply", buf_len);
404 strlcpy(buf, "Unknown", buf_len);
408 #define MaxIPv4String 16
410 ipv4_addr_to_dot(uint32_t be_ipv4_addr, char *buf, uint8_t buf_size)
414 ipv4_addr = rte_be_to_cpu_32(be_ipv4_addr);
415 snprintf(buf, buf_size, "%d.%d.%d.%d", (ipv4_addr >> 24) & 0xFF,
416 (ipv4_addr >> 16) & 0xFF, (ipv4_addr >> 8) & 0xFF,
420 #define MAX_CLIENTS_NUMBER 128
421 uint8_t active_clients;
422 struct client_stats_t {
425 uint32_t ipv4_rx_packets;
426 uint32_t ipv4_tx_packets;
428 struct client_stats_t client_stats[MAX_CLIENTS_NUMBER];
431 update_client_stats(uint32_t addr, uint16_t port, uint32_t *TXorRXindicator)
435 for (; i < MAX_CLIENTS_NUMBER; i++) {
436 if ((client_stats[i].ipv4_addr == addr) && (client_stats[i].port == port)) {
437 /* Just update RX packets number for this client */
438 if (TXorRXindicator == &burstnumberRX)
439 client_stats[i].ipv4_rx_packets++;
441 client_stats[i].ipv4_tx_packets++;
445 /* We have a new client. Insert him to the table, and increment stats */
446 if (TXorRXindicator == &burstnumberRX)
447 client_stats[active_clients].ipv4_rx_packets++;
449 client_stats[active_clients].ipv4_tx_packets++;
450 client_stats[active_clients].ipv4_addr = addr;
451 client_stats[active_clients].port = port;
456 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
457 #define MODE6_DEBUG(info, src_ip, dst_ip, eth_h, arp_op, port, burstnumber) \
458 rte_log(RTE_LOG_DEBUG, bond_logtype, \
459 "%s port:%d SrcMAC:%02X:%02X:%02X:%02X:%02X:%02X SrcIP:%s " \
460 "DstMAC:%02X:%02X:%02X:%02X:%02X:%02X DstIP:%s %s %d\n", \
463 eth_h->s_addr.addr_bytes[0], eth_h->s_addr.addr_bytes[1], \
464 eth_h->s_addr.addr_bytes[2], eth_h->s_addr.addr_bytes[3], \
465 eth_h->s_addr.addr_bytes[4], eth_h->s_addr.addr_bytes[5], \
467 eth_h->d_addr.addr_bytes[0], eth_h->d_addr.addr_bytes[1], \
468 eth_h->d_addr.addr_bytes[2], eth_h->d_addr.addr_bytes[3], \
469 eth_h->d_addr.addr_bytes[4], eth_h->d_addr.addr_bytes[5], \
471 arp_op, ++burstnumber)
475 mode6_debug(const char __attribute__((unused)) *info,
476 struct rte_ether_hdr *eth_h, uint16_t port,
477 uint32_t __attribute__((unused)) *burstnumber)
479 struct rte_ipv4_hdr *ipv4_h;
480 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
481 struct rte_arp_hdr *arp_h;
488 uint16_t ether_type = eth_h->ether_type;
489 uint16_t offset = get_vlan_offset(eth_h, ðer_type);
491 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
492 strlcpy(buf, info, 16);
495 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
496 ipv4_h = (struct rte_ipv4_hdr *)((char *)(eth_h + 1) + offset);
497 ipv4_addr_to_dot(ipv4_h->src_addr, src_ip, MaxIPv4String);
498 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
499 ipv4_addr_to_dot(ipv4_h->dst_addr, dst_ip, MaxIPv4String);
500 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, "", port, *burstnumber);
502 update_client_stats(ipv4_h->src_addr, port, burstnumber);
504 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
505 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) {
506 arp_h = (struct rte_arp_hdr *)((char *)(eth_h + 1) + offset);
507 ipv4_addr_to_dot(arp_h->arp_data.arp_sip, src_ip, MaxIPv4String);
508 ipv4_addr_to_dot(arp_h->arp_data.arp_tip, dst_ip, MaxIPv4String);
509 arp_op_name(rte_be_to_cpu_16(arp_h->arp_opcode),
510 ArpOp, sizeof(ArpOp));
511 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, ArpOp, port, *burstnumber);
518 bond_ethdev_rx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
520 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
521 struct bond_dev_private *internals = bd_tx_q->dev_private;
522 struct rte_ether_hdr *eth_h;
523 uint16_t ether_type, offset;
524 uint16_t nb_recv_pkts;
527 nb_recv_pkts = bond_ethdev_rx_burst(queue, bufs, nb_pkts);
529 for (i = 0; i < nb_recv_pkts; i++) {
530 eth_h = rte_pktmbuf_mtod(bufs[i], struct rte_ether_hdr *);
531 ether_type = eth_h->ether_type;
532 offset = get_vlan_offset(eth_h, ðer_type);
534 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) {
535 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
536 mode6_debug("RX ARP:", eth_h, bufs[i]->port, &burstnumberRX);
538 bond_mode_alb_arp_recv(eth_h, offset, internals);
540 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
541 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4))
542 mode6_debug("RX IPv4:", eth_h, bufs[i]->port, &burstnumberRX);
550 bond_ethdev_tx_burst_round_robin(void *queue, struct rte_mbuf **bufs,
553 struct bond_dev_private *internals;
554 struct bond_tx_queue *bd_tx_q;
556 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts];
557 uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
559 uint16_t num_of_slaves;
560 uint16_t slaves[RTE_MAX_ETHPORTS];
562 uint16_t num_tx_total = 0, num_tx_slave;
564 static int slave_idx = 0;
565 int i, cslave_idx = 0, tx_fail_total = 0;
567 bd_tx_q = (struct bond_tx_queue *)queue;
568 internals = bd_tx_q->dev_private;
570 /* Copy slave list to protect against slave up/down changes during tx
572 num_of_slaves = internals->active_slave_count;
573 memcpy(slaves, internals->active_slaves,
574 sizeof(internals->active_slaves[0]) * num_of_slaves);
576 if (num_of_slaves < 1)
579 /* Populate slaves mbuf with which packets are to be sent on it */
580 for (i = 0; i < nb_pkts; i++) {
581 cslave_idx = (slave_idx + i) % num_of_slaves;
582 slave_bufs[cslave_idx][(slave_nb_pkts[cslave_idx])++] = bufs[i];
585 /* increment current slave index so the next call to tx burst starts on the
587 slave_idx = ++cslave_idx;
589 /* Send packet burst on each slave device */
590 for (i = 0; i < num_of_slaves; i++) {
591 if (slave_nb_pkts[i] > 0) {
592 num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
593 slave_bufs[i], slave_nb_pkts[i]);
595 /* if tx burst fails move packets to end of bufs */
596 if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
597 int tx_fail_slave = slave_nb_pkts[i] - num_tx_slave;
599 tx_fail_total += tx_fail_slave;
601 memcpy(&bufs[nb_pkts - tx_fail_total],
602 &slave_bufs[i][num_tx_slave],
603 tx_fail_slave * sizeof(bufs[0]));
605 num_tx_total += num_tx_slave;
613 bond_ethdev_tx_burst_active_backup(void *queue,
614 struct rte_mbuf **bufs, uint16_t nb_pkts)
616 struct bond_dev_private *internals;
617 struct bond_tx_queue *bd_tx_q;
619 bd_tx_q = (struct bond_tx_queue *)queue;
620 internals = bd_tx_q->dev_private;
622 if (internals->active_slave_count < 1)
625 return rte_eth_tx_burst(internals->current_primary_port, bd_tx_q->queue_id,
629 static inline uint16_t
630 ether_hash(struct rte_ether_hdr *eth_hdr)
632 unaligned_uint16_t *word_src_addr =
633 (unaligned_uint16_t *)eth_hdr->s_addr.addr_bytes;
634 unaligned_uint16_t *word_dst_addr =
635 (unaligned_uint16_t *)eth_hdr->d_addr.addr_bytes;
637 return (word_src_addr[0] ^ word_dst_addr[0]) ^
638 (word_src_addr[1] ^ word_dst_addr[1]) ^
639 (word_src_addr[2] ^ word_dst_addr[2]);
642 static inline uint32_t
643 ipv4_hash(struct rte_ipv4_hdr *ipv4_hdr)
645 return ipv4_hdr->src_addr ^ ipv4_hdr->dst_addr;
648 static inline uint32_t
649 ipv6_hash(struct rte_ipv6_hdr *ipv6_hdr)
651 unaligned_uint32_t *word_src_addr =
652 (unaligned_uint32_t *)&(ipv6_hdr->src_addr[0]);
653 unaligned_uint32_t *word_dst_addr =
654 (unaligned_uint32_t *)&(ipv6_hdr->dst_addr[0]);
656 return (word_src_addr[0] ^ word_dst_addr[0]) ^
657 (word_src_addr[1] ^ word_dst_addr[1]) ^
658 (word_src_addr[2] ^ word_dst_addr[2]) ^
659 (word_src_addr[3] ^ word_dst_addr[3]);
664 burst_xmit_l2_hash(struct rte_mbuf **buf, uint16_t nb_pkts,
665 uint16_t slave_count, uint16_t *slaves)
667 struct rte_ether_hdr *eth_hdr;
671 for (i = 0; i < nb_pkts; i++) {
672 eth_hdr = rte_pktmbuf_mtod(buf[i], struct rte_ether_hdr *);
674 hash = ether_hash(eth_hdr);
676 slaves[i] = (hash ^= hash >> 8) % slave_count;
681 burst_xmit_l23_hash(struct rte_mbuf **buf, uint16_t nb_pkts,
682 uint16_t slave_count, uint16_t *slaves)
685 struct rte_ether_hdr *eth_hdr;
688 uint32_t hash, l3hash;
690 for (i = 0; i < nb_pkts; i++) {
691 eth_hdr = rte_pktmbuf_mtod(buf[i], struct rte_ether_hdr *);
694 proto = eth_hdr->ether_type;
695 hash = ether_hash(eth_hdr);
697 vlan_offset = get_vlan_offset(eth_hdr, &proto);
699 if (rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4) == proto) {
700 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)
701 ((char *)(eth_hdr + 1) + vlan_offset);
702 l3hash = ipv4_hash(ipv4_hdr);
704 } else if (rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6) == proto) {
705 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)
706 ((char *)(eth_hdr + 1) + vlan_offset);
707 l3hash = ipv6_hash(ipv6_hdr);
710 hash = hash ^ l3hash;
714 slaves[i] = hash % slave_count;
719 burst_xmit_l34_hash(struct rte_mbuf **buf, uint16_t nb_pkts,
720 uint16_t slave_count, uint16_t *slaves)
722 struct rte_ether_hdr *eth_hdr;
727 struct rte_udp_hdr *udp_hdr;
728 struct rte_tcp_hdr *tcp_hdr;
729 uint32_t hash, l3hash, l4hash;
731 for (i = 0; i < nb_pkts; i++) {
732 eth_hdr = rte_pktmbuf_mtod(buf[i], struct rte_ether_hdr *);
733 size_t pkt_end = (size_t)eth_hdr + rte_pktmbuf_data_len(buf[i]);
734 proto = eth_hdr->ether_type;
735 vlan_offset = get_vlan_offset(eth_hdr, &proto);
739 if (rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4) == proto) {
740 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)
741 ((char *)(eth_hdr + 1) + vlan_offset);
742 size_t ip_hdr_offset;
744 l3hash = ipv4_hash(ipv4_hdr);
746 /* there is no L4 header in fragmented packet */
747 if (likely(rte_ipv4_frag_pkt_is_fragmented(ipv4_hdr)
749 ip_hdr_offset = (ipv4_hdr->version_ihl
750 & RTE_IPV4_HDR_IHL_MASK) *
751 RTE_IPV4_IHL_MULTIPLIER;
753 if (ipv4_hdr->next_proto_id == IPPROTO_TCP) {
754 tcp_hdr = (struct rte_tcp_hdr *)
757 if ((size_t)tcp_hdr + sizeof(*tcp_hdr)
759 l4hash = HASH_L4_PORTS(tcp_hdr);
760 } else if (ipv4_hdr->next_proto_id ==
762 udp_hdr = (struct rte_udp_hdr *)
765 if ((size_t)udp_hdr + sizeof(*udp_hdr)
767 l4hash = HASH_L4_PORTS(udp_hdr);
770 } else if (rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6) == proto) {
771 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)
772 ((char *)(eth_hdr + 1) + vlan_offset);
773 l3hash = ipv6_hash(ipv6_hdr);
775 if (ipv6_hdr->proto == IPPROTO_TCP) {
776 tcp_hdr = (struct rte_tcp_hdr *)(ipv6_hdr + 1);
777 l4hash = HASH_L4_PORTS(tcp_hdr);
778 } else if (ipv6_hdr->proto == IPPROTO_UDP) {
779 udp_hdr = (struct rte_udp_hdr *)(ipv6_hdr + 1);
780 l4hash = HASH_L4_PORTS(udp_hdr);
784 hash = l3hash ^ l4hash;
788 slaves[i] = hash % slave_count;
793 uint64_t bwg_left_int;
794 uint64_t bwg_left_remainder;
799 bond_tlb_activate_slave(struct bond_dev_private *internals) {
802 for (i = 0; i < internals->active_slave_count; i++) {
803 tlb_last_obytets[internals->active_slaves[i]] = 0;
808 bandwidth_cmp(const void *a, const void *b)
810 const struct bwg_slave *bwg_a = a;
811 const struct bwg_slave *bwg_b = b;
812 int64_t diff = (int64_t)bwg_b->bwg_left_int - (int64_t)bwg_a->bwg_left_int;
813 int64_t diff2 = (int64_t)bwg_b->bwg_left_remainder -
814 (int64_t)bwg_a->bwg_left_remainder;
828 bandwidth_left(uint16_t port_id, uint64_t load, uint8_t update_idx,
829 struct bwg_slave *bwg_slave)
831 struct rte_eth_link link_status;
833 rte_eth_link_get_nowait(port_id, &link_status);
834 uint64_t link_bwg = link_status.link_speed * 1000000ULL / 8;
837 link_bwg = link_bwg * (update_idx+1) * REORDER_PERIOD_MS;
838 bwg_slave->bwg_left_int = (link_bwg - 1000*load) / link_bwg;
839 bwg_slave->bwg_left_remainder = (link_bwg - 1000*load) % link_bwg;
843 bond_ethdev_update_tlb_slave_cb(void *arg)
845 struct bond_dev_private *internals = arg;
846 struct rte_eth_stats slave_stats;
847 struct bwg_slave bwg_array[RTE_MAX_ETHPORTS];
848 uint16_t slave_count;
851 uint8_t update_stats = 0;
855 internals->slave_update_idx++;
858 if (internals->slave_update_idx >= REORDER_PERIOD_MS)
861 for (i = 0; i < internals->active_slave_count; i++) {
862 slave_id = internals->active_slaves[i];
863 rte_eth_stats_get(slave_id, &slave_stats);
864 tx_bytes = slave_stats.obytes - tlb_last_obytets[slave_id];
865 bandwidth_left(slave_id, tx_bytes,
866 internals->slave_update_idx, &bwg_array[i]);
867 bwg_array[i].slave = slave_id;
870 tlb_last_obytets[slave_id] = slave_stats.obytes;
874 if (update_stats == 1)
875 internals->slave_update_idx = 0;
878 qsort(bwg_array, slave_count, sizeof(bwg_array[0]), bandwidth_cmp);
879 for (i = 0; i < slave_count; i++)
880 internals->tlb_slaves_order[i] = bwg_array[i].slave;
882 rte_eal_alarm_set(REORDER_PERIOD_MS * 1000, bond_ethdev_update_tlb_slave_cb,
883 (struct bond_dev_private *)internals);
887 bond_ethdev_tx_burst_tlb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
889 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
890 struct bond_dev_private *internals = bd_tx_q->dev_private;
892 struct rte_eth_dev *primary_port =
893 &rte_eth_devices[internals->primary_port];
894 uint16_t num_tx_total = 0;
897 uint16_t num_of_slaves = internals->active_slave_count;
898 uint16_t slaves[RTE_MAX_ETHPORTS];
900 struct rte_ether_hdr *ether_hdr;
901 struct rte_ether_addr primary_slave_addr;
902 struct rte_ether_addr active_slave_addr;
904 if (num_of_slaves < 1)
907 memcpy(slaves, internals->tlb_slaves_order,
908 sizeof(internals->tlb_slaves_order[0]) * num_of_slaves);
911 rte_ether_addr_copy(primary_port->data->mac_addrs, &primary_slave_addr);
914 for (i = 0; i < 3; i++)
915 rte_prefetch0(rte_pktmbuf_mtod(bufs[i], void*));
918 for (i = 0; i < num_of_slaves; i++) {
919 rte_eth_macaddr_get(slaves[i], &active_slave_addr);
920 for (j = num_tx_total; j < nb_pkts; j++) {
922 rte_prefetch0(rte_pktmbuf_mtod(bufs[j+3], void*));
924 ether_hdr = rte_pktmbuf_mtod(bufs[j],
925 struct rte_ether_hdr *);
926 if (rte_is_same_ether_addr(ðer_hdr->s_addr,
927 &primary_slave_addr))
928 rte_ether_addr_copy(&active_slave_addr,
930 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
931 mode6_debug("TX IPv4:", ether_hdr, slaves[i], &burstnumberTX);
935 num_tx_total += rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
936 bufs + num_tx_total, nb_pkts - num_tx_total);
938 if (num_tx_total == nb_pkts)
946 bond_tlb_disable(struct bond_dev_private *internals)
948 rte_eal_alarm_cancel(bond_ethdev_update_tlb_slave_cb, internals);
952 bond_tlb_enable(struct bond_dev_private *internals)
954 bond_ethdev_update_tlb_slave_cb(internals);
958 bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
960 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
961 struct bond_dev_private *internals = bd_tx_q->dev_private;
963 struct rte_ether_hdr *eth_h;
964 uint16_t ether_type, offset;
966 struct client_data *client_info;
969 * We create transmit buffers for every slave and one additional to send
970 * through tlb. In worst case every packet will be send on one port.
972 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS + 1][nb_pkts];
973 uint16_t slave_bufs_pkts[RTE_MAX_ETHPORTS + 1] = { 0 };
976 * We create separate transmit buffers for update packets as they won't
977 * be counted in num_tx_total.
979 struct rte_mbuf *update_bufs[RTE_MAX_ETHPORTS][ALB_HASH_TABLE_SIZE];
980 uint16_t update_bufs_pkts[RTE_MAX_ETHPORTS] = { 0 };
982 struct rte_mbuf *upd_pkt;
985 uint16_t num_send, num_not_send = 0;
986 uint16_t num_tx_total = 0;
991 /* Search tx buffer for ARP packets and forward them to alb */
992 for (i = 0; i < nb_pkts; i++) {
993 eth_h = rte_pktmbuf_mtod(bufs[i], struct rte_ether_hdr *);
994 ether_type = eth_h->ether_type;
995 offset = get_vlan_offset(eth_h, ðer_type);
997 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) {
998 slave_idx = bond_mode_alb_arp_xmit(eth_h, offset, internals);
1000 /* Change src mac in eth header */
1001 rte_eth_macaddr_get(slave_idx, ð_h->s_addr);
1003 /* Add packet to slave tx buffer */
1004 slave_bufs[slave_idx][slave_bufs_pkts[slave_idx]] = bufs[i];
1005 slave_bufs_pkts[slave_idx]++;
1007 /* If packet is not ARP, send it with TLB policy */
1008 slave_bufs[RTE_MAX_ETHPORTS][slave_bufs_pkts[RTE_MAX_ETHPORTS]] =
1010 slave_bufs_pkts[RTE_MAX_ETHPORTS]++;
1014 /* Update connected client ARP tables */
1015 if (internals->mode6.ntt) {
1016 for (i = 0; i < ALB_HASH_TABLE_SIZE; i++) {
1017 client_info = &internals->mode6.client_table[i];
1019 if (client_info->in_use) {
1020 /* Allocate new packet to send ARP update on current slave */
1021 upd_pkt = rte_pktmbuf_alloc(internals->mode6.mempool);
1022 if (upd_pkt == NULL) {
1024 "Failed to allocate ARP packet from pool");
1027 pkt_size = sizeof(struct rte_ether_hdr) +
1028 sizeof(struct rte_arp_hdr) +
1029 client_info->vlan_count *
1030 sizeof(struct rte_vlan_hdr);
1031 upd_pkt->data_len = pkt_size;
1032 upd_pkt->pkt_len = pkt_size;
1034 slave_idx = bond_mode_alb_arp_upd(client_info, upd_pkt,
1037 /* Add packet to update tx buffer */
1038 update_bufs[slave_idx][update_bufs_pkts[slave_idx]] = upd_pkt;
1039 update_bufs_pkts[slave_idx]++;
1042 internals->mode6.ntt = 0;
1045 /* Send ARP packets on proper slaves */
1046 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
1047 if (slave_bufs_pkts[i] > 0) {
1048 num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id,
1049 slave_bufs[i], slave_bufs_pkts[i]);
1050 for (j = 0; j < slave_bufs_pkts[i] - num_send; j++) {
1051 bufs[nb_pkts - 1 - num_not_send - j] =
1052 slave_bufs[i][nb_pkts - 1 - j];
1055 num_tx_total += num_send;
1056 num_not_send += slave_bufs_pkts[i] - num_send;
1058 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
1059 /* Print TX stats including update packets */
1060 for (j = 0; j < slave_bufs_pkts[i]; j++) {
1061 eth_h = rte_pktmbuf_mtod(slave_bufs[i][j],
1062 struct rte_ether_hdr *);
1063 mode6_debug("TX ARP:", eth_h, i, &burstnumberTX);
1069 /* Send update packets on proper slaves */
1070 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
1071 if (update_bufs_pkts[i] > 0) {
1072 num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id, update_bufs[i],
1073 update_bufs_pkts[i]);
1074 for (j = num_send; j < update_bufs_pkts[i]; j++) {
1075 rte_pktmbuf_free(update_bufs[i][j]);
1077 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
1078 for (j = 0; j < update_bufs_pkts[i]; j++) {
1079 eth_h = rte_pktmbuf_mtod(update_bufs[i][j],
1080 struct rte_ether_hdr *);
1081 mode6_debug("TX ARPupd:", eth_h, i, &burstnumberTX);
1087 /* Send non-ARP packets using tlb policy */
1088 if (slave_bufs_pkts[RTE_MAX_ETHPORTS] > 0) {
1089 num_send = bond_ethdev_tx_burst_tlb(queue,
1090 slave_bufs[RTE_MAX_ETHPORTS],
1091 slave_bufs_pkts[RTE_MAX_ETHPORTS]);
1093 for (j = 0; j < slave_bufs_pkts[RTE_MAX_ETHPORTS]; j++) {
1094 bufs[nb_pkts - 1 - num_not_send - j] =
1095 slave_bufs[RTE_MAX_ETHPORTS][nb_pkts - 1 - j];
1098 num_tx_total += num_send;
1101 return num_tx_total;
1104 static inline uint16_t
1105 tx_burst_balance(void *queue, struct rte_mbuf **bufs, uint16_t nb_bufs,
1106 uint16_t *slave_port_ids, uint16_t slave_count)
1108 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
1109 struct bond_dev_private *internals = bd_tx_q->dev_private;
1111 /* Array to sort mbufs for transmission on each slave into */
1112 struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_bufs];
1113 /* Number of mbufs for transmission on each slave */
1114 uint16_t slave_nb_bufs[RTE_MAX_ETHPORTS] = { 0 };
1115 /* Mapping array generated by hash function to map mbufs to slaves */
1116 uint16_t bufs_slave_port_idxs[nb_bufs];
1118 uint16_t slave_tx_count;
1119 uint16_t total_tx_count = 0, total_tx_fail_count = 0;
1124 * Populate slaves mbuf with the packets which are to be sent on it
1125 * selecting output slave using hash based on xmit policy
1127 internals->burst_xmit_hash(bufs, nb_bufs, slave_count,
1128 bufs_slave_port_idxs);
1130 for (i = 0; i < nb_bufs; i++) {
1131 /* Populate slave mbuf arrays with mbufs for that slave. */
1132 uint16_t slave_idx = bufs_slave_port_idxs[i];
1134 slave_bufs[slave_idx][slave_nb_bufs[slave_idx]++] = bufs[i];
1137 /* Send packet burst on each slave device */
1138 for (i = 0; i < slave_count; i++) {
1139 if (slave_nb_bufs[i] == 0)
1142 slave_tx_count = rte_eth_tx_burst(slave_port_ids[i],
1143 bd_tx_q->queue_id, slave_bufs[i],
1146 total_tx_count += slave_tx_count;
1148 /* If tx burst fails move packets to end of bufs */
1149 if (unlikely(slave_tx_count < slave_nb_bufs[i])) {
1150 int slave_tx_fail_count = slave_nb_bufs[i] -
1152 total_tx_fail_count += slave_tx_fail_count;
1153 memcpy(&bufs[nb_bufs - total_tx_fail_count],
1154 &slave_bufs[i][slave_tx_count],
1155 slave_tx_fail_count * sizeof(bufs[0]));
1159 return total_tx_count;
1163 bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,
1166 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
1167 struct bond_dev_private *internals = bd_tx_q->dev_private;
1169 uint16_t slave_port_ids[RTE_MAX_ETHPORTS];
1170 uint16_t slave_count;
1172 if (unlikely(nb_bufs == 0))
1175 /* Copy slave list to protect against slave up/down changes during tx
1178 slave_count = internals->active_slave_count;
1179 if (unlikely(slave_count < 1))
1182 memcpy(slave_port_ids, internals->active_slaves,
1183 sizeof(slave_port_ids[0]) * slave_count);
1184 return tx_burst_balance(queue, bufs, nb_bufs, slave_port_ids,
1188 static inline uint16_t
1189 tx_burst_8023ad(void *queue, struct rte_mbuf **bufs, uint16_t nb_bufs,
1192 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
1193 struct bond_dev_private *internals = bd_tx_q->dev_private;
1195 uint16_t slave_port_ids[RTE_MAX_ETHPORTS];
1196 uint16_t slave_count;
1198 uint16_t dist_slave_port_ids[RTE_MAX_ETHPORTS];
1199 uint16_t dist_slave_count;
1201 uint16_t slave_tx_count;
1205 /* Copy slave list to protect against slave up/down changes during tx
1207 slave_count = internals->active_slave_count;
1208 if (unlikely(slave_count < 1))
1211 memcpy(slave_port_ids, internals->active_slaves,
1212 sizeof(slave_port_ids[0]) * slave_count);
1217 /* Check for LACP control packets and send if available */
1218 for (i = 0; i < slave_count; i++) {
1219 struct port *port = &bond_mode_8023ad_ports[slave_port_ids[i]];
1220 struct rte_mbuf *ctrl_pkt = NULL;
1222 if (likely(rte_ring_empty(port->tx_ring)))
1225 if (rte_ring_dequeue(port->tx_ring,
1226 (void **)&ctrl_pkt) != -ENOENT) {
1227 slave_tx_count = rte_eth_tx_burst(slave_port_ids[i],
1228 bd_tx_q->queue_id, &ctrl_pkt, 1);
1230 * re-enqueue LAG control plane packets to buffering
1231 * ring if transmission fails so the packet isn't lost.
1233 if (slave_tx_count != 1)
1234 rte_ring_enqueue(port->tx_ring, ctrl_pkt);
1239 if (unlikely(nb_bufs == 0))
1242 dist_slave_count = 0;
1243 for (i = 0; i < slave_count; i++) {
1244 struct port *port = &bond_mode_8023ad_ports[slave_port_ids[i]];
1246 if (ACTOR_STATE(port, DISTRIBUTING))
1247 dist_slave_port_ids[dist_slave_count++] =
1251 if (unlikely(dist_slave_count < 1))
1254 return tx_burst_balance(queue, bufs, nb_bufs, dist_slave_port_ids,
1259 bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
1262 return tx_burst_8023ad(queue, bufs, nb_bufs, false);
1266 bond_ethdev_tx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs,
1269 return tx_burst_8023ad(queue, bufs, nb_bufs, true);
1273 bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs,
1276 struct bond_dev_private *internals;
1277 struct bond_tx_queue *bd_tx_q;
1279 uint16_t slaves[RTE_MAX_ETHPORTS];
1280 uint8_t tx_failed_flag = 0;
1281 uint16_t num_of_slaves;
1283 uint16_t max_nb_of_tx_pkts = 0;
1285 int slave_tx_total[RTE_MAX_ETHPORTS];
1286 int i, most_successful_tx_slave = -1;
1288 bd_tx_q = (struct bond_tx_queue *)queue;
1289 internals = bd_tx_q->dev_private;
1291 /* Copy slave list to protect against slave up/down changes during tx
1293 num_of_slaves = internals->active_slave_count;
1294 memcpy(slaves, internals->active_slaves,
1295 sizeof(internals->active_slaves[0]) * num_of_slaves);
1297 if (num_of_slaves < 1)
1300 /* Increment reference count on mbufs */
1301 for (i = 0; i < nb_pkts; i++)
1302 rte_mbuf_refcnt_update(bufs[i], num_of_slaves - 1);
1304 /* Transmit burst on each active slave */
1305 for (i = 0; i < num_of_slaves; i++) {
1306 slave_tx_total[i] = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
1309 if (unlikely(slave_tx_total[i] < nb_pkts))
1312 /* record the value and slave index for the slave which transmits the
1313 * maximum number of packets */
1314 if (slave_tx_total[i] > max_nb_of_tx_pkts) {
1315 max_nb_of_tx_pkts = slave_tx_total[i];
1316 most_successful_tx_slave = i;
1320 /* if slaves fail to transmit packets from burst, the calling application
1321 * is not expected to know about multiple references to packets so we must
1322 * handle failures of all packets except those of the most successful slave
1324 if (unlikely(tx_failed_flag))
1325 for (i = 0; i < num_of_slaves; i++)
1326 if (i != most_successful_tx_slave)
1327 while (slave_tx_total[i] < nb_pkts)
1328 rte_pktmbuf_free(bufs[slave_tx_total[i]++]);
1330 return max_nb_of_tx_pkts;
1334 link_properties_set(struct rte_eth_dev *ethdev, struct rte_eth_link *slave_link)
1336 struct bond_dev_private *bond_ctx = ethdev->data->dev_private;
1338 if (bond_ctx->mode == BONDING_MODE_8023AD) {
1340 * If in mode 4 then save the link properties of the first
1341 * slave, all subsequent slaves must match these properties
1343 struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link;
1345 bond_link->link_autoneg = slave_link->link_autoneg;
1346 bond_link->link_duplex = slave_link->link_duplex;
1347 bond_link->link_speed = slave_link->link_speed;
1350 * In any other mode the link properties are set to default
1351 * values of AUTONEG/DUPLEX
1353 ethdev->data->dev_link.link_autoneg = ETH_LINK_AUTONEG;
1354 ethdev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX;
1359 link_properties_valid(struct rte_eth_dev *ethdev,
1360 struct rte_eth_link *slave_link)
1362 struct bond_dev_private *bond_ctx = ethdev->data->dev_private;
1364 if (bond_ctx->mode == BONDING_MODE_8023AD) {
1365 struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link;
1367 if (bond_link->link_duplex != slave_link->link_duplex ||
1368 bond_link->link_autoneg != slave_link->link_autoneg ||
1369 bond_link->link_speed != slave_link->link_speed)
1377 mac_address_get(struct rte_eth_dev *eth_dev,
1378 struct rte_ether_addr *dst_mac_addr)
1380 struct rte_ether_addr *mac_addr;
1382 if (eth_dev == NULL) {
1383 RTE_BOND_LOG(ERR, "NULL pointer eth_dev specified");
1387 if (dst_mac_addr == NULL) {
1388 RTE_BOND_LOG(ERR, "NULL pointer MAC specified");
1392 mac_addr = eth_dev->data->mac_addrs;
1394 rte_ether_addr_copy(mac_addr, dst_mac_addr);
1399 mac_address_set(struct rte_eth_dev *eth_dev,
1400 struct rte_ether_addr *new_mac_addr)
1402 struct rte_ether_addr *mac_addr;
1404 if (eth_dev == NULL) {
1405 RTE_BOND_LOG(ERR, "NULL pointer eth_dev specified");
1409 if (new_mac_addr == NULL) {
1410 RTE_BOND_LOG(ERR, "NULL pointer MAC specified");
1414 mac_addr = eth_dev->data->mac_addrs;
1416 /* If new MAC is different to current MAC then update */
1417 if (memcmp(mac_addr, new_mac_addr, sizeof(*mac_addr)) != 0)
1418 memcpy(mac_addr, new_mac_addr, sizeof(*mac_addr));
1423 static const struct rte_ether_addr null_mac_addr;
1426 * Add additional MAC addresses to the slave
1429 slave_add_mac_addresses(struct rte_eth_dev *bonded_eth_dev,
1430 uint16_t slave_port_id)
1433 struct rte_ether_addr *mac_addr;
1435 for (i = 1; i < BOND_MAX_MAC_ADDRS; i++) {
1436 mac_addr = &bonded_eth_dev->data->mac_addrs[i];
1437 if (rte_is_same_ether_addr(mac_addr, &null_mac_addr))
1440 ret = rte_eth_dev_mac_addr_add(slave_port_id, mac_addr, 0);
1443 for (i--; i > 0; i--)
1444 rte_eth_dev_mac_addr_remove(slave_port_id,
1445 &bonded_eth_dev->data->mac_addrs[i]);
1454 * Remove additional MAC addresses from the slave
1457 slave_remove_mac_addresses(struct rte_eth_dev *bonded_eth_dev,
1458 uint16_t slave_port_id)
1461 struct rte_ether_addr *mac_addr;
1464 for (i = 1; i < BOND_MAX_MAC_ADDRS; i++) {
1465 mac_addr = &bonded_eth_dev->data->mac_addrs[i];
1466 if (rte_is_same_ether_addr(mac_addr, &null_mac_addr))
1469 ret = rte_eth_dev_mac_addr_remove(slave_port_id, mac_addr);
1470 /* save only the first error */
1471 if (ret < 0 && rc == 0)
1479 mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev)
1481 struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1484 /* Update slave devices MAC addresses */
1485 if (internals->slave_count < 1)
1488 switch (internals->mode) {
1489 case BONDING_MODE_ROUND_ROBIN:
1490 case BONDING_MODE_BALANCE:
1491 case BONDING_MODE_BROADCAST:
1492 for (i = 0; i < internals->slave_count; i++) {
1493 if (rte_eth_dev_default_mac_addr_set(
1494 internals->slaves[i].port_id,
1495 bonded_eth_dev->data->mac_addrs)) {
1496 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1497 internals->slaves[i].port_id);
1502 case BONDING_MODE_8023AD:
1503 bond_mode_8023ad_mac_address_update(bonded_eth_dev);
1505 case BONDING_MODE_ACTIVE_BACKUP:
1506 case BONDING_MODE_TLB:
1507 case BONDING_MODE_ALB:
1509 for (i = 0; i < internals->slave_count; i++) {
1510 if (internals->slaves[i].port_id ==
1511 internals->current_primary_port) {
1512 if (rte_eth_dev_default_mac_addr_set(
1513 internals->primary_port,
1514 bonded_eth_dev->data->mac_addrs)) {
1515 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1516 internals->current_primary_port);
1520 if (rte_eth_dev_default_mac_addr_set(
1521 internals->slaves[i].port_id,
1522 &internals->slaves[i].persisted_mac_addr)) {
1523 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1524 internals->slaves[i].port_id);
1535 bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, int mode)
1537 struct bond_dev_private *internals;
1539 internals = eth_dev->data->dev_private;
1542 case BONDING_MODE_ROUND_ROBIN:
1543 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_round_robin;
1544 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1546 case BONDING_MODE_ACTIVE_BACKUP:
1547 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_active_backup;
1548 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
1550 case BONDING_MODE_BALANCE:
1551 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_balance;
1552 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1554 case BONDING_MODE_BROADCAST:
1555 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_broadcast;
1556 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1558 case BONDING_MODE_8023AD:
1559 if (bond_mode_8023ad_enable(eth_dev) != 0)
1562 if (internals->mode4.dedicated_queues.enabled == 0) {
1563 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad;
1564 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad;
1565 RTE_BOND_LOG(WARNING,
1566 "Using mode 4, it is necessary to do TX burst "
1567 "and RX burst at least every 100ms.");
1569 /* Use flow director's optimization */
1570 eth_dev->rx_pkt_burst =
1571 bond_ethdev_rx_burst_8023ad_fast_queue;
1572 eth_dev->tx_pkt_burst =
1573 bond_ethdev_tx_burst_8023ad_fast_queue;
1576 case BONDING_MODE_TLB:
1577 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb;
1578 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
1580 case BONDING_MODE_ALB:
1581 if (bond_mode_alb_enable(eth_dev) != 0)
1584 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_alb;
1585 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_alb;
1591 internals->mode = mode;
1598 slave_configure_slow_queue(struct rte_eth_dev *bonded_eth_dev,
1599 struct rte_eth_dev *slave_eth_dev)
1602 struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1603 struct port *port = &bond_mode_8023ad_ports[slave_eth_dev->data->port_id];
1605 if (port->slow_pool == NULL) {
1607 int slave_id = slave_eth_dev->data->port_id;
1609 snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_slow_pool",
1611 port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191,
1612 250, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
1613 slave_eth_dev->data->numa_node);
1615 /* Any memory allocation failure in initialization is critical because
1616 * resources can't be free, so reinitialization is impossible. */
1617 if (port->slow_pool == NULL) {
1618 rte_panic("Slave %u: Failed to create memory pool '%s': %s\n",
1619 slave_id, mem_name, rte_strerror(rte_errno));
1623 if (internals->mode4.dedicated_queues.enabled == 1) {
1624 /* Configure slow Rx queue */
1626 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id,
1627 internals->mode4.dedicated_queues.rx_qid, 128,
1628 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1629 NULL, port->slow_pool);
1632 "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)",
1633 slave_eth_dev->data->port_id,
1634 internals->mode4.dedicated_queues.rx_qid,
1639 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id,
1640 internals->mode4.dedicated_queues.tx_qid, 512,
1641 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1645 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)",
1646 slave_eth_dev->data->port_id,
1647 internals->mode4.dedicated_queues.tx_qid,
1656 slave_configure(struct rte_eth_dev *bonded_eth_dev,
1657 struct rte_eth_dev *slave_eth_dev)
1659 struct bond_rx_queue *bd_rx_q;
1660 struct bond_tx_queue *bd_tx_q;
1661 uint16_t nb_rx_queues;
1662 uint16_t nb_tx_queues;
1666 struct rte_flow_error flow_error;
1668 struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1671 rte_eth_dev_stop(slave_eth_dev->data->port_id);
1673 /* Enable interrupts on slave device if supported */
1674 if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1675 slave_eth_dev->data->dev_conf.intr_conf.lsc = 1;
1677 /* If RSS is enabled for bonding, try to enable it for slaves */
1678 if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
1679 if (internals->rss_key_len != 0) {
1680 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len =
1681 internals->rss_key_len;
1682 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key =
1685 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
1688 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf =
1689 bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
1690 slave_eth_dev->data->dev_conf.rxmode.mq_mode =
1691 bonded_eth_dev->data->dev_conf.rxmode.mq_mode;
1694 if (bonded_eth_dev->data->dev_conf.rxmode.offloads &
1695 DEV_RX_OFFLOAD_VLAN_FILTER)
1696 slave_eth_dev->data->dev_conf.rxmode.offloads |=
1697 DEV_RX_OFFLOAD_VLAN_FILTER;
1699 slave_eth_dev->data->dev_conf.rxmode.offloads &=
1700 ~DEV_RX_OFFLOAD_VLAN_FILTER;
1702 nb_rx_queues = bonded_eth_dev->data->nb_rx_queues;
1703 nb_tx_queues = bonded_eth_dev->data->nb_tx_queues;
1705 if (internals->mode == BONDING_MODE_8023AD) {
1706 if (internals->mode4.dedicated_queues.enabled == 1) {
1712 errval = rte_eth_dev_set_mtu(slave_eth_dev->data->port_id,
1713 bonded_eth_dev->data->mtu);
1714 if (errval != 0 && errval != -ENOTSUP) {
1715 RTE_BOND_LOG(ERR, "rte_eth_dev_set_mtu: port %u, err (%d)",
1716 slave_eth_dev->data->port_id, errval);
1720 /* Configure device */
1721 errval = rte_eth_dev_configure(slave_eth_dev->data->port_id,
1722 nb_rx_queues, nb_tx_queues,
1723 &(slave_eth_dev->data->dev_conf));
1725 RTE_BOND_LOG(ERR, "Cannot configure slave device: port %u, err (%d)",
1726 slave_eth_dev->data->port_id, errval);
1730 /* Setup Rx Queues */
1731 for (q_id = 0; q_id < bonded_eth_dev->data->nb_rx_queues; q_id++) {
1732 bd_rx_q = (struct bond_rx_queue *)bonded_eth_dev->data->rx_queues[q_id];
1734 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, q_id,
1735 bd_rx_q->nb_rx_desc,
1736 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1737 &(bd_rx_q->rx_conf), bd_rx_q->mb_pool);
1740 "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)",
1741 slave_eth_dev->data->port_id, q_id, errval);
1746 /* Setup Tx Queues */
1747 for (q_id = 0; q_id < bonded_eth_dev->data->nb_tx_queues; q_id++) {
1748 bd_tx_q = (struct bond_tx_queue *)bonded_eth_dev->data->tx_queues[q_id];
1750 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, q_id,
1751 bd_tx_q->nb_tx_desc,
1752 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1756 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)",
1757 slave_eth_dev->data->port_id, q_id, errval);
1762 if (internals->mode == BONDING_MODE_8023AD &&
1763 internals->mode4.dedicated_queues.enabled == 1) {
1764 if (slave_configure_slow_queue(bonded_eth_dev, slave_eth_dev)
1768 if (bond_ethdev_8023ad_flow_verify(bonded_eth_dev,
1769 slave_eth_dev->data->port_id) != 0) {
1771 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)",
1772 slave_eth_dev->data->port_id, q_id, errval);
1776 if (internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id] != NULL)
1777 rte_flow_destroy(slave_eth_dev->data->port_id,
1778 internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id],
1781 bond_ethdev_8023ad_flow_set(bonded_eth_dev,
1782 slave_eth_dev->data->port_id);
1786 errval = rte_eth_dev_start(slave_eth_dev->data->port_id);
1788 RTE_BOND_LOG(ERR, "rte_eth_dev_start: port=%u, err (%d)",
1789 slave_eth_dev->data->port_id, errval);
1793 /* If RSS is enabled for bonding, synchronize RETA */
1794 if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
1796 struct bond_dev_private *internals;
1798 internals = bonded_eth_dev->data->dev_private;
1800 for (i = 0; i < internals->slave_count; i++) {
1801 if (internals->slaves[i].port_id == slave_eth_dev->data->port_id) {
1802 errval = rte_eth_dev_rss_reta_update(
1803 slave_eth_dev->data->port_id,
1804 &internals->reta_conf[0],
1805 internals->slaves[i].reta_size);
1807 RTE_BOND_LOG(WARNING,
1808 "rte_eth_dev_rss_reta_update on slave port %d fails (err %d)."
1809 " RSS Configuration for bonding may be inconsistent.",
1810 slave_eth_dev->data->port_id, errval);
1817 /* If lsc interrupt is set, check initial slave's link status */
1818 if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1819 slave_eth_dev->dev_ops->link_update(slave_eth_dev, 0);
1820 bond_ethdev_lsc_event_callback(slave_eth_dev->data->port_id,
1821 RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id,
1829 slave_remove(struct bond_dev_private *internals,
1830 struct rte_eth_dev *slave_eth_dev)
1834 for (i = 0; i < internals->slave_count; i++)
1835 if (internals->slaves[i].port_id ==
1836 slave_eth_dev->data->port_id)
1839 if (i < (internals->slave_count - 1)) {
1840 struct rte_flow *flow;
1842 memmove(&internals->slaves[i], &internals->slaves[i + 1],
1843 sizeof(internals->slaves[0]) *
1844 (internals->slave_count - i - 1));
1845 TAILQ_FOREACH(flow, &internals->flow_list, next) {
1846 memmove(&flow->flows[i], &flow->flows[i + 1],
1847 sizeof(flow->flows[0]) *
1848 (internals->slave_count - i - 1));
1849 flow->flows[internals->slave_count - 1] = NULL;
1853 internals->slave_count--;
1855 /* force reconfiguration of slave interfaces */
1856 _rte_eth_dev_reset(slave_eth_dev);
1860 bond_ethdev_slave_link_status_change_monitor(void *cb_arg);
1863 slave_add(struct bond_dev_private *internals,
1864 struct rte_eth_dev *slave_eth_dev)
1866 struct bond_slave_details *slave_details =
1867 &internals->slaves[internals->slave_count];
1869 slave_details->port_id = slave_eth_dev->data->port_id;
1870 slave_details->last_link_status = 0;
1872 /* Mark slave devices that don't support interrupts so we can
1873 * compensate when we start the bond
1875 if (!(slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
1876 slave_details->link_status_poll_enabled = 1;
1879 slave_details->link_status_wait_to_complete = 0;
1880 /* clean tlb_last_obytes when adding port for bonding device */
1881 memcpy(&(slave_details->persisted_mac_addr), slave_eth_dev->data->mac_addrs,
1882 sizeof(struct rte_ether_addr));
1886 bond_ethdev_primary_set(struct bond_dev_private *internals,
1887 uint16_t slave_port_id)
1891 if (internals->active_slave_count < 1)
1892 internals->current_primary_port = slave_port_id;
1894 /* Search bonded device slave ports for new proposed primary port */
1895 for (i = 0; i < internals->active_slave_count; i++) {
1896 if (internals->active_slaves[i] == slave_port_id)
1897 internals->current_primary_port = slave_port_id;
1902 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev);
1905 bond_ethdev_start(struct rte_eth_dev *eth_dev)
1907 struct bond_dev_private *internals;
1910 /* slave eth dev will be started by bonded device */
1911 if (check_for_bonded_ethdev(eth_dev)) {
1912 RTE_BOND_LOG(ERR, "User tried to explicitly start a slave eth_dev (%d)",
1913 eth_dev->data->port_id);
1917 eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
1918 eth_dev->data->dev_started = 1;
1920 internals = eth_dev->data->dev_private;
1922 if (internals->slave_count == 0) {
1923 RTE_BOND_LOG(ERR, "Cannot start port since there are no slave devices");
1927 if (internals->user_defined_mac == 0) {
1928 struct rte_ether_addr *new_mac_addr = NULL;
1930 for (i = 0; i < internals->slave_count; i++)
1931 if (internals->slaves[i].port_id == internals->primary_port)
1932 new_mac_addr = &internals->slaves[i].persisted_mac_addr;
1934 if (new_mac_addr == NULL)
1937 if (mac_address_set(eth_dev, new_mac_addr) != 0) {
1938 RTE_BOND_LOG(ERR, "bonded port (%d) failed to update MAC address",
1939 eth_dev->data->port_id);
1944 if (internals->mode == BONDING_MODE_8023AD) {
1945 if (internals->mode4.dedicated_queues.enabled == 1) {
1946 internals->mode4.dedicated_queues.rx_qid =
1947 eth_dev->data->nb_rx_queues;
1948 internals->mode4.dedicated_queues.tx_qid =
1949 eth_dev->data->nb_tx_queues;
1954 /* Reconfigure each slave device if starting bonded device */
1955 for (i = 0; i < internals->slave_count; i++) {
1956 struct rte_eth_dev *slave_ethdev =
1957 &(rte_eth_devices[internals->slaves[i].port_id]);
1958 if (slave_configure(eth_dev, slave_ethdev) != 0) {
1960 "bonded port (%d) failed to reconfigure slave device (%d)",
1961 eth_dev->data->port_id,
1962 internals->slaves[i].port_id);
1965 /* We will need to poll for link status if any slave doesn't
1966 * support interrupts
1968 if (internals->slaves[i].link_status_poll_enabled)
1969 internals->link_status_polling_enabled = 1;
1972 /* start polling if needed */
1973 if (internals->link_status_polling_enabled) {
1975 internals->link_status_polling_interval_ms * 1000,
1976 bond_ethdev_slave_link_status_change_monitor,
1977 (void *)&rte_eth_devices[internals->port_id]);
1980 /* Update all slave devices MACs*/
1981 if (mac_address_slaves_update(eth_dev) != 0)
1984 if (internals->user_defined_primary_port)
1985 bond_ethdev_primary_set(internals, internals->primary_port);
1987 if (internals->mode == BONDING_MODE_8023AD)
1988 bond_mode_8023ad_start(eth_dev);
1990 if (internals->mode == BONDING_MODE_TLB ||
1991 internals->mode == BONDING_MODE_ALB)
1992 bond_tlb_enable(internals);
1997 eth_dev->data->dev_started = 0;
2002 bond_ethdev_free_queues(struct rte_eth_dev *dev)
2006 if (dev->data->rx_queues != NULL) {
2007 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2008 rte_free(dev->data->rx_queues[i]);
2009 dev->data->rx_queues[i] = NULL;
2011 dev->data->nb_rx_queues = 0;
2014 if (dev->data->tx_queues != NULL) {
2015 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2016 rte_free(dev->data->tx_queues[i]);
2017 dev->data->tx_queues[i] = NULL;
2019 dev->data->nb_tx_queues = 0;
2024 bond_ethdev_stop(struct rte_eth_dev *eth_dev)
2026 struct bond_dev_private *internals = eth_dev->data->dev_private;
2029 if (internals->mode == BONDING_MODE_8023AD) {
2033 bond_mode_8023ad_stop(eth_dev);
2035 /* Discard all messages to/from mode 4 state machines */
2036 for (i = 0; i < internals->active_slave_count; i++) {
2037 port = &bond_mode_8023ad_ports[internals->active_slaves[i]];
2039 RTE_ASSERT(port->rx_ring != NULL);
2040 while (rte_ring_dequeue(port->rx_ring, &pkt) != -ENOENT)
2041 rte_pktmbuf_free(pkt);
2043 RTE_ASSERT(port->tx_ring != NULL);
2044 while (rte_ring_dequeue(port->tx_ring, &pkt) != -ENOENT)
2045 rte_pktmbuf_free(pkt);
2049 if (internals->mode == BONDING_MODE_TLB ||
2050 internals->mode == BONDING_MODE_ALB) {
2051 bond_tlb_disable(internals);
2052 for (i = 0; i < internals->active_slave_count; i++)
2053 tlb_last_obytets[internals->active_slaves[i]] = 0;
2056 eth_dev->data->dev_link.link_status = ETH_LINK_DOWN;
2057 eth_dev->data->dev_started = 0;
2059 internals->link_status_polling_enabled = 0;
2060 for (i = 0; i < internals->slave_count; i++) {
2061 uint16_t slave_id = internals->slaves[i].port_id;
2062 if (find_slave_by_id(internals->active_slaves,
2063 internals->active_slave_count, slave_id) !=
2064 internals->active_slave_count) {
2065 internals->slaves[i].last_link_status = 0;
2066 rte_eth_dev_stop(slave_id);
2067 deactivate_slave(eth_dev, slave_id);
2073 bond_ethdev_close(struct rte_eth_dev *dev)
2075 struct bond_dev_private *internals = dev->data->dev_private;
2076 uint16_t bond_port_id = internals->port_id;
2078 struct rte_flow_error ferror;
2080 RTE_BOND_LOG(INFO, "Closing bonded device %s", dev->device->name);
2081 while (internals->slave_count != skipped) {
2082 uint16_t port_id = internals->slaves[skipped].port_id;
2084 rte_eth_dev_stop(port_id);
2086 if (rte_eth_bond_slave_remove(bond_port_id, port_id) != 0) {
2088 "Failed to remove port %d from bonded device %s",
2089 port_id, dev->device->name);
2093 bond_flow_ops.flush(dev, &ferror);
2094 bond_ethdev_free_queues(dev);
2095 rte_bitmap_reset(internals->vlan_filter_bmp);
2098 /* forward declaration */
2099 static int bond_ethdev_configure(struct rte_eth_dev *dev);
2102 bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2104 struct bond_dev_private *internals = dev->data->dev_private;
2106 uint16_t max_nb_rx_queues = UINT16_MAX;
2107 uint16_t max_nb_tx_queues = UINT16_MAX;
2108 uint16_t max_rx_desc_lim = UINT16_MAX;
2109 uint16_t max_tx_desc_lim = UINT16_MAX;
2111 dev_info->max_mac_addrs = BOND_MAX_MAC_ADDRS;
2113 dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen ?
2114 internals->candidate_max_rx_pktlen :
2115 RTE_ETHER_MAX_JUMBO_FRAME_LEN;
2117 /* Max number of tx/rx queues that the bonded device can support is the
2118 * minimum values of the bonded slaves, as all slaves must be capable
2119 * of supporting the same number of tx/rx queues.
2121 if (internals->slave_count > 0) {
2122 struct rte_eth_dev_info slave_info;
2125 for (idx = 0; idx < internals->slave_count; idx++) {
2126 rte_eth_dev_info_get(internals->slaves[idx].port_id,
2129 if (slave_info.max_rx_queues < max_nb_rx_queues)
2130 max_nb_rx_queues = slave_info.max_rx_queues;
2132 if (slave_info.max_tx_queues < max_nb_tx_queues)
2133 max_nb_tx_queues = slave_info.max_tx_queues;
2135 if (slave_info.rx_desc_lim.nb_max < max_rx_desc_lim)
2136 max_rx_desc_lim = slave_info.rx_desc_lim.nb_max;
2138 if (slave_info.tx_desc_lim.nb_max < max_tx_desc_lim)
2139 max_tx_desc_lim = slave_info.tx_desc_lim.nb_max;
2143 dev_info->max_rx_queues = max_nb_rx_queues;
2144 dev_info->max_tx_queues = max_nb_tx_queues;
2146 memcpy(&dev_info->default_rxconf, &internals->default_rxconf,
2147 sizeof(dev_info->default_rxconf));
2148 memcpy(&dev_info->default_txconf, &internals->default_txconf,
2149 sizeof(dev_info->default_txconf));
2151 dev_info->rx_desc_lim.nb_max = max_rx_desc_lim;
2152 dev_info->tx_desc_lim.nb_max = max_tx_desc_lim;
2155 * If dedicated hw queues enabled for link bonding device in LACP mode
2156 * then we need to reduce the maximum number of data path queues by 1.
2158 if (internals->mode == BONDING_MODE_8023AD &&
2159 internals->mode4.dedicated_queues.enabled == 1) {
2160 dev_info->max_rx_queues--;
2161 dev_info->max_tx_queues--;
2164 dev_info->min_rx_bufsize = 0;
2166 dev_info->rx_offload_capa = internals->rx_offload_capa;
2167 dev_info->tx_offload_capa = internals->tx_offload_capa;
2168 dev_info->rx_queue_offload_capa = internals->rx_queue_offload_capa;
2169 dev_info->tx_queue_offload_capa = internals->tx_queue_offload_capa;
2170 dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads;
2172 dev_info->reta_size = internals->reta_size;
2176 bond_ethdev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2180 struct bond_dev_private *internals = dev->data->dev_private;
2182 /* don't do this while a slave is being added */
2183 rte_spinlock_lock(&internals->lock);
2186 rte_bitmap_set(internals->vlan_filter_bmp, vlan_id);
2188 rte_bitmap_clear(internals->vlan_filter_bmp, vlan_id);
2190 for (i = 0; i < internals->slave_count; i++) {
2191 uint16_t port_id = internals->slaves[i].port_id;
2193 res = rte_eth_dev_vlan_filter(port_id, vlan_id, on);
2195 RTE_BOND_LOG(WARNING,
2196 "Setting VLAN filter on slave port %u not supported.",
2200 rte_spinlock_unlock(&internals->lock);
2205 bond_ethdev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
2206 uint16_t nb_rx_desc, unsigned int socket_id __rte_unused,
2207 const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool)
2209 struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)
2210 rte_zmalloc_socket(NULL, sizeof(struct bond_rx_queue),
2211 0, dev->data->numa_node);
2212 if (bd_rx_q == NULL)
2215 bd_rx_q->queue_id = rx_queue_id;
2216 bd_rx_q->dev_private = dev->data->dev_private;
2218 bd_rx_q->nb_rx_desc = nb_rx_desc;
2220 memcpy(&(bd_rx_q->rx_conf), rx_conf, sizeof(struct rte_eth_rxconf));
2221 bd_rx_q->mb_pool = mb_pool;
2223 dev->data->rx_queues[rx_queue_id] = bd_rx_q;
2229 bond_ethdev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
2230 uint16_t nb_tx_desc, unsigned int socket_id __rte_unused,
2231 const struct rte_eth_txconf *tx_conf)
2233 struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)
2234 rte_zmalloc_socket(NULL, sizeof(struct bond_tx_queue),
2235 0, dev->data->numa_node);
2237 if (bd_tx_q == NULL)
2240 bd_tx_q->queue_id = tx_queue_id;
2241 bd_tx_q->dev_private = dev->data->dev_private;
2243 bd_tx_q->nb_tx_desc = nb_tx_desc;
2244 memcpy(&(bd_tx_q->tx_conf), tx_conf, sizeof(bd_tx_q->tx_conf));
2246 dev->data->tx_queues[tx_queue_id] = bd_tx_q;
2252 bond_ethdev_rx_queue_release(void *queue)
2261 bond_ethdev_tx_queue_release(void *queue)
2270 bond_ethdev_slave_link_status_change_monitor(void *cb_arg)
2272 struct rte_eth_dev *bonded_ethdev, *slave_ethdev;
2273 struct bond_dev_private *internals;
2275 /* Default value for polling slave found is true as we don't want to
2276 * disable the polling thread if we cannot get the lock */
2277 int i, polling_slave_found = 1;
2282 bonded_ethdev = cb_arg;
2283 internals = bonded_ethdev->data->dev_private;
2285 if (!bonded_ethdev->data->dev_started ||
2286 !internals->link_status_polling_enabled)
2289 /* If device is currently being configured then don't check slaves link
2290 * status, wait until next period */
2291 if (rte_spinlock_trylock(&internals->lock)) {
2292 if (internals->slave_count > 0)
2293 polling_slave_found = 0;
2295 for (i = 0; i < internals->slave_count; i++) {
2296 if (!internals->slaves[i].link_status_poll_enabled)
2299 slave_ethdev = &rte_eth_devices[internals->slaves[i].port_id];
2300 polling_slave_found = 1;
2302 /* Update slave link status */
2303 (*slave_ethdev->dev_ops->link_update)(slave_ethdev,
2304 internals->slaves[i].link_status_wait_to_complete);
2306 /* if link status has changed since last checked then call lsc
2308 if (slave_ethdev->data->dev_link.link_status !=
2309 internals->slaves[i].last_link_status) {
2310 internals->slaves[i].last_link_status =
2311 slave_ethdev->data->dev_link.link_status;
2313 bond_ethdev_lsc_event_callback(internals->slaves[i].port_id,
2314 RTE_ETH_EVENT_INTR_LSC,
2315 &bonded_ethdev->data->port_id,
2319 rte_spinlock_unlock(&internals->lock);
2322 if (polling_slave_found)
2323 /* Set alarm to continue monitoring link status of slave ethdev's */
2324 rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000,
2325 bond_ethdev_slave_link_status_change_monitor, cb_arg);
2329 bond_ethdev_link_update(struct rte_eth_dev *ethdev, int wait_to_complete)
2331 void (*link_update)(uint16_t port_id, struct rte_eth_link *eth_link);
2333 struct bond_dev_private *bond_ctx;
2334 struct rte_eth_link slave_link;
2338 bond_ctx = ethdev->data->dev_private;
2340 ethdev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
2342 if (ethdev->data->dev_started == 0 ||
2343 bond_ctx->active_slave_count == 0) {
2344 ethdev->data->dev_link.link_status = ETH_LINK_DOWN;
2348 ethdev->data->dev_link.link_status = ETH_LINK_UP;
2350 if (wait_to_complete)
2351 link_update = rte_eth_link_get;
2353 link_update = rte_eth_link_get_nowait;
2355 switch (bond_ctx->mode) {
2356 case BONDING_MODE_BROADCAST:
2358 * Setting link speed to UINT32_MAX to ensure we pick up the
2359 * value of the first active slave
2361 ethdev->data->dev_link.link_speed = UINT32_MAX;
2364 * link speed is minimum value of all the slaves link speed as
2365 * packet loss will occur on this slave if transmission at rates
2366 * greater than this are attempted
2368 for (idx = 1; idx < bond_ctx->active_slave_count; idx++) {
2369 link_update(bond_ctx->active_slaves[0], &slave_link);
2371 if (slave_link.link_speed <
2372 ethdev->data->dev_link.link_speed)
2373 ethdev->data->dev_link.link_speed =
2374 slave_link.link_speed;
2377 case BONDING_MODE_ACTIVE_BACKUP:
2378 /* Current primary slave */
2379 link_update(bond_ctx->current_primary_port, &slave_link);
2381 ethdev->data->dev_link.link_speed = slave_link.link_speed;
2383 case BONDING_MODE_8023AD:
2384 ethdev->data->dev_link.link_autoneg =
2385 bond_ctx->mode4.slave_link.link_autoneg;
2386 ethdev->data->dev_link.link_duplex =
2387 bond_ctx->mode4.slave_link.link_duplex;
2388 /* fall through to update link speed */
2389 case BONDING_MODE_ROUND_ROBIN:
2390 case BONDING_MODE_BALANCE:
2391 case BONDING_MODE_TLB:
2392 case BONDING_MODE_ALB:
2395 * In theses mode the maximum theoretical link speed is the sum
2398 ethdev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
2400 for (idx = 0; idx < bond_ctx->active_slave_count; idx++) {
2401 link_update(bond_ctx->active_slaves[idx], &slave_link);
2403 ethdev->data->dev_link.link_speed +=
2404 slave_link.link_speed;
2414 bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2416 struct bond_dev_private *internals = dev->data->dev_private;
2417 struct rte_eth_stats slave_stats;
2420 for (i = 0; i < internals->slave_count; i++) {
2421 rte_eth_stats_get(internals->slaves[i].port_id, &slave_stats);
2423 stats->ipackets += slave_stats.ipackets;
2424 stats->opackets += slave_stats.opackets;
2425 stats->ibytes += slave_stats.ibytes;
2426 stats->obytes += slave_stats.obytes;
2427 stats->imissed += slave_stats.imissed;
2428 stats->ierrors += slave_stats.ierrors;
2429 stats->oerrors += slave_stats.oerrors;
2430 stats->rx_nombuf += slave_stats.rx_nombuf;
2432 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
2433 stats->q_ipackets[j] += slave_stats.q_ipackets[j];
2434 stats->q_opackets[j] += slave_stats.q_opackets[j];
2435 stats->q_ibytes[j] += slave_stats.q_ibytes[j];
2436 stats->q_obytes[j] += slave_stats.q_obytes[j];
2437 stats->q_errors[j] += slave_stats.q_errors[j];
2446 bond_ethdev_stats_reset(struct rte_eth_dev *dev)
2448 struct bond_dev_private *internals = dev->data->dev_private;
2451 for (i = 0; i < internals->slave_count; i++)
2452 rte_eth_stats_reset(internals->slaves[i].port_id);
2456 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev)
2458 struct bond_dev_private *internals = eth_dev->data->dev_private;
2461 switch (internals->mode) {
2462 /* Promiscuous mode is propagated to all slaves */
2463 case BONDING_MODE_ROUND_ROBIN:
2464 case BONDING_MODE_BALANCE:
2465 case BONDING_MODE_BROADCAST:
2466 case BONDING_MODE_8023AD:
2467 for (i = 0; i < internals->slave_count; i++) {
2468 uint16_t port_id = internals->slaves[i].port_id;
2470 rte_eth_promiscuous_enable(port_id);
2473 /* Promiscuous mode is propagated only to primary slave */
2474 case BONDING_MODE_ACTIVE_BACKUP:
2475 case BONDING_MODE_TLB:
2476 case BONDING_MODE_ALB:
2478 /* Do not touch promisc when there cannot be primary ports */
2479 if (internals->slave_count == 0)
2481 rte_eth_promiscuous_enable(internals->current_primary_port);
2486 bond_ethdev_promiscuous_disable(struct rte_eth_dev *dev)
2488 struct bond_dev_private *internals = dev->data->dev_private;
2491 switch (internals->mode) {
2492 /* Promiscuous mode is propagated to all slaves */
2493 case BONDING_MODE_ROUND_ROBIN:
2494 case BONDING_MODE_BALANCE:
2495 case BONDING_MODE_BROADCAST:
2496 case BONDING_MODE_8023AD:
2497 for (i = 0; i < internals->slave_count; i++) {
2498 uint16_t port_id = internals->slaves[i].port_id;
2500 if (internals->mode == BONDING_MODE_8023AD &&
2501 bond_mode_8023ad_ports[port_id].forced_rx_flags ==
2502 BOND_8023AD_FORCED_PROMISC)
2504 rte_eth_promiscuous_disable(port_id);
2507 /* Promiscuous mode is propagated only to primary slave */
2508 case BONDING_MODE_ACTIVE_BACKUP:
2509 case BONDING_MODE_TLB:
2510 case BONDING_MODE_ALB:
2512 /* Do not touch promisc when there cannot be primary ports */
2513 if (internals->slave_count == 0)
2515 rte_eth_promiscuous_disable(internals->current_primary_port);
2520 bond_ethdev_allmulticast_enable(struct rte_eth_dev *eth_dev)
2522 struct bond_dev_private *internals = eth_dev->data->dev_private;
2525 switch (internals->mode) {
2526 /* allmulti mode is propagated to all slaves */
2527 case BONDING_MODE_ROUND_ROBIN:
2528 case BONDING_MODE_BALANCE:
2529 case BONDING_MODE_BROADCAST:
2530 case BONDING_MODE_8023AD:
2531 for (i = 0; i < internals->slave_count; i++) {
2532 uint16_t port_id = internals->slaves[i].port_id;
2534 rte_eth_allmulticast_enable(port_id);
2537 /* allmulti mode is propagated only to primary slave */
2538 case BONDING_MODE_ACTIVE_BACKUP:
2539 case BONDING_MODE_TLB:
2540 case BONDING_MODE_ALB:
2542 /* Do not touch allmulti when there cannot be primary ports */
2543 if (internals->slave_count == 0)
2545 rte_eth_allmulticast_enable(internals->current_primary_port);
2550 bond_ethdev_allmulticast_disable(struct rte_eth_dev *eth_dev)
2552 struct bond_dev_private *internals = eth_dev->data->dev_private;
2555 switch (internals->mode) {
2556 /* allmulti mode is propagated to all slaves */
2557 case BONDING_MODE_ROUND_ROBIN:
2558 case BONDING_MODE_BALANCE:
2559 case BONDING_MODE_BROADCAST:
2560 case BONDING_MODE_8023AD:
2561 for (i = 0; i < internals->slave_count; i++) {
2562 uint16_t port_id = internals->slaves[i].port_id;
2564 if (internals->mode == BONDING_MODE_8023AD &&
2565 bond_mode_8023ad_ports[port_id].forced_rx_flags ==
2566 BOND_8023AD_FORCED_ALLMULTI)
2568 rte_eth_allmulticast_disable(port_id);
2571 /* allmulti mode is propagated only to primary slave */
2572 case BONDING_MODE_ACTIVE_BACKUP:
2573 case BONDING_MODE_TLB:
2574 case BONDING_MODE_ALB:
2576 /* Do not touch allmulti when there cannot be primary ports */
2577 if (internals->slave_count == 0)
2579 rte_eth_allmulticast_disable(internals->current_primary_port);
2584 bond_ethdev_delayed_lsc_propagation(void *arg)
2589 _rte_eth_dev_callback_process((struct rte_eth_dev *)arg,
2590 RTE_ETH_EVENT_INTR_LSC, NULL);
2594 bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type,
2595 void *param, void *ret_param __rte_unused)
2597 struct rte_eth_dev *bonded_eth_dev;
2598 struct bond_dev_private *internals;
2599 struct rte_eth_link link;
2602 uint8_t lsc_flag = 0;
2603 int valid_slave = 0;
2604 uint16_t active_pos;
2607 if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL)
2610 bonded_eth_dev = &rte_eth_devices[*(uint16_t *)param];
2612 if (check_for_bonded_ethdev(bonded_eth_dev))
2615 internals = bonded_eth_dev->data->dev_private;
2617 /* If the device isn't started don't handle interrupts */
2618 if (!bonded_eth_dev->data->dev_started)
2621 /* verify that port_id is a valid slave of bonded port */
2622 for (i = 0; i < internals->slave_count; i++) {
2623 if (internals->slaves[i].port_id == port_id) {
2632 /* Synchronize lsc callback parallel calls either by real link event
2633 * from the slaves PMDs or by the bonding PMD itself.
2635 rte_spinlock_lock(&internals->lsc_lock);
2637 /* Search for port in active port list */
2638 active_pos = find_slave_by_id(internals->active_slaves,
2639 internals->active_slave_count, port_id);
2641 rte_eth_link_get_nowait(port_id, &link);
2642 if (link.link_status) {
2643 if (active_pos < internals->active_slave_count)
2646 /* check link state properties if bonded link is up*/
2647 if (bonded_eth_dev->data->dev_link.link_status == ETH_LINK_UP) {
2648 if (link_properties_valid(bonded_eth_dev, &link) != 0)
2649 RTE_BOND_LOG(ERR, "Invalid link properties "
2650 "for slave %d in bonding mode %d",
2651 port_id, internals->mode);
2653 /* inherit slave link properties */
2654 link_properties_set(bonded_eth_dev, &link);
2657 /* If no active slave ports then set this port to be
2660 if (internals->active_slave_count < 1) {
2661 /* If first active slave, then change link status */
2662 bonded_eth_dev->data->dev_link.link_status =
2664 internals->current_primary_port = port_id;
2667 mac_address_slaves_update(bonded_eth_dev);
2670 activate_slave(bonded_eth_dev, port_id);
2672 /* If the user has defined the primary port then default to
2675 if (internals->user_defined_primary_port &&
2676 internals->primary_port == port_id)
2677 bond_ethdev_primary_set(internals, port_id);
2679 if (active_pos == internals->active_slave_count)
2682 /* Remove from active slave list */
2683 deactivate_slave(bonded_eth_dev, port_id);
2685 if (internals->active_slave_count < 1)
2688 /* Update primary id, take first active slave from list or if none
2689 * available set to -1 */
2690 if (port_id == internals->current_primary_port) {
2691 if (internals->active_slave_count > 0)
2692 bond_ethdev_primary_set(internals,
2693 internals->active_slaves[0]);
2695 internals->current_primary_port = internals->primary_port;
2701 * Update bonded device link properties after any change to active
2704 bond_ethdev_link_update(bonded_eth_dev, 0);
2707 /* Cancel any possible outstanding interrupts if delays are enabled */
2708 if (internals->link_up_delay_ms > 0 ||
2709 internals->link_down_delay_ms > 0)
2710 rte_eal_alarm_cancel(bond_ethdev_delayed_lsc_propagation,
2713 if (bonded_eth_dev->data->dev_link.link_status) {
2714 if (internals->link_up_delay_ms > 0)
2715 rte_eal_alarm_set(internals->link_up_delay_ms * 1000,
2716 bond_ethdev_delayed_lsc_propagation,
2717 (void *)bonded_eth_dev);
2719 _rte_eth_dev_callback_process(bonded_eth_dev,
2720 RTE_ETH_EVENT_INTR_LSC,
2724 if (internals->link_down_delay_ms > 0)
2725 rte_eal_alarm_set(internals->link_down_delay_ms * 1000,
2726 bond_ethdev_delayed_lsc_propagation,
2727 (void *)bonded_eth_dev);
2729 _rte_eth_dev_callback_process(bonded_eth_dev,
2730 RTE_ETH_EVENT_INTR_LSC,
2735 rte_spinlock_unlock(&internals->lsc_lock);
2741 bond_ethdev_rss_reta_update(struct rte_eth_dev *dev,
2742 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size)
2746 int slave_reta_size;
2747 unsigned reta_count;
2748 struct bond_dev_private *internals = dev->data->dev_private;
2750 if (reta_size != internals->reta_size)
2753 /* Copy RETA table */
2754 reta_count = reta_size / RTE_RETA_GROUP_SIZE;
2756 for (i = 0; i < reta_count; i++) {
2757 internals->reta_conf[i].mask = reta_conf[i].mask;
2758 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
2759 if ((reta_conf[i].mask >> j) & 0x01)
2760 internals->reta_conf[i].reta[j] = reta_conf[i].reta[j];
2763 /* Fill rest of array */
2764 for (; i < RTE_DIM(internals->reta_conf); i += reta_count)
2765 memcpy(&internals->reta_conf[i], &internals->reta_conf[0],
2766 sizeof(internals->reta_conf[0]) * reta_count);
2768 /* Propagate RETA over slaves */
2769 for (i = 0; i < internals->slave_count; i++) {
2770 slave_reta_size = internals->slaves[i].reta_size;
2771 result = rte_eth_dev_rss_reta_update(internals->slaves[i].port_id,
2772 &internals->reta_conf[0], slave_reta_size);
2781 bond_ethdev_rss_reta_query(struct rte_eth_dev *dev,
2782 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size)
2785 struct bond_dev_private *internals = dev->data->dev_private;
2787 if (reta_size != internals->reta_size)
2790 /* Copy RETA table */
2791 for (i = 0; i < reta_size / RTE_RETA_GROUP_SIZE; i++)
2792 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
2793 if ((reta_conf[i].mask >> j) & 0x01)
2794 reta_conf[i].reta[j] = internals->reta_conf[i].reta[j];
2800 bond_ethdev_rss_hash_update(struct rte_eth_dev *dev,
2801 struct rte_eth_rss_conf *rss_conf)
2804 struct bond_dev_private *internals = dev->data->dev_private;
2805 struct rte_eth_rss_conf bond_rss_conf;
2807 memcpy(&bond_rss_conf, rss_conf, sizeof(struct rte_eth_rss_conf));
2809 bond_rss_conf.rss_hf &= internals->flow_type_rss_offloads;
2811 if (bond_rss_conf.rss_hf != 0)
2812 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf = bond_rss_conf.rss_hf;
2814 if (bond_rss_conf.rss_key && bond_rss_conf.rss_key_len <
2815 sizeof(internals->rss_key)) {
2816 if (bond_rss_conf.rss_key_len == 0)
2817 bond_rss_conf.rss_key_len = 40;
2818 internals->rss_key_len = bond_rss_conf.rss_key_len;
2819 memcpy(internals->rss_key, bond_rss_conf.rss_key,
2820 internals->rss_key_len);
2823 for (i = 0; i < internals->slave_count; i++) {
2824 result = rte_eth_dev_rss_hash_update(internals->slaves[i].port_id,
2834 bond_ethdev_rss_hash_conf_get(struct rte_eth_dev *dev,
2835 struct rte_eth_rss_conf *rss_conf)
2837 struct bond_dev_private *internals = dev->data->dev_private;
2839 rss_conf->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
2840 rss_conf->rss_key_len = internals->rss_key_len;
2841 if (rss_conf->rss_key)
2842 memcpy(rss_conf->rss_key, internals->rss_key, internals->rss_key_len);
2848 bond_ethdev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
2850 struct rte_eth_dev *slave_eth_dev;
2851 struct bond_dev_private *internals = dev->data->dev_private;
2854 rte_spinlock_lock(&internals->lock);
2856 for (i = 0; i < internals->slave_count; i++) {
2857 slave_eth_dev = &rte_eth_devices[internals->slaves[i].port_id];
2858 if (*slave_eth_dev->dev_ops->mtu_set == NULL) {
2859 rte_spinlock_unlock(&internals->lock);
2863 for (i = 0; i < internals->slave_count; i++) {
2864 ret = rte_eth_dev_set_mtu(internals->slaves[i].port_id, mtu);
2866 rte_spinlock_unlock(&internals->lock);
2871 rte_spinlock_unlock(&internals->lock);
2876 bond_ethdev_mac_address_set(struct rte_eth_dev *dev,
2877 struct rte_ether_addr *addr)
2879 if (mac_address_set(dev, addr)) {
2880 RTE_BOND_LOG(ERR, "Failed to update MAC address");
2888 bond_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
2889 enum rte_filter_type type, enum rte_filter_op op, void *arg)
2891 if (type == RTE_ETH_FILTER_GENERIC && op == RTE_ETH_FILTER_GET) {
2892 *(const void **)arg = &bond_flow_ops;
2899 bond_ethdev_mac_addr_add(struct rte_eth_dev *dev,
2900 struct rte_ether_addr *mac_addr,
2901 __rte_unused uint32_t index, uint32_t vmdq)
2903 struct rte_eth_dev *slave_eth_dev;
2904 struct bond_dev_private *internals = dev->data->dev_private;
2907 rte_spinlock_lock(&internals->lock);
2909 for (i = 0; i < internals->slave_count; i++) {
2910 slave_eth_dev = &rte_eth_devices[internals->slaves[i].port_id];
2911 if (*slave_eth_dev->dev_ops->mac_addr_add == NULL ||
2912 *slave_eth_dev->dev_ops->mac_addr_remove == NULL) {
2918 for (i = 0; i < internals->slave_count; i++) {
2919 ret = rte_eth_dev_mac_addr_add(internals->slaves[i].port_id,
2923 for (i--; i >= 0; i--)
2924 rte_eth_dev_mac_addr_remove(
2925 internals->slaves[i].port_id, mac_addr);
2932 rte_spinlock_unlock(&internals->lock);
2937 bond_ethdev_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
2939 struct rte_eth_dev *slave_eth_dev;
2940 struct bond_dev_private *internals = dev->data->dev_private;
2943 rte_spinlock_lock(&internals->lock);
2945 for (i = 0; i < internals->slave_count; i++) {
2946 slave_eth_dev = &rte_eth_devices[internals->slaves[i].port_id];
2947 if (*slave_eth_dev->dev_ops->mac_addr_remove == NULL)
2951 struct rte_ether_addr *mac_addr = &dev->data->mac_addrs[index];
2953 for (i = 0; i < internals->slave_count; i++)
2954 rte_eth_dev_mac_addr_remove(internals->slaves[i].port_id,
2958 rte_spinlock_unlock(&internals->lock);
2961 const struct eth_dev_ops default_dev_ops = {
2962 .dev_start = bond_ethdev_start,
2963 .dev_stop = bond_ethdev_stop,
2964 .dev_close = bond_ethdev_close,
2965 .dev_configure = bond_ethdev_configure,
2966 .dev_infos_get = bond_ethdev_info,
2967 .vlan_filter_set = bond_ethdev_vlan_filter_set,
2968 .rx_queue_setup = bond_ethdev_rx_queue_setup,
2969 .tx_queue_setup = bond_ethdev_tx_queue_setup,
2970 .rx_queue_release = bond_ethdev_rx_queue_release,
2971 .tx_queue_release = bond_ethdev_tx_queue_release,
2972 .link_update = bond_ethdev_link_update,
2973 .stats_get = bond_ethdev_stats_get,
2974 .stats_reset = bond_ethdev_stats_reset,
2975 .promiscuous_enable = bond_ethdev_promiscuous_enable,
2976 .promiscuous_disable = bond_ethdev_promiscuous_disable,
2977 .allmulticast_enable = bond_ethdev_allmulticast_enable,
2978 .allmulticast_disable = bond_ethdev_allmulticast_disable,
2979 .reta_update = bond_ethdev_rss_reta_update,
2980 .reta_query = bond_ethdev_rss_reta_query,
2981 .rss_hash_update = bond_ethdev_rss_hash_update,
2982 .rss_hash_conf_get = bond_ethdev_rss_hash_conf_get,
2983 .mtu_set = bond_ethdev_mtu_set,
2984 .mac_addr_set = bond_ethdev_mac_address_set,
2985 .mac_addr_add = bond_ethdev_mac_addr_add,
2986 .mac_addr_remove = bond_ethdev_mac_addr_remove,
2987 .filter_ctrl = bond_filter_ctrl
2991 bond_alloc(struct rte_vdev_device *dev, uint8_t mode)
2993 const char *name = rte_vdev_device_name(dev);
2994 uint8_t socket_id = dev->device.numa_node;
2995 struct bond_dev_private *internals = NULL;
2996 struct rte_eth_dev *eth_dev = NULL;
2997 uint32_t vlan_filter_bmp_size;
2999 /* now do all data allocation - for eth_dev structure, dummy pci driver
3000 * and internal (private) data
3003 /* reserve an ethdev entry */
3004 eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals));
3005 if (eth_dev == NULL) {
3006 RTE_BOND_LOG(ERR, "Unable to allocate rte_eth_dev");
3010 internals = eth_dev->data->dev_private;
3011 eth_dev->data->nb_rx_queues = (uint16_t)1;
3012 eth_dev->data->nb_tx_queues = (uint16_t)1;
3014 /* Allocate memory for storing MAC addresses */
3015 eth_dev->data->mac_addrs = rte_zmalloc_socket(name, RTE_ETHER_ADDR_LEN *
3016 BOND_MAX_MAC_ADDRS, 0, socket_id);
3017 if (eth_dev->data->mac_addrs == NULL) {
3019 "Failed to allocate %u bytes needed to store MAC addresses",
3020 RTE_ETHER_ADDR_LEN * BOND_MAX_MAC_ADDRS);
3024 eth_dev->dev_ops = &default_dev_ops;
3025 eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC;
3027 rte_spinlock_init(&internals->lock);
3028 rte_spinlock_init(&internals->lsc_lock);
3030 internals->port_id = eth_dev->data->port_id;
3031 internals->mode = BONDING_MODE_INVALID;
3032 internals->current_primary_port = RTE_MAX_ETHPORTS + 1;
3033 internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2;
3034 internals->burst_xmit_hash = burst_xmit_l2_hash;
3035 internals->user_defined_mac = 0;
3037 internals->link_status_polling_enabled = 0;
3039 internals->link_status_polling_interval_ms =
3040 DEFAULT_POLLING_INTERVAL_10_MS;
3041 internals->link_down_delay_ms = 0;
3042 internals->link_up_delay_ms = 0;
3044 internals->slave_count = 0;
3045 internals->active_slave_count = 0;
3046 internals->rx_offload_capa = 0;
3047 internals->tx_offload_capa = 0;
3048 internals->rx_queue_offload_capa = 0;
3049 internals->tx_queue_offload_capa = 0;
3050 internals->candidate_max_rx_pktlen = 0;
3051 internals->max_rx_pktlen = 0;
3053 /* Initially allow to choose any offload type */
3054 internals->flow_type_rss_offloads = ETH_RSS_PROTO_MASK;
3056 memset(&internals->default_rxconf, 0,
3057 sizeof(internals->default_rxconf));
3058 memset(&internals->default_txconf, 0,
3059 sizeof(internals->default_txconf));
3061 memset(&internals->rx_desc_lim, 0, sizeof(internals->rx_desc_lim));
3062 memset(&internals->tx_desc_lim, 0, sizeof(internals->tx_desc_lim));
3064 memset(internals->active_slaves, 0, sizeof(internals->active_slaves));
3065 memset(internals->slaves, 0, sizeof(internals->slaves));
3067 TAILQ_INIT(&internals->flow_list);
3068 internals->flow_isolated_valid = 0;
3070 /* Set mode 4 default configuration */
3071 bond_mode_8023ad_setup(eth_dev, NULL);
3072 if (bond_ethdev_mode_set(eth_dev, mode)) {
3073 RTE_BOND_LOG(ERR, "Failed to set bonded device %d mode to %d",
3074 eth_dev->data->port_id, mode);
3078 vlan_filter_bmp_size =
3079 rte_bitmap_get_memory_footprint(RTE_ETHER_MAX_VLAN_ID + 1);
3080 internals->vlan_filter_bmpmem = rte_malloc(name, vlan_filter_bmp_size,
3081 RTE_CACHE_LINE_SIZE);
3082 if (internals->vlan_filter_bmpmem == NULL) {
3084 "Failed to allocate vlan bitmap for bonded device %u",
3085 eth_dev->data->port_id);
3089 internals->vlan_filter_bmp = rte_bitmap_init(RTE_ETHER_MAX_VLAN_ID + 1,
3090 internals->vlan_filter_bmpmem, vlan_filter_bmp_size);
3091 if (internals->vlan_filter_bmp == NULL) {
3093 "Failed to init vlan bitmap for bonded device %u",
3094 eth_dev->data->port_id);
3095 rte_free(internals->vlan_filter_bmpmem);
3099 return eth_dev->data->port_id;
3102 rte_free(internals);
3103 if (eth_dev != NULL)
3104 eth_dev->data->dev_private = NULL;
3105 rte_eth_dev_release_port(eth_dev);
3110 bond_probe(struct rte_vdev_device *dev)
3113 struct bond_dev_private *internals;
3114 struct rte_kvargs *kvlist;
3115 uint8_t bonding_mode, socket_id/*, agg_mode*/;
3116 int arg_count, port_id;
3118 struct rte_eth_dev *eth_dev;
3123 name = rte_vdev_device_name(dev);
3124 RTE_BOND_LOG(INFO, "Initializing pmd_bond for %s", name);
3126 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
3127 eth_dev = rte_eth_dev_attach_secondary(name);
3129 RTE_BOND_LOG(ERR, "Failed to probe %s", name);
3132 /* TODO: request info from primary to set up Rx and Tx */
3133 eth_dev->dev_ops = &default_dev_ops;
3134 eth_dev->device = &dev->device;
3135 rte_eth_dev_probing_finish(eth_dev);
3139 kvlist = rte_kvargs_parse(rte_vdev_device_args(dev),
3140 pmd_bond_init_valid_arguments);
3144 /* Parse link bonding mode */
3145 if (rte_kvargs_count(kvlist, PMD_BOND_MODE_KVARG) == 1) {
3146 if (rte_kvargs_process(kvlist, PMD_BOND_MODE_KVARG,
3147 &bond_ethdev_parse_slave_mode_kvarg,
3148 &bonding_mode) != 0) {
3149 RTE_BOND_LOG(ERR, "Invalid mode for bonded device %s",
3154 RTE_BOND_LOG(ERR, "Mode must be specified only once for bonded "
3159 /* Parse socket id to create bonding device on */
3160 arg_count = rte_kvargs_count(kvlist, PMD_BOND_SOCKET_ID_KVARG);
3161 if (arg_count == 1) {
3162 if (rte_kvargs_process(kvlist, PMD_BOND_SOCKET_ID_KVARG,
3163 &bond_ethdev_parse_socket_id_kvarg, &socket_id)
3165 RTE_BOND_LOG(ERR, "Invalid socket Id specified for "
3166 "bonded device %s", name);
3169 } else if (arg_count > 1) {
3170 RTE_BOND_LOG(ERR, "Socket Id can be specified only once for "
3171 "bonded device %s", name);
3174 socket_id = rte_socket_id();
3177 dev->device.numa_node = socket_id;
3179 /* Create link bonding eth device */
3180 port_id = bond_alloc(dev, bonding_mode);
3182 RTE_BOND_LOG(ERR, "Failed to create socket %s in mode %u on "
3183 "socket %u.", name, bonding_mode, socket_id);
3186 internals = rte_eth_devices[port_id].data->dev_private;
3187 internals->kvlist = kvlist;
3189 if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) {
3190 if (rte_kvargs_process(kvlist,
3191 PMD_BOND_AGG_MODE_KVARG,
3192 &bond_ethdev_parse_slave_agg_mode_kvarg,
3195 "Failed to parse agg selection mode for bonded device %s",
3200 if (internals->mode == BONDING_MODE_8023AD)
3201 internals->mode4.agg_selection = agg_mode;
3203 internals->mode4.agg_selection = AGG_STABLE;
3206 rte_eth_dev_probing_finish(&rte_eth_devices[port_id]);
3207 RTE_BOND_LOG(INFO, "Create bonded device %s on port %d in mode %u on "
3208 "socket %u.", name, port_id, bonding_mode, socket_id);
3212 rte_kvargs_free(kvlist);
3218 bond_remove(struct rte_vdev_device *dev)
3220 struct rte_eth_dev *eth_dev;
3221 struct bond_dev_private *internals;
3227 name = rte_vdev_device_name(dev);
3228 RTE_BOND_LOG(INFO, "Uninitializing pmd_bond for %s", name);
3230 /* now free all data allocation - for eth_dev structure,
3231 * dummy pci driver and internal (private) data
3234 /* find an ethdev entry */
3235 eth_dev = rte_eth_dev_allocated(name);
3236 if (eth_dev == NULL)
3239 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3240 return rte_eth_dev_release_port(eth_dev);
3242 RTE_ASSERT(eth_dev->device == &dev->device);
3244 internals = eth_dev->data->dev_private;
3245 if (internals->slave_count != 0)
3248 if (eth_dev->data->dev_started == 1) {
3249 bond_ethdev_stop(eth_dev);
3250 bond_ethdev_close(eth_dev);
3253 eth_dev->dev_ops = NULL;
3254 eth_dev->rx_pkt_burst = NULL;
3255 eth_dev->tx_pkt_burst = NULL;
3257 internals = eth_dev->data->dev_private;
3258 /* Try to release mempool used in mode6. If the bond
3259 * device is not mode6, free the NULL is not problem.
3261 rte_mempool_free(internals->mode6.mempool);
3262 rte_bitmap_free(internals->vlan_filter_bmp);
3263 rte_free(internals->vlan_filter_bmpmem);
3265 rte_eth_dev_release_port(eth_dev);
3270 /* this part will resolve the slave portids after all the other pdev and vdev
3271 * have been allocated */
3273 bond_ethdev_configure(struct rte_eth_dev *dev)
3275 const char *name = dev->device->name;
3276 struct bond_dev_private *internals = dev->data->dev_private;
3277 struct rte_kvargs *kvlist = internals->kvlist;
3279 uint16_t port_id = dev - rte_eth_devices;
3282 static const uint8_t default_rss_key[40] = {
3283 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D,
3284 0x43, 0xA3, 0x8F, 0xB0, 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
3285 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 0x6A, 0x42, 0xB7, 0x3B,
3286 0xBE, 0xAC, 0x01, 0xFA
3292 * If RSS is enabled, fill table with default values and
3293 * set key to the the value specified in port RSS configuration.
3294 * Fall back to default RSS key if the key is not specified
3296 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) {
3297 if (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key != NULL) {
3298 internals->rss_key_len =
3299 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len;
3300 memcpy(internals->rss_key,
3301 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key,
3302 internals->rss_key_len);
3304 internals->rss_key_len = sizeof(default_rss_key);
3305 memcpy(internals->rss_key, default_rss_key,
3306 internals->rss_key_len);
3309 for (i = 0; i < RTE_DIM(internals->reta_conf); i++) {
3310 internals->reta_conf[i].mask = ~0LL;
3311 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
3312 internals->reta_conf[i].reta[j] =
3313 (i * RTE_RETA_GROUP_SIZE + j) %
3314 dev->data->nb_rx_queues;
3318 /* set the max_rx_pktlen */
3319 internals->max_rx_pktlen = internals->candidate_max_rx_pktlen;
3322 * if no kvlist, it means that this bonded device has been created
3323 * through the bonding api.
3328 /* Parse MAC address for bonded device */
3329 arg_count = rte_kvargs_count(kvlist, PMD_BOND_MAC_ADDR_KVARG);
3330 if (arg_count == 1) {
3331 struct rte_ether_addr bond_mac;
3333 if (rte_kvargs_process(kvlist, PMD_BOND_MAC_ADDR_KVARG,
3334 &bond_ethdev_parse_bond_mac_addr_kvarg, &bond_mac) < 0) {
3335 RTE_BOND_LOG(INFO, "Invalid mac address for bonded device %s",
3340 /* Set MAC address */
3341 if (rte_eth_bond_mac_address_set(port_id, &bond_mac) != 0) {
3343 "Failed to set mac address on bonded device %s",
3347 } else if (arg_count > 1) {
3349 "MAC address can be specified only once for bonded device %s",
3354 /* Parse/set balance mode transmit policy */
3355 arg_count = rte_kvargs_count(kvlist, PMD_BOND_XMIT_POLICY_KVARG);
3356 if (arg_count == 1) {
3357 uint8_t xmit_policy;
3359 if (rte_kvargs_process(kvlist, PMD_BOND_XMIT_POLICY_KVARG,
3360 &bond_ethdev_parse_balance_xmit_policy_kvarg, &xmit_policy) !=
3363 "Invalid xmit policy specified for bonded device %s",
3368 /* Set balance mode transmit policy*/
3369 if (rte_eth_bond_xmit_policy_set(port_id, xmit_policy) != 0) {
3371 "Failed to set balance xmit policy on bonded device %s",
3375 } else if (arg_count > 1) {
3377 "Transmit policy can be specified only once for bonded device %s",
3382 if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) {
3383 if (rte_kvargs_process(kvlist,
3384 PMD_BOND_AGG_MODE_KVARG,
3385 &bond_ethdev_parse_slave_agg_mode_kvarg,
3388 "Failed to parse agg selection mode for bonded device %s",
3391 if (internals->mode == BONDING_MODE_8023AD) {
3392 int ret = rte_eth_bond_8023ad_agg_selection_set(port_id,
3396 "Invalid args for agg selection set for bonded device %s",
3403 /* Parse/add slave ports to bonded device */
3404 if (rte_kvargs_count(kvlist, PMD_BOND_SLAVE_PORT_KVARG) > 0) {
3405 struct bond_ethdev_slave_ports slave_ports;
3408 memset(&slave_ports, 0, sizeof(slave_ports));
3410 if (rte_kvargs_process(kvlist, PMD_BOND_SLAVE_PORT_KVARG,
3411 &bond_ethdev_parse_slave_port_kvarg, &slave_ports) != 0) {
3413 "Failed to parse slave ports for bonded device %s",
3418 for (i = 0; i < slave_ports.slave_count; i++) {
3419 if (rte_eth_bond_slave_add(port_id, slave_ports.slaves[i]) != 0) {
3421 "Failed to add port %d as slave to bonded device %s",
3422 slave_ports.slaves[i], name);
3427 RTE_BOND_LOG(INFO, "No slaves specified for bonded device %s", name);
3431 /* Parse/set primary slave port id*/
3432 arg_count = rte_kvargs_count(kvlist, PMD_BOND_PRIMARY_SLAVE_KVARG);
3433 if (arg_count == 1) {
3434 uint16_t primary_slave_port_id;
3436 if (rte_kvargs_process(kvlist,
3437 PMD_BOND_PRIMARY_SLAVE_KVARG,
3438 &bond_ethdev_parse_primary_slave_port_id_kvarg,
3439 &primary_slave_port_id) < 0) {
3441 "Invalid primary slave port id specified for bonded device %s",
3446 /* Set balance mode transmit policy*/
3447 if (rte_eth_bond_primary_set(port_id, primary_slave_port_id)
3450 "Failed to set primary slave port %d on bonded device %s",
3451 primary_slave_port_id, name);
3454 } else if (arg_count > 1) {
3456 "Primary slave can be specified only once for bonded device %s",
3461 /* Parse link status monitor polling interval */
3462 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LSC_POLL_PERIOD_KVARG);
3463 if (arg_count == 1) {
3464 uint32_t lsc_poll_interval_ms;
3466 if (rte_kvargs_process(kvlist,
3467 PMD_BOND_LSC_POLL_PERIOD_KVARG,
3468 &bond_ethdev_parse_time_ms_kvarg,
3469 &lsc_poll_interval_ms) < 0) {
3471 "Invalid lsc polling interval value specified for bonded"
3472 " device %s", name);
3476 if (rte_eth_bond_link_monitoring_set(port_id, lsc_poll_interval_ms)
3479 "Failed to set lsc monitor polling interval (%u ms) on bonded device %s",
3480 lsc_poll_interval_ms, name);
3483 } else if (arg_count > 1) {
3485 "LSC polling interval can be specified only once for bonded"
3486 " device %s", name);
3490 /* Parse link up interrupt propagation delay */
3491 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_UP_PROP_DELAY_KVARG);
3492 if (arg_count == 1) {
3493 uint32_t link_up_delay_ms;
3495 if (rte_kvargs_process(kvlist,
3496 PMD_BOND_LINK_UP_PROP_DELAY_KVARG,
3497 &bond_ethdev_parse_time_ms_kvarg,
3498 &link_up_delay_ms) < 0) {
3500 "Invalid link up propagation delay value specified for"
3501 " bonded device %s", name);
3505 /* Set balance mode transmit policy*/
3506 if (rte_eth_bond_link_up_prop_delay_set(port_id, link_up_delay_ms)
3509 "Failed to set link up propagation delay (%u ms) on bonded"
3510 " device %s", link_up_delay_ms, name);
3513 } else if (arg_count > 1) {
3515 "Link up propagation delay can be specified only once for"
3516 " bonded device %s", name);
3520 /* Parse link down interrupt propagation delay */
3521 arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG);
3522 if (arg_count == 1) {
3523 uint32_t link_down_delay_ms;
3525 if (rte_kvargs_process(kvlist,
3526 PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG,
3527 &bond_ethdev_parse_time_ms_kvarg,
3528 &link_down_delay_ms) < 0) {
3530 "Invalid link down propagation delay value specified for"
3531 " bonded device %s", name);
3535 /* Set balance mode transmit policy*/
3536 if (rte_eth_bond_link_down_prop_delay_set(port_id, link_down_delay_ms)
3539 "Failed to set link down propagation delay (%u ms) on bonded device %s",
3540 link_down_delay_ms, name);
3543 } else if (arg_count > 1) {
3545 "Link down propagation delay can be specified only once for bonded device %s",
3553 struct rte_vdev_driver pmd_bond_drv = {
3554 .probe = bond_probe,
3555 .remove = bond_remove,
3558 RTE_PMD_REGISTER_VDEV(net_bonding, pmd_bond_drv);
3559 RTE_PMD_REGISTER_ALIAS(net_bonding, eth_bond);
3561 RTE_PMD_REGISTER_PARAM_STRING(net_bonding,
3565 "xmit_policy=[l2 | l23 | l34] "
3566 "agg_mode=[count | stable | bandwidth] "
3569 "lsc_poll_period_ms=<int> "
3571 "down_delay=<int>");
3575 RTE_INIT(bond_init_log)
3577 bond_logtype = rte_log_register("pmd.net.bond");
3578 if (bond_logtype >= 0)
3579 rte_log_set_level(bond_logtype, RTE_LOG_NOTICE);