X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fbonding%2Frte_eth_bond_pmd.c;h=35fe90673fea6a4ee22fc3abadb5cdddd29880ce;hb=d6af1a13d7a14d062d11b37f6e31caa0f3823fe0;hp=c937e6b07407dfd2eab000fa1d1036a9a401ad74;hpb=3eb6bdd89e49db7ca1b9b5827b8aa194ecfb2e60;p=dpdk.git diff --git a/drivers/net/bonding/rte_eth_bond_pmd.c b/drivers/net/bonding/rte_eth_bond_pmd.c index c937e6b074..35fe90673f 100644 --- a/drivers/net/bonding/rte_eth_bond_pmd.c +++ b/drivers/net/bonding/rte_eth_bond_pmd.c @@ -1,7 +1,7 @@ /*- * BSD LICENSE * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,12 +36,14 @@ #include #include #include +#include #include #include #include +#include #include #include -#include +#include #include #include @@ -50,6 +52,7 @@ #include "rte_eth_bond_8023ad_private.h" #define REORDER_PERIOD_MS 10 +#define DEFAULT_POLLING_INTERVAL_10_MS (10) #define HASH_L4_PORTS(h) ((h)->src_port ^ (h)->dst_port) @@ -121,6 +124,15 @@ bond_ethdev_rx_burst_active_backup(void *queue, struct rte_mbuf **bufs, bd_rx_q->queue_id, bufs, nb_pkts); } +static inline uint8_t +is_lacp_packets(uint16_t ethertype, uint8_t subtype, uint16_t vlan_tci) +{ + const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW); + + return !vlan_tci && (ethertype == ether_type_slow_be && + (subtype == SLOW_SUBTYPE_MARKER || subtype == SLOW_SUBTYPE_LACP)); +} + static uint16_t bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) @@ -135,11 +147,12 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, const uint16_t ether_type_slow_be = rte_be_to_cpu_16(ETHER_TYPE_SLOW); uint16_t num_rx_total = 0; /* Total number of received packets */ uint8_t slaves[RTE_MAX_ETHPORTS]; - uint8_t slave_count; + uint8_t slave_count, idx; uint8_t collecting; /* current slave collecting status */ const uint8_t promisc = internals->promiscuous_en; uint8_t i, j, k; + uint8_t subtype; rte_eth_macaddr_get(internals->port_id, &bond_mac); /* Copy slave list to protect against slave up/down changes during tx @@ -148,12 +161,18 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, memcpy(slaves, internals->active_slaves, sizeof(internals->active_slaves[0]) * slave_count); + idx = internals->active_slave; + if (idx >= slave_count) { + internals->active_slave = 0; + idx = 0; + } for (i = 0; i < slave_count && num_rx_total < nb_pkts; i++) { j = num_rx_total; - collecting = ACTOR_STATE(&mode_8023ad_ports[slaves[i]], COLLECTING); + collecting = ACTOR_STATE(&mode_8023ad_ports[slaves[idx]], + COLLECTING); /* Read packets from this slave */ - num_rx_total += rte_eth_rx_burst(slaves[i], bd_rx_q->queue_id, + num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id, &bufs[num_rx_total], nb_pkts - num_rx_total); for (k = j; k < 2 && k < num_rx_total; k++) @@ -165,16 +184,19 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, rte_prefetch0(rte_pktmbuf_mtod(bufs[j + 3], void *)); hdr = rte_pktmbuf_mtod(bufs[j], struct ether_hdr *); + subtype = ((struct slow_protocol_frame *)hdr)->slow_protocol.subtype; + /* Remove packet from array if it is slow packet or slave is not * in collecting state or bondign interface is not in promiscus * mode and packet address does not match. */ - if (unlikely(hdr->ether_type == ether_type_slow_be || + if (unlikely(is_lacp_packets(hdr->ether_type, subtype, bufs[j]->vlan_tci) || !collecting || (!promisc && + !is_multicast_ether_addr(&hdr->d_addr) && !is_same_ether_addr(&bond_mac, &hdr->d_addr)))) { if (hdr->ether_type == ether_type_slow_be) { - bond_mode_8023ad_handle_slow_pkt(internals, slaves[i], - bufs[j]); + bond_mode_8023ad_handle_slow_pkt( + internals, slaves[idx], bufs[j]); } else rte_pktmbuf_free(bufs[j]); @@ -187,8 +209,11 @@ bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, } else j++; } + if (unlikely(++idx == slave_count)) + idx = 0; } + internals->active_slave = idx; return num_rx_total; } @@ -466,8 +491,10 @@ bond_ethdev_tx_burst_active_backup(void *queue, static inline uint16_t ether_hash(struct ether_hdr *eth_hdr) { - uint16_t *word_src_addr = (uint16_t *)eth_hdr->s_addr.addr_bytes; - uint16_t *word_dst_addr = (uint16_t *)eth_hdr->d_addr.addr_bytes; + unaligned_uint16_t *word_src_addr = + (unaligned_uint16_t *)eth_hdr->s_addr.addr_bytes; + unaligned_uint16_t *word_dst_addr = + (unaligned_uint16_t *)eth_hdr->d_addr.addr_bytes; return (word_src_addr[0] ^ word_dst_addr[0]) ^ (word_src_addr[1] ^ word_dst_addr[1]) ^ @@ -477,14 +504,16 @@ ether_hash(struct ether_hdr *eth_hdr) static inline uint32_t ipv4_hash(struct ipv4_hdr *ipv4_hdr) { - return (ipv4_hdr->src_addr ^ ipv4_hdr->dst_addr); + return ipv4_hdr->src_addr ^ ipv4_hdr->dst_addr; } static inline uint32_t ipv6_hash(struct ipv6_hdr *ipv6_hdr) { - uint32_t *word_src_addr = (uint32_t *)&(ipv6_hdr->src_addr[0]); - uint32_t *word_dst_addr = (uint32_t *)&(ipv6_hdr->dst_addr[0]); + unaligned_uint32_t *word_src_addr = + (unaligned_uint32_t *)&(ipv6_hdr->src_addr[0]); + unaligned_uint32_t *word_dst_addr = + (unaligned_uint32_t *)&(ipv6_hdr->dst_addr[0]); return (word_src_addr[0] ^ word_dst_addr[0]) ^ (word_src_addr[1] ^ word_dst_addr[1]) ^ @@ -548,17 +577,20 @@ xmit_l34_hash(const struct rte_mbuf *buf, uint8_t slave_count) l3hash = ipv4_hash(ipv4_hdr); - ip_hdr_offset = (ipv4_hdr->version_ihl & IPV4_HDR_IHL_MASK) * - IPV4_IHL_MULTIPLIER; - - if (ipv4_hdr->next_proto_id == IPPROTO_TCP) { - tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + - ip_hdr_offset); - l4hash = HASH_L4_PORTS(tcp_hdr); - } else if (ipv4_hdr->next_proto_id == IPPROTO_UDP) { - udp_hdr = (struct udp_hdr *)((char *)ipv4_hdr + - ip_hdr_offset); - l4hash = HASH_L4_PORTS(udp_hdr); + /* there is no L4 header in fragmented packet */ + if (likely(rte_ipv4_frag_pkt_is_fragmented(ipv4_hdr) == 0)) { + ip_hdr_offset = (ipv4_hdr->version_ihl & IPV4_HDR_IHL_MASK) * + IPV4_IHL_MULTIPLIER; + + if (ipv4_hdr->next_proto_id == IPPROTO_TCP) { + tcp_hdr = (struct tcp_hdr *)((char *)ipv4_hdr + + ip_hdr_offset); + l4hash = HASH_L4_PORTS(tcp_hdr); + } else if (ipv4_hdr->next_proto_id == IPPROTO_UDP) { + udp_hdr = (struct udp_hdr *)((char *)ipv4_hdr + + ip_hdr_offset); + l4hash = HASH_L4_PORTS(udp_hdr); + } } } else if (rte_cpu_to_be_16(ETHER_TYPE_IPv6) == proto) { struct ipv6_hdr *ipv6_hdr = (struct ipv6_hdr *) @@ -761,8 +793,8 @@ bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) uint16_t slave_bufs_pkts[RTE_MAX_ETHPORTS + 1] = { 0 }; /* - * We create separate transmit buffers for update packets as they wont be - * counted in num_tx_total. + * We create separate transmit buffers for update packets as they won't + * be counted in num_tx_total. */ struct rte_mbuf *update_bufs[RTE_MAX_ETHPORTS][ALB_HASH_TABLE_SIZE]; uint16_t update_bufs_pkts[RTE_MAX_ETHPORTS] = { 0 }; @@ -879,7 +911,6 @@ bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts) } num_tx_total += num_send; - num_not_send += slave_bufs_pkts[RTE_MAX_ETHPORTS] - num_send; } return num_tx_total; @@ -988,7 +1019,8 @@ bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs, struct port *port = &mode_8023ad_ports[slaves[i]]; slave_slow_nb_pkts[i] = rte_ring_dequeue_burst(port->tx_ring, - slow_pkts, BOND_MODE_8023AX_SLAVE_TX_PKTS); + slow_pkts, BOND_MODE_8023AX_SLAVE_TX_PKTS, + NULL); slave_nb_pkts[i] = slave_slow_nb_pkts[i]; for (j = 0; j < slave_slow_nb_pkts[i]; j++) @@ -1303,9 +1335,30 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, rte_eth_dev_stop(slave_eth_dev->data->port_id); /* Enable interrupts on slave device if supported */ - if (slave_eth_dev->driver->pci_drv.drv_flags & RTE_PCI_DRV_INTR_LSC) + if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) slave_eth_dev->data->dev_conf.intr_conf.lsc = 1; + /* If RSS is enabled for bonding, try to enable it for slaves */ + if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) { + if (bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len + != 0) { + slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len = + bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len; + slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = + bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key; + } else { + slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; + } + + slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf = + bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; + slave_eth_dev->data->dev_conf.rxmode.mq_mode = + bonded_eth_dev->data->dev_conf.rxmode.mq_mode; + } + + slave_eth_dev->data->dev_conf.rxmode.hw_vlan_filter = + bonded_eth_dev->data->dev_conf.rxmode.hw_vlan_filter; + /* Configure device */ errval = rte_eth_dev_configure(slave_eth_dev->data->port_id, bonded_eth_dev->data->nb_rx_queues, @@ -1357,6 +1410,38 @@ slave_configure(struct rte_eth_dev *bonded_eth_dev, return -1; } + /* If RSS is enabled for bonding, synchronize RETA */ + if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) { + int i; + struct bond_dev_private *internals; + + internals = bonded_eth_dev->data->dev_private; + + for (i = 0; i < internals->slave_count; i++) { + if (internals->slaves[i].port_id == slave_eth_dev->data->port_id) { + errval = rte_eth_dev_rss_reta_update( + slave_eth_dev->data->port_id, + &internals->reta_conf[0], + internals->slaves[i].reta_size); + if (errval != 0) { + RTE_LOG(WARNING, PMD, + "rte_eth_dev_rss_reta_update on slave port %d fails (err %d)." + " RSS Configuration for bonding may be inconsistent.\n", + slave_eth_dev->data->port_id, errval); + } + break; + } + } + } + + /* If lsc interrupt is set, check initial slave's link status */ + if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { + slave_eth_dev->dev_ops->link_update(slave_eth_dev, 0); + bond_ethdev_lsc_event_callback(slave_eth_dev->data->port_id, + RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id, + NULL); + } + return 0; } @@ -1377,6 +1462,9 @@ slave_remove(struct bond_dev_private *internals, (internals->slave_count - i - 1)); internals->slave_count--; + + /* force reconfiguration of slave interfaces */ + _rte_eth_dev_reset(slave_eth_dev); } static void @@ -1392,18 +1480,11 @@ slave_add(struct bond_dev_private *internals, slave_details->port_id = slave_eth_dev->data->port_id; slave_details->last_link_status = 0; - /* If slave device doesn't support interrupts then we need to enabled - * polling to monitor link status */ - if (!(slave_eth_dev->pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC)) { + /* Mark slave devices that don't support interrupts so we can + * compensate when we start the bond + */ + if (!(slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) { slave_details->link_status_poll_enabled = 1; - - if (!internals->link_status_polling_enabled) { - internals->link_status_polling_enabled = 1; - - rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000, - bond_ethdev_slave_link_status_change_monitor, - (void *)&rte_eth_devices[internals->port_id]); - } } slave_details->link_status_wait_to_complete = 0; @@ -1438,13 +1519,13 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) int i; /* slave eth dev will be started by bonded device */ - if (valid_bonded_ethdev(eth_dev)) { + if (check_for_bonded_ethdev(eth_dev)) { RTE_BOND_LOG(ERR, "User tried to explicitly start a slave eth_dev (%d)", eth_dev->data->port_id); return -1; } - eth_dev->data->dev_link.link_status = 0; + eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; eth_dev->data->dev_started = 1; internals = eth_dev->data->dev_private; @@ -1488,6 +1569,18 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) eth_dev->data->port_id, internals->slaves[i].port_id); return -1; } + /* We will need to poll for link status if any slave doesn't + * support interrupts + */ + if (internals->slaves[i].link_status_poll_enabled) + internals->link_status_polling_enabled = 1; + } + /* start polling if needed */ + if (internals->link_status_polling_enabled) { + rte_eal_alarm_set( + internals->link_status_polling_interval_ms * 1000, + bond_ethdev_slave_link_status_change_monitor, + (void *)&rte_eth_devices[internals->port_id]); } if (internals->user_defined_primary_port) @@ -1504,6 +1597,28 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) } static void +bond_ethdev_free_queues(struct rte_eth_dev *dev) +{ + uint8_t i; + + if (dev->data->rx_queues != NULL) { + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rte_free(dev->data->rx_queues[i]); + dev->data->rx_queues[i] = NULL; + } + dev->data->nb_rx_queues = 0; + } + + if (dev->data->tx_queues != NULL) { + for (i = 0; i < dev->data->nb_tx_queues; i++) { + rte_free(dev->data->tx_queues[i]); + dev->data->tx_queues[i] = NULL; + } + dev->data->nb_tx_queues = 0; + } +} + +void bond_ethdev_stop(struct rte_eth_dev *eth_dev) { struct bond_dev_private *internals = eth_dev->data->dev_private; @@ -1516,14 +1631,14 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev) bond_mode_8023ad_stop(eth_dev); /* Discard all messages to/from mode 4 state machines */ - for (i = 0; i < internals->slave_count; i++) { - port = &mode_8023ad_ports[internals->slaves[i].port_id]; + for (i = 0; i < internals->active_slave_count; i++) { + port = &mode_8023ad_ports[internals->active_slaves[i]]; - RTE_VERIFY(port->rx_ring != NULL); + RTE_ASSERT(port->rx_ring != NULL); while (rte_ring_dequeue(port->rx_ring, &pkt) != -ENOENT) rte_pktmbuf_free(pkt); - RTE_VERIFY(port->tx_ring != NULL); + RTE_ASSERT(port->tx_ring != NULL); while (rte_ring_dequeue(port->tx_ring, &pkt) != -ENOENT) rte_pktmbuf_free(pkt); } @@ -1538,14 +1653,35 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev) internals->active_slave_count = 0; internals->link_status_polling_enabled = 0; + for (i = 0; i < internals->slave_count; i++) + internals->slaves[i].last_link_status = 0; - eth_dev->data->dev_link.link_status = 0; + eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; eth_dev->data->dev_started = 0; } -static void -bond_ethdev_close(struct rte_eth_dev *dev __rte_unused) +void +bond_ethdev_close(struct rte_eth_dev *dev) { + struct bond_dev_private *internals = dev->data->dev_private; + uint8_t bond_port_id = internals->port_id; + int skipped = 0; + + RTE_LOG(INFO, EAL, "Closing bonded device %s\n", dev->data->name); + while (internals->slave_count != skipped) { + uint8_t port_id = internals->slaves[skipped].port_id; + + rte_eth_dev_stop(port_id); + + if (rte_eth_bond_slave_remove(bond_port_id, port_id) != 0) { + RTE_LOG(ERR, EAL, + "Failed to remove port %d from bonded device " + "%s\n", port_id, dev->data->name); + skipped++; + } + } + bond_ethdev_free_queues(dev); + rte_bitmap_reset(internals->vlan_filter_bmp); } /* forward declaration */ @@ -1556,19 +1692,51 @@ bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct bond_dev_private *internals = dev->data->dev_private; - dev_info->driver_name = driver_name; dev_info->max_mac_addrs = 1; - dev_info->max_rx_pktlen = (uint32_t)2048; + dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen + ? internals->candidate_max_rx_pktlen + : ETHER_MAX_JUMBO_FRAME_LEN; dev_info->max_rx_queues = (uint16_t)128; dev_info->max_tx_queues = (uint16_t)512; dev_info->min_rx_bufsize = 0; - dev_info->pci_dev = dev->pci_dev; dev_info->rx_offload_capa = internals->rx_offload_capa; dev_info->tx_offload_capa = internals->tx_offload_capa; + dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads; + + dev_info->reta_size = internals->reta_size; +} + +static int +bond_ethdev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) +{ + int res; + uint8_t i; + struct bond_dev_private *internals = dev->data->dev_private; + + /* don't do this while a slave is being added */ + rte_spinlock_lock(&internals->lock); + + if (on) + rte_bitmap_set(internals->vlan_filter_bmp, vlan_id); + else + rte_bitmap_clear(internals->vlan_filter_bmp, vlan_id); + + for (i = 0; i < internals->slave_count; i++) { + uint8_t port_id = internals->slaves[i].port_id; + + res = rte_eth_dev_vlan_filter(port_id, vlan_id, on); + if (res == ENOTSUP) + RTE_LOG(WARNING, PMD, + "Setting VLAN filter on slave port %u not supported.\n", + port_id); + } + + rte_spinlock_unlock(&internals->lock); + return 0; } static int @@ -1578,7 +1746,7 @@ bond_ethdev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, { struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *) rte_zmalloc_socket(NULL, sizeof(struct bond_rx_queue), - 0, dev->pci_dev->numa_node); + 0, dev->data->numa_node); if (bd_rx_q == NULL) return -1; @@ -1602,7 +1770,7 @@ bond_ethdev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, { struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *) rte_zmalloc_socket(NULL, sizeof(struct bond_tx_queue), - 0, dev->pci_dev->numa_node); + 0, dev->data->numa_node); if (bd_tx_q == NULL) return -1; @@ -1682,7 +1850,8 @@ bond_ethdev_slave_link_status_change_monitor(void *cb_arg) bond_ethdev_lsc_event_callback(internals->slaves[i].port_id, RTE_ETH_EVENT_INTR_LSC, - &bonded_ethdev->data->port_id); + &bonded_ethdev->data->port_id, + NULL); } } rte_spinlock_unlock(&internals->lock); @@ -1702,7 +1871,7 @@ bond_ethdev_link_update(struct rte_eth_dev *bonded_eth_dev, if (!bonded_eth_dev->data->dev_started || internals->active_slave_count == 0) { - bonded_eth_dev->data->dev_link.link_status = 0; + bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; return 0; } else { struct rte_eth_dev *slave_eth_dev; @@ -1713,7 +1882,7 @@ bond_ethdev_link_update(struct rte_eth_dev *bonded_eth_dev, (*slave_eth_dev->dev_ops->link_update)(slave_eth_dev, wait_to_complete); - if (slave_eth_dev->data->dev_link.link_status == 1) { + if (slave_eth_dev->data->dev_link.link_status == ETH_LINK_UP) { link_up = 1; break; } @@ -1730,11 +1899,7 @@ bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { struct bond_dev_private *internals = dev->data->dev_private; struct rte_eth_stats slave_stats; - - int i; - - /* clear bonded stats before populating from slaves */ - memset(stats, 0, sizeof(*stats)); + int i, j; for (i = 0; i < internals->slave_count; i++) { rte_eth_stats_get(internals->slaves[i].port_id, &slave_stats); @@ -1743,16 +1908,19 @@ bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) stats->opackets += slave_stats.opackets; stats->ibytes += slave_stats.ibytes; stats->obytes += slave_stats.obytes; + stats->imissed += slave_stats.imissed; stats->ierrors += slave_stats.ierrors; stats->oerrors += slave_stats.oerrors; - stats->imcasts += slave_stats.imcasts; stats->rx_nombuf += slave_stats.rx_nombuf; - stats->fdirmatch += slave_stats.fdirmatch; - stats->fdirmiss += slave_stats.fdirmiss; - stats->tx_pause_xon += slave_stats.tx_pause_xon; - stats->rx_pause_xon += slave_stats.rx_pause_xon; - stats->tx_pause_xoff += slave_stats.tx_pause_xoff; - stats->rx_pause_xoff += slave_stats.rx_pause_xoff; + + for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { + stats->q_ipackets[j] += slave_stats.q_ipackets[j]; + stats->q_opackets[j] += slave_stats.q_opackets[j]; + stats->q_ibytes[j] += slave_stats.q_ibytes[j]; + stats->q_obytes[j] += slave_stats.q_obytes[j]; + stats->q_errors[j] += slave_stats.q_errors[j]; + } + } } @@ -1829,35 +1997,36 @@ bond_ethdev_delayed_lsc_propagation(void *arg) return; _rte_eth_dev_callback_process((struct rte_eth_dev *)arg, - RTE_ETH_EVENT_INTR_LSC); + RTE_ETH_EVENT_INTR_LSC, NULL, NULL); } -void +int bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, - void *param) + void *param, void *ret_param __rte_unused) { struct rte_eth_dev *bonded_eth_dev, *slave_eth_dev; struct bond_dev_private *internals; struct rte_eth_link link; + int rc = -1; int i, valid_slave = 0; uint8_t active_pos; uint8_t lsc_flag = 0; if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL) - return; + return rc; bonded_eth_dev = &rte_eth_devices[*(uint8_t *)param]; slave_eth_dev = &rte_eth_devices[port_id]; - if (valid_bonded_ethdev(bonded_eth_dev)) - return; + if (check_for_bonded_ethdev(bonded_eth_dev)) + return rc; internals = bonded_eth_dev->data->dev_private; /* If the device isn't started don't handle interrupts */ if (!bonded_eth_dev->data->dev_started) - return; + return rc; /* verify that port_id is a valid slave of bonded port */ for (i = 0; i < internals->slave_count; i++) { @@ -1868,7 +2037,7 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, } if (!valid_slave) - return; + return rc; /* Search for port in active port list */ active_pos = find_slave_by_id(internals->active_slaves, @@ -1877,12 +2046,12 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, rte_eth_link_get_nowait(port_id, &link); if (link.link_status) { if (active_pos < internals->active_slave_count) - return; + return rc; /* if no active slave ports then set this port to be primary port */ if (internals->active_slave_count < 1) { /* If first active slave, then change link status */ - bonded_eth_dev->data->dev_link.link_status = 1; + bonded_eth_dev->data->dev_link.link_status = ETH_LINK_UP; internals->current_primary_port = port_id; lsc_flag = 1; @@ -1891,6 +2060,16 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, /* Inherit eth dev link properties from first active slave */ link_properties_set(bonded_eth_dev, &(slave_eth_dev->data->dev_link)); + } else { + if (link_properties_valid( + &bonded_eth_dev->data->dev_link, &link) != 0) { + slave_eth_dev->data->dev_flags &= + (~RTE_ETH_DEV_BONDED_SLAVE); + RTE_LOG(ERR, PMD, + "port %u invalid speed/duplex\n", + port_id); + return rc; + } } activate_slave(bonded_eth_dev, port_id); @@ -1901,7 +2080,7 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, bond_ethdev_primary_set(internals, port_id); } else { if (active_pos == internals->active_slave_count) - return; + return rc; /* Remove from active slave list */ deactivate_slave(bonded_eth_dev, port_id); @@ -1910,7 +2089,7 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, * link properties */ if (internals->active_slave_count < 1) { lsc_flag = 1; - bonded_eth_dev->data->dev_link.link_status = 0; + bonded_eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; link_properties_reset(bonded_eth_dev); } @@ -1940,7 +2119,8 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, (void *)bonded_eth_dev); else _rte_eth_dev_callback_process(bonded_eth_dev, - RTE_ETH_EVENT_INTR_LSC); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } else { if (internals->link_down_delay_ms > 0) @@ -1949,39 +2129,264 @@ bond_ethdev_lsc_event_callback(uint8_t port_id, enum rte_eth_event_type type, (void *)bonded_eth_dev); else _rte_eth_dev_callback_process(bonded_eth_dev, - RTE_ETH_EVENT_INTR_LSC); + RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } } + return 0; } -struct eth_dev_ops default_dev_ops = { - .dev_start = bond_ethdev_start, - .dev_stop = bond_ethdev_stop, - .dev_close = bond_ethdev_close, - .dev_configure = bond_ethdev_configure, - .dev_infos_get = bond_ethdev_info, - .rx_queue_setup = bond_ethdev_rx_queue_setup, - .tx_queue_setup = bond_ethdev_tx_queue_setup, - .rx_queue_release = bond_ethdev_rx_queue_release, - .tx_queue_release = bond_ethdev_tx_queue_release, - .link_update = bond_ethdev_link_update, - .stats_get = bond_ethdev_stats_get, - .stats_reset = bond_ethdev_stats_reset, - .promiscuous_enable = bond_ethdev_promiscuous_enable, - .promiscuous_disable = bond_ethdev_promiscuous_disable +static int +bond_ethdev_rss_reta_update(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) +{ + unsigned i, j; + int result = 0; + int slave_reta_size; + unsigned reta_count; + struct bond_dev_private *internals = dev->data->dev_private; + + if (reta_size != internals->reta_size) + return -EINVAL; + + /* Copy RETA table */ + reta_count = reta_size / RTE_RETA_GROUP_SIZE; + + for (i = 0; i < reta_count; i++) { + internals->reta_conf[i].mask = reta_conf[i].mask; + for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) + if ((reta_conf[i].mask >> j) & 0x01) + internals->reta_conf[i].reta[j] = reta_conf[i].reta[j]; + } + + /* Fill rest of array */ + for (; i < RTE_DIM(internals->reta_conf); i += reta_count) + memcpy(&internals->reta_conf[i], &internals->reta_conf[0], + sizeof(internals->reta_conf[0]) * reta_count); + + /* Propagate RETA over slaves */ + for (i = 0; i < internals->slave_count; i++) { + slave_reta_size = internals->slaves[i].reta_size; + result = rte_eth_dev_rss_reta_update(internals->slaves[i].port_id, + &internals->reta_conf[0], slave_reta_size); + if (result < 0) + return result; + } + + return 0; +} + +static int +bond_ethdev_rss_reta_query(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) +{ + int i, j; + struct bond_dev_private *internals = dev->data->dev_private; + + if (reta_size != internals->reta_size) + return -EINVAL; + + /* Copy RETA table */ + for (i = 0; i < reta_size / RTE_RETA_GROUP_SIZE; i++) + for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) + if ((reta_conf[i].mask >> j) & 0x01) + reta_conf[i].reta[j] = internals->reta_conf[i].reta[j]; + + return 0; +} + +static int +bond_ethdev_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + int i, result = 0; + struct bond_dev_private *internals = dev->data->dev_private; + struct rte_eth_rss_conf bond_rss_conf; + + memcpy(&bond_rss_conf, rss_conf, sizeof(struct rte_eth_rss_conf)); + + bond_rss_conf.rss_hf &= internals->flow_type_rss_offloads; + + if (bond_rss_conf.rss_hf != 0) + dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf = bond_rss_conf.rss_hf; + + if (bond_rss_conf.rss_key && bond_rss_conf.rss_key_len < + sizeof(internals->rss_key)) { + if (bond_rss_conf.rss_key_len == 0) + bond_rss_conf.rss_key_len = 40; + internals->rss_key_len = bond_rss_conf.rss_key_len; + memcpy(internals->rss_key, bond_rss_conf.rss_key, + internals->rss_key_len); + } + + for (i = 0; i < internals->slave_count; i++) { + result = rte_eth_dev_rss_hash_update(internals->slaves[i].port_id, + &bond_rss_conf); + if (result < 0) + return result; + } + + return 0; +} + +static int +bond_ethdev_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct bond_dev_private *internals = dev->data->dev_private; + + rss_conf->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; + rss_conf->rss_key_len = internals->rss_key_len; + if (rss_conf->rss_key) + memcpy(rss_conf->rss_key, internals->rss_key, internals->rss_key_len); + + return 0; +} + +const struct eth_dev_ops default_dev_ops = { + .dev_start = bond_ethdev_start, + .dev_stop = bond_ethdev_stop, + .dev_close = bond_ethdev_close, + .dev_configure = bond_ethdev_configure, + .dev_infos_get = bond_ethdev_info, + .vlan_filter_set = bond_ethdev_vlan_filter_set, + .rx_queue_setup = bond_ethdev_rx_queue_setup, + .tx_queue_setup = bond_ethdev_tx_queue_setup, + .rx_queue_release = bond_ethdev_rx_queue_release, + .tx_queue_release = bond_ethdev_tx_queue_release, + .link_update = bond_ethdev_link_update, + .stats_get = bond_ethdev_stats_get, + .stats_reset = bond_ethdev_stats_reset, + .promiscuous_enable = bond_ethdev_promiscuous_enable, + .promiscuous_disable = bond_ethdev_promiscuous_disable, + .reta_update = bond_ethdev_rss_reta_update, + .reta_query = bond_ethdev_rss_reta_query, + .rss_hash_update = bond_ethdev_rss_hash_update, + .rss_hash_conf_get = bond_ethdev_rss_hash_conf_get }; static int -bond_init(const char *name, const char *params) +bond_alloc(struct rte_vdev_device *dev, uint8_t mode) +{ + const char *name = rte_vdev_device_name(dev); + uint8_t socket_id = dev->device.numa_node; + struct bond_dev_private *internals = NULL; + struct rte_eth_dev *eth_dev = NULL; + uint32_t vlan_filter_bmp_size; + + /* now do all data allocation - for eth_dev structure, dummy pci driver + * and internal (private) data + */ + + /* reserve an ethdev entry */ + eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals)); + if (eth_dev == NULL) { + RTE_BOND_LOG(ERR, "Unable to allocate rte_eth_dev"); + goto err; + } + + internals = eth_dev->data->dev_private; + eth_dev->data->nb_rx_queues = (uint16_t)1; + eth_dev->data->nb_tx_queues = (uint16_t)1; + + eth_dev->data->mac_addrs = rte_zmalloc_socket(name, ETHER_ADDR_LEN, 0, + socket_id); + if (eth_dev->data->mac_addrs == NULL) { + RTE_BOND_LOG(ERR, "Unable to malloc mac_addrs"); + goto err; + } + + eth_dev->dev_ops = &default_dev_ops; + eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC | + RTE_ETH_DEV_DETACHABLE; + + rte_spinlock_init(&internals->lock); + + internals->port_id = eth_dev->data->port_id; + internals->mode = BONDING_MODE_INVALID; + internals->current_primary_port = RTE_MAX_ETHPORTS + 1; + internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2; + internals->xmit_hash = xmit_l2_hash; + internals->user_defined_mac = 0; + internals->link_props_set = 0; + + internals->link_status_polling_enabled = 0; + + internals->link_status_polling_interval_ms = + DEFAULT_POLLING_INTERVAL_10_MS; + internals->link_down_delay_ms = 0; + internals->link_up_delay_ms = 0; + + internals->slave_count = 0; + internals->active_slave_count = 0; + internals->rx_offload_capa = 0; + internals->tx_offload_capa = 0; + internals->candidate_max_rx_pktlen = 0; + internals->max_rx_pktlen = 0; + + /* Initially allow to choose any offload type */ + internals->flow_type_rss_offloads = ETH_RSS_PROTO_MASK; + + memset(internals->active_slaves, 0, sizeof(internals->active_slaves)); + memset(internals->slaves, 0, sizeof(internals->slaves)); + + /* Set mode 4 default configuration */ + bond_mode_8023ad_setup(eth_dev, NULL); + if (bond_ethdev_mode_set(eth_dev, mode)) { + RTE_BOND_LOG(ERR, "Failed to set bonded device %d mode too %d", + eth_dev->data->port_id, mode); + goto err; + } + + vlan_filter_bmp_size = + rte_bitmap_get_memory_footprint(ETHER_MAX_VLAN_ID + 1); + internals->vlan_filter_bmpmem = rte_malloc(name, vlan_filter_bmp_size, + RTE_CACHE_LINE_SIZE); + if (internals->vlan_filter_bmpmem == NULL) { + RTE_BOND_LOG(ERR, + "Failed to allocate vlan bitmap for bonded device %u\n", + eth_dev->data->port_id); + goto err; + } + + internals->vlan_filter_bmp = rte_bitmap_init(ETHER_MAX_VLAN_ID + 1, + internals->vlan_filter_bmpmem, vlan_filter_bmp_size); + if (internals->vlan_filter_bmp == NULL) { + RTE_BOND_LOG(ERR, + "Failed to init vlan bitmap for bonded device %u\n", + eth_dev->data->port_id); + rte_free(internals->vlan_filter_bmpmem); + goto err; + } + + return eth_dev->data->port_id; + +err: + rte_free(internals); + if (eth_dev != NULL) { + rte_free(eth_dev->data->mac_addrs); + rte_eth_dev_release_port(eth_dev); + } + return -1; +} + +static int +bond_probe(struct rte_vdev_device *dev) { + const char *name; struct bond_dev_private *internals; struct rte_kvargs *kvlist; uint8_t bonding_mode, socket_id; int arg_count, port_id; + if (!dev) + return -EINVAL; + + name = rte_vdev_device_name(dev); RTE_LOG(INFO, EAL, "Initializing pmd_bond for %s\n", name); - kvlist = rte_kvargs_parse(params, pmd_bond_init_valid_arguments); + kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), + pmd_bond_init_valid_arguments); if (kvlist == NULL) return -1; @@ -2018,8 +2423,10 @@ bond_init(const char *name, const char *params) socket_id = rte_socket_id(); } + dev->device.numa_node = socket_id; + /* Create link bonding eth device */ - port_id = rte_eth_bond_create(name, bonding_mode, socket_id); + port_id = bond_alloc(dev, bonding_mode); if (port_id < 0) { RTE_LOG(ERR, EAL, "Failed to create socket %s in mode %u on " "socket %u.\n", name, bonding_mode, socket_id); @@ -2038,6 +2445,54 @@ parse_error: return -1; } +static int +bond_remove(struct rte_vdev_device *dev) +{ + struct rte_eth_dev *eth_dev; + struct bond_dev_private *internals; + const char *name; + + if (!dev) + return -EINVAL; + + name = rte_vdev_device_name(dev); + RTE_LOG(INFO, EAL, "Uninitializing pmd_bond for %s\n", name); + + /* now free all data allocation - for eth_dev structure, + * dummy pci driver and internal (private) data + */ + + /* find an ethdev entry */ + eth_dev = rte_eth_dev_allocated(name); + if (eth_dev == NULL) + return -ENODEV; + + RTE_ASSERT(eth_dev->device == &dev->device); + + internals = eth_dev->data->dev_private; + if (internals->slave_count != 0) + return -EBUSY; + + if (eth_dev->data->dev_started == 1) { + bond_ethdev_stop(eth_dev); + bond_ethdev_close(eth_dev); + } + + eth_dev->dev_ops = NULL; + eth_dev->rx_pkt_burst = NULL; + eth_dev->tx_pkt_burst = NULL; + + internals = eth_dev->data->dev_private; + rte_bitmap_free(internals->vlan_filter_bmp); + rte_free(internals->vlan_filter_bmpmem); + rte_free(eth_dev->data->dev_private); + rte_free(eth_dev->data->mac_addrs); + + rte_eth_dev_release_port(eth_dev); + + return 0; +} + /* this part will resolve the slave portids after all the other pdev and vdev * have been allocated */ static int @@ -2049,6 +2504,31 @@ bond_ethdev_configure(struct rte_eth_dev *dev) int arg_count; uint8_t port_id = dev - rte_eth_devices; + static const uint8_t default_rss_key[40] = { + 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D, + 0x43, 0xA3, 0x8F, 0xB0, 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4, + 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 0x6A, 0x42, 0xB7, 0x3B, + 0xBE, 0xAC, 0x01, 0xFA + }; + + unsigned i, j; + + /* If RSS is enabled, fill table and key with default values */ + if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) { + dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key = internals->rss_key; + dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len = 0; + memcpy(internals->rss_key, default_rss_key, 40); + + for (i = 0; i < RTE_DIM(internals->reta_conf); i++) { + internals->reta_conf[i].mask = ~0LL; + for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) + internals->reta_conf[i].reta[j] = j % dev->data->nb_rx_queues; + } + } + + /* set the max_rx_pktlen */ + internals->max_rx_pktlen = internals->candidate_max_rx_pktlen; + /* * if no kvlist, it means that this bonded device has been created * through the bonding api. @@ -2260,10 +2740,21 @@ bond_ethdev_configure(struct rte_eth_dev *dev) return 0; } -static struct rte_driver bond_drv = { - .name = "eth_bond", - .type = PMD_VDEV, - .init = bond_init, +struct rte_vdev_driver pmd_bond_drv = { + .probe = bond_probe, + .remove = bond_remove, }; -PMD_REGISTER_DRIVER(bond_drv); +RTE_PMD_REGISTER_VDEV(net_bonding, pmd_bond_drv); +RTE_PMD_REGISTER_ALIAS(net_bonding, eth_bond); + +RTE_PMD_REGISTER_PARAM_STRING(net_bonding, + "slave= " + "primary= " + "mode=[0-6] " + "xmit_policy=[l2 | l23 | l34] " + "socket_id= " + "mac= " + "lsc_poll_period_ms= " + "up_delay= " + "down_delay=");