drivers/net/bonding/rte_eth_bond_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4 #include <stdlib.h>
   5 #include <stdbool.h>
   6 #include <netinet/in.h>
   7
   8 #include <rte_mbuf.h>
   9 #include <rte_malloc.h>
  10 #include <ethdev_driver.h>
  11 #include <ethdev_vdev.h>
  12 #include <rte_tcp.h>
  13 #include <rte_udp.h>
  14 #include <rte_ip.h>
  15 #include <rte_ip_frag.h>
  16 #include <rte_devargs.h>
  17 #include <rte_kvargs.h>
  18 #include <rte_bus_vdev.h>
  19 #include <rte_alarm.h>
  20 #include <rte_cycles.h>
  21 #include <rte_string_fns.h>
  22
  23 #include "rte_eth_bond.h"
  24 #include "eth_bond_private.h"
  25 #include "eth_bond_8023ad_private.h"
  26
  27 #define REORDER_PERIOD_MS 10
  28 #define DEFAULT_POLLING_INTERVAL_10_MS (10)
  29 #define BOND_MAX_MAC_ADDRS 16
  30
  31 #define HASH_L4_PORTS(h) ((h)->src_port ^ (h)->dst_port)
  32
  33 /* Table for statistics in mode 5 TLB */
  34 static uint64_t tlb_last_obytets[RTE_MAX_ETHPORTS];
  35
  36 static inline size_t
  37 get_vlan_offset(struct rte_ether_hdr *eth_hdr, uint16_t *proto)
  38 {
  39         size_t vlan_offset = 0;
  40
  41         if (rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN) == *proto ||
  42                 rte_cpu_to_be_16(RTE_ETHER_TYPE_QINQ) == *proto) {
  43                 struct rte_vlan_hdr *vlan_hdr =
  44                         (struct rte_vlan_hdr *)(eth_hdr + 1);
  45
  46                 vlan_offset = sizeof(struct rte_vlan_hdr);
  47                 *proto = vlan_hdr->eth_proto;
  48
  49                 if (rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN) == *proto) {
  50                         vlan_hdr = vlan_hdr + 1;
  51                         *proto = vlan_hdr->eth_proto;
  52                         vlan_offset += sizeof(struct rte_vlan_hdr);
  53                 }
  54         }
  55         return vlan_offset;
  56 }
  57
  58 static uint16_t
  59 bond_ethdev_rx_burst(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
  60 {
  61         struct bond_dev_private *internals;
  62
  63         uint16_t num_rx_total = 0;
  64         uint16_t slave_count;
  65         uint16_t active_slave;
  66         int i;
  67
  68         /* Cast to structure, containing bonded device's port id and queue id */
  69         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
  70         internals = bd_rx_q->dev_private;
  71         slave_count = internals->active_slave_count;
  72         active_slave = bd_rx_q->active_slave;
  73
  74         for (i = 0; i < slave_count && nb_pkts; i++) {
  75                 uint16_t num_rx_slave;
  76
  77                 /* Offset of pointer to *bufs increases as packets are received
  78                  * from other slaves */
  79                 num_rx_slave =
  80                         rte_eth_rx_burst(internals->active_slaves[active_slave],
  81                                          bd_rx_q->queue_id,
  82                                          bufs + num_rx_total, nb_pkts);
  83                 num_rx_total += num_rx_slave;
  84                 nb_pkts -= num_rx_slave;
  85                 if (++active_slave == slave_count)
  86                         active_slave = 0;
  87         }
  88
  89         if (++bd_rx_q->active_slave >= slave_count)
  90                 bd_rx_q->active_slave = 0;
  91         return num_rx_total;
  92 }
  93
  94 static uint16_t
  95 bond_ethdev_rx_burst_active_backup(void *queue, struct rte_mbuf **bufs,
  96                 uint16_t nb_pkts)
  97 {
  98         struct bond_dev_private *internals;
  99
 100         /* Cast to structure, containing bonded device's port id and queue id */
 101         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
 102
 103         internals = bd_rx_q->dev_private;
 104
 105         return rte_eth_rx_burst(internals->current_primary_port,
 106                         bd_rx_q->queue_id, bufs, nb_pkts);
 107 }
 108
 109 static inline uint8_t
 110 is_lacp_packets(uint16_t ethertype, uint8_t subtype, struct rte_mbuf *mbuf)
 111 {
 112         const uint16_t ether_type_slow_be =
 113                 rte_be_to_cpu_16(RTE_ETHER_TYPE_SLOW);
 114
 115         return !((mbuf->ol_flags & RTE_MBUF_F_RX_VLAN) ? mbuf->vlan_tci : 0) &&
 116                 (ethertype == ether_type_slow_be &&
 117                 (subtype == SLOW_SUBTYPE_MARKER || subtype == SLOW_SUBTYPE_LACP));
 118 }
 119
 120 /*****************************************************************************
 121  * Flow director's setup for mode 4 optimization
 122  */
 123
 124 static struct rte_flow_item_eth flow_item_eth_type_8023ad = {
 125         .dst.addr_bytes = { 0 },
 126         .src.addr_bytes = { 0 },
 127         .type = RTE_BE16(RTE_ETHER_TYPE_SLOW),
 128 };
 129
 130 static struct rte_flow_item_eth flow_item_eth_mask_type_8023ad = {
 131         .dst.addr_bytes = { 0 },
 132         .src.addr_bytes = { 0 },
 133         .type = 0xFFFF,
 134 };
 135
 136 static struct rte_flow_item flow_item_8023ad[] = {
 137         {
 138                 .type = RTE_FLOW_ITEM_TYPE_ETH,
 139                 .spec = &flow_item_eth_type_8023ad,
 140                 .last = NULL,
 141                 .mask = &flow_item_eth_mask_type_8023ad,
 142         },
 143         {
 144                 .type = RTE_FLOW_ITEM_TYPE_END,
 145                 .spec = NULL,
 146                 .last = NULL,
 147                 .mask = NULL,
 148         }
 149 };
 150
 151 const struct rte_flow_attr flow_attr_8023ad = {
 152         .group = 0,
 153         .priority = 0,
 154         .ingress = 1,
 155         .egress = 0,
 156         .reserved = 0,
 157 };
 158
 159 int
 160 bond_ethdev_8023ad_flow_verify(struct rte_eth_dev *bond_dev,
 161                 uint16_t slave_port) {
 162         struct rte_eth_dev_info slave_info;
 163         struct rte_flow_error error;
 164         struct bond_dev_private *internals = bond_dev->data->dev_private;
 165
 166         const struct rte_flow_action_queue lacp_queue_conf = {
 167                 .index = 0,
 168         };
 169
 170         const struct rte_flow_action actions[] = {
 171                 {
 172                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
 173                         .conf = &lacp_queue_conf
 174                 },
 175                 {
 176                         .type = RTE_FLOW_ACTION_TYPE_END,
 177                 }
 178         };
 179
 180         int ret = rte_flow_validate(slave_port, &flow_attr_8023ad,
 181                         flow_item_8023ad, actions, &error);
 182         if (ret < 0) {
 183                 RTE_BOND_LOG(ERR, "%s: %s (slave_port=%d queue_id=%d)",
 184                                 __func__, error.message, slave_port,
 185                                 internals->mode4.dedicated_queues.rx_qid);
 186                 return -1;
 187         }
 188
 189         ret = rte_eth_dev_info_get(slave_port, &slave_info);
 190         if (ret != 0) {
 191                 RTE_BOND_LOG(ERR,
 192                         "%s: Error during getting device (port %u) info: %s\n",
 193                         __func__, slave_port, strerror(-ret));
 194
 195                 return ret;
 196         }
 197
 198         if (slave_info.max_rx_queues < bond_dev->data->nb_rx_queues ||
 199                         slave_info.max_tx_queues < bond_dev->data->nb_tx_queues) {
 200                 RTE_BOND_LOG(ERR,
 201                         "%s: Slave %d capabilities doesn't allow to allocate additional queues",
 202                         __func__, slave_port);
 203                 return -1;
 204         }
 205
 206         return 0;
 207 }
 208
 209 int
 210 bond_8023ad_slow_pkt_hw_filter_supported(uint16_t port_id) {
 211         struct rte_eth_dev *bond_dev = &rte_eth_devices[port_id];
 212         struct bond_dev_private *internals = bond_dev->data->dev_private;
 213         struct rte_eth_dev_info bond_info;
 214         uint16_t idx;
 215         int ret;
 216
 217         /* Verify if all slaves in bonding supports flow director and */
 218         if (internals->slave_count > 0) {
 219                 ret = rte_eth_dev_info_get(bond_dev->data->port_id, &bond_info);
 220                 if (ret != 0) {
 221                         RTE_BOND_LOG(ERR,
 222                                 "%s: Error during getting device (port %u) info: %s\n",
 223                                 __func__, bond_dev->data->port_id,
 224                                 strerror(-ret));
 225
 226                         return ret;
 227                 }
 228
 229                 internals->mode4.dedicated_queues.rx_qid = bond_info.nb_rx_queues;
 230                 internals->mode4.dedicated_queues.tx_qid = bond_info.nb_tx_queues;
 231
 232                 for (idx = 0; idx < internals->slave_count; idx++) {
 233                         if (bond_ethdev_8023ad_flow_verify(bond_dev,
 234                                         internals->slaves[idx].port_id) != 0)
 235                                 return -1;
 236                 }
 237         }
 238
 239         return 0;
 240 }
 241
 242 int
 243 bond_ethdev_8023ad_flow_set(struct rte_eth_dev *bond_dev, uint16_t slave_port) {
 244
 245         struct rte_flow_error error;
 246         struct bond_dev_private *internals = bond_dev->data->dev_private;
 247         struct rte_flow_action_queue lacp_queue_conf = {
 248                 .index = internals->mode4.dedicated_queues.rx_qid,
 249         };
 250
 251         const struct rte_flow_action actions[] = {
 252                 {
 253                         .type = RTE_FLOW_ACTION_TYPE_QUEUE,
 254                         .conf = &lacp_queue_conf
 255                 },
 256                 {
 257                         .type = RTE_FLOW_ACTION_TYPE_END,
 258                 }
 259         };
 260
 261         internals->mode4.dedicated_queues.flow[slave_port] = rte_flow_create(slave_port,
 262                         &flow_attr_8023ad, flow_item_8023ad, actions, &error);
 263         if (internals->mode4.dedicated_queues.flow[slave_port] == NULL) {
 264                 RTE_BOND_LOG(ERR, "bond_ethdev_8023ad_flow_set: %s "
 265                                 "(slave_port=%d queue_id=%d)",
 266                                 error.message, slave_port,
 267                                 internals->mode4.dedicated_queues.rx_qid);
 268                 return -1;
 269         }
 270
 271         return 0;
 272 }
 273
 274 static inline uint16_t
 275 rx_burst_8023ad(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts,
 276                 bool dedicated_rxq)
 277 {
 278         /* Cast to structure, containing bonded device's port id and queue id */
 279         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
 280         struct bond_dev_private *internals = bd_rx_q->dev_private;
 281         struct rte_eth_dev *bonded_eth_dev =
 282                                         &rte_eth_devices[internals->port_id];
 283         struct rte_ether_addr *bond_mac = bonded_eth_dev->data->mac_addrs;
 284         struct rte_ether_hdr *hdr;
 285
 286         const uint16_t ether_type_slow_be =
 287                 rte_be_to_cpu_16(RTE_ETHER_TYPE_SLOW);
 288         uint16_t num_rx_total = 0;      /* Total number of received packets */
 289         uint16_t slaves[RTE_MAX_ETHPORTS];
 290         uint16_t slave_count, idx;
 291
 292         uint8_t collecting;  /* current slave collecting status */
 293         const uint8_t promisc = rte_eth_promiscuous_get(internals->port_id);
 294         const uint8_t allmulti = rte_eth_allmulticast_get(internals->port_id);
 295         uint8_t subtype;
 296         uint16_t i;
 297         uint16_t j;
 298         uint16_t k;
 299
 300         /* Copy slave list to protect against slave up/down changes during tx
 301          * bursting */
 302         slave_count = internals->active_slave_count;
 303         memcpy(slaves, internals->active_slaves,
 304                         sizeof(internals->active_slaves[0]) * slave_count);
 305
 306         idx = bd_rx_q->active_slave;
 307         if (idx >= slave_count) {
 308                 bd_rx_q->active_slave = 0;
 309                 idx = 0;
 310         }
 311         for (i = 0; i < slave_count && num_rx_total < nb_pkts; i++) {
 312                 j = num_rx_total;
 313                 collecting = ACTOR_STATE(&bond_mode_8023ad_ports[slaves[idx]],
 314                                          COLLECTING);
 315
 316                 /* Read packets from this slave */
 317                 num_rx_total += rte_eth_rx_burst(slaves[idx], bd_rx_q->queue_id,
 318                                 &bufs[num_rx_total], nb_pkts - num_rx_total);
 319
 320                 for (k = j; k < 2 && k < num_rx_total; k++)
 321                         rte_prefetch0(rte_pktmbuf_mtod(bufs[k], void *));
 322
 323                 /* Handle slow protocol packets. */
 324                 while (j < num_rx_total) {
 325                         if (j + 3 < num_rx_total)
 326                                 rte_prefetch0(rte_pktmbuf_mtod(bufs[j + 3], void *));
 327
 328                         hdr = rte_pktmbuf_mtod(bufs[j], struct rte_ether_hdr *);
 329                         subtype = ((struct slow_protocol_frame *)hdr)->slow_protocol.subtype;
 330
 331                         /* Remove packet from array if:
 332                          * - it is slow packet but no dedicated rxq is present,
 333                          * - slave is not in collecting state,
 334                          * - bonding interface is not in promiscuous mode:
 335                          *   - packet is unicast and address does not match,
 336                          *   - packet is multicast and bonding interface
 337                          *     is not in allmulti,
 338                          */
 339                         if (unlikely(
 340                                 (!dedicated_rxq &&
 341                                  is_lacp_packets(hdr->ether_type, subtype,
 342                                                  bufs[j])) ||
 343                                 !collecting ||
 344                                 (!promisc &&
 345                                  ((rte_is_unicast_ether_addr(&hdr->dst_addr) &&
 346                                    !rte_is_same_ether_addr(bond_mac,
 347                                                        &hdr->dst_addr)) ||
 348                                   (!allmulti &&
 349                                    rte_is_multicast_ether_addr(&hdr->dst_addr)))))) {
 350
 351                                 if (hdr->ether_type == ether_type_slow_be) {
 352                                         bond_mode_8023ad_handle_slow_pkt(
 353                                             internals, slaves[idx], bufs[j]);
 354                                 } else
 355                                         rte_pktmbuf_free(bufs[j]);
 356
 357                                 /* Packet is managed by mode 4 or dropped, shift the array */
 358                                 num_rx_total--;
 359                                 if (j < num_rx_total) {
 360                                         memmove(&bufs[j], &bufs[j + 1], sizeof(bufs[0]) *
 361                                                 (num_rx_total - j));
 362                                 }
 363                         } else
 364                                 j++;
 365                 }
 366                 if (unlikely(++idx == slave_count))
 367                         idx = 0;
 368         }
 369
 370         if (++bd_rx_q->active_slave >= slave_count)
 371                 bd_rx_q->active_slave = 0;
 372
 373         return num_rx_total;
 374 }
 375
 376 static uint16_t
 377 bond_ethdev_rx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
 378                 uint16_t nb_pkts)
 379 {
 380         return rx_burst_8023ad(queue, bufs, nb_pkts, false);
 381 }
 382
 383 static uint16_t
 384 bond_ethdev_rx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs,
 385                 uint16_t nb_pkts)
 386 {
 387         return rx_burst_8023ad(queue, bufs, nb_pkts, true);
 388 }
 389
 390 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 391 uint32_t burstnumberRX;
 392 uint32_t burstnumberTX;
 393
 394 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 395
 396 static void
 397 arp_op_name(uint16_t arp_op, char *buf, size_t buf_len)
 398 {
 399         switch (arp_op) {
 400         case RTE_ARP_OP_REQUEST:
 401                 strlcpy(buf, "ARP Request", buf_len);
 402                 return;
 403         case RTE_ARP_OP_REPLY:
 404                 strlcpy(buf, "ARP Reply", buf_len);
 405                 return;
 406         case RTE_ARP_OP_REVREQUEST:
 407                 strlcpy(buf, "Reverse ARP Request", buf_len);
 408                 return;
 409         case RTE_ARP_OP_REVREPLY:
 410                 strlcpy(buf, "Reverse ARP Reply", buf_len);
 411                 return;
 412         case RTE_ARP_OP_INVREQUEST:
 413                 strlcpy(buf, "Peer Identify Request", buf_len);
 414                 return;
 415         case RTE_ARP_OP_INVREPLY:
 416                 strlcpy(buf, "Peer Identify Reply", buf_len);
 417                 return;
 418         default:
 419                 break;
 420         }
 421         strlcpy(buf, "Unknown", buf_len);
 422         return;
 423 }
 424 #endif
 425 #define MaxIPv4String   16
 426 static void
 427 ipv4_addr_to_dot(uint32_t be_ipv4_addr, char *buf, uint8_t buf_size)
 428 {
 429         uint32_t ipv4_addr;
 430
 431         ipv4_addr = rte_be_to_cpu_32(be_ipv4_addr);
 432         snprintf(buf, buf_size, "%d.%d.%d.%d", (ipv4_addr >> 24) & 0xFF,
 433                 (ipv4_addr >> 16) & 0xFF, (ipv4_addr >> 8) & 0xFF,
 434                 ipv4_addr & 0xFF);
 435 }
 436
 437 #define MAX_CLIENTS_NUMBER      128
 438 uint8_t active_clients;
 439 struct client_stats_t {
 440         uint16_t port;
 441         uint32_t ipv4_addr;
 442         uint32_t ipv4_rx_packets;
 443         uint32_t ipv4_tx_packets;
 444 };
 445 struct client_stats_t client_stats[MAX_CLIENTS_NUMBER];
 446
 447 static void
 448 update_client_stats(uint32_t addr, uint16_t port, uint32_t *TXorRXindicator)
 449 {
 450         int i = 0;
 451
 452         for (; i < MAX_CLIENTS_NUMBER; i++)     {
 453                 if ((client_stats[i].ipv4_addr == addr) && (client_stats[i].port == port))      {
 454                         /* Just update RX packets number for this client */
 455                         if (TXorRXindicator == &burstnumberRX)
 456                                 client_stats[i].ipv4_rx_packets++;
 457                         else
 458                                 client_stats[i].ipv4_tx_packets++;
 459                         return;
 460                 }
 461         }
 462         /* We have a new client. Insert him to the table, and increment stats */
 463         if (TXorRXindicator == &burstnumberRX)
 464                 client_stats[active_clients].ipv4_rx_packets++;
 465         else
 466                 client_stats[active_clients].ipv4_tx_packets++;
 467         client_stats[active_clients].ipv4_addr = addr;
 468         client_stats[active_clients].port = port;
 469         active_clients++;
 470
 471 }
 472
 473 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 474 #define MODE6_DEBUG(info, src_ip, dst_ip, eth_h, arp_op, port, burstnumber) \
 475         rte_log(RTE_LOG_DEBUG, bond_logtype,                            \
 476                 "%s port:%d SrcMAC:" RTE_ETHER_ADDR_PRT_FMT " SrcIP:%s " \
 477                 "DstMAC:" RTE_ETHER_ADDR_PRT_FMT " DstIP:%s %s %d\n", \
 478                 info,                                                   \
 479                 port,                                                   \
 480                 RTE_ETHER_ADDR_BYTES(&eth_h->src_addr),                  \
 481                 src_ip,                                                 \
 482                 RTE_ETHER_ADDR_BYTES(&eth_h->dst_addr),                  \
 483                 dst_ip,                                                 \
 484                 arp_op, ++burstnumber)
 485 #endif
 486
 487 static void
 488 mode6_debug(const char __rte_unused *info,
 489         struct rte_ether_hdr *eth_h, uint16_t port,
 490         uint32_t __rte_unused *burstnumber)
 491 {
 492         struct rte_ipv4_hdr *ipv4_h;
 493 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 494         struct rte_arp_hdr *arp_h;
 495         char dst_ip[16];
 496         char ArpOp[24];
 497         char buf[16];
 498 #endif
 499         char src_ip[16];
 500
 501         uint16_t ether_type = eth_h->ether_type;
 502         uint16_t offset = get_vlan_offset(eth_h, &ether_type);
 503
 504 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 505         strlcpy(buf, info, 16);
 506 #endif
 507
 508         if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
 509                 ipv4_h = (struct rte_ipv4_hdr *)((char *)(eth_h + 1) + offset);
 510                 ipv4_addr_to_dot(ipv4_h->src_addr, src_ip, MaxIPv4String);
 511 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 512                 ipv4_addr_to_dot(ipv4_h->dst_addr, dst_ip, MaxIPv4String);
 513                 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, "", port, *burstnumber);
 514 #endif
 515                 update_client_stats(ipv4_h->src_addr, port, burstnumber);
 516         }
 517 #ifdef RTE_LIBRTE_BOND_DEBUG_ALB
 518         else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) {
 519                 arp_h = (struct rte_arp_hdr *)((char *)(eth_h + 1) + offset);
 520                 ipv4_addr_to_dot(arp_h->arp_data.arp_sip, src_ip, MaxIPv4String);
 521                 ipv4_addr_to_dot(arp_h->arp_data.arp_tip, dst_ip, MaxIPv4String);
 522                 arp_op_name(rte_be_to_cpu_16(arp_h->arp_opcode),
 523                                 ArpOp, sizeof(ArpOp));
 524                 MODE6_DEBUG(buf, src_ip, dst_ip, eth_h, ArpOp, port, *burstnumber);
 525         }
 526 #endif
 527 }
 528 #endif
 529
 530 static uint16_t
 531 bond_ethdev_rx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 532 {
 533         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)queue;
 534         struct bond_dev_private *internals = bd_rx_q->dev_private;
 535         struct rte_ether_hdr *eth_h;
 536         uint16_t ether_type, offset;
 537         uint16_t nb_recv_pkts;
 538         int i;
 539
 540         nb_recv_pkts = bond_ethdev_rx_burst(queue, bufs, nb_pkts);
 541
 542         for (i = 0; i < nb_recv_pkts; i++) {
 543                 eth_h = rte_pktmbuf_mtod(bufs[i], struct rte_ether_hdr *);
 544                 ether_type = eth_h->ether_type;
 545                 offset = get_vlan_offset(eth_h, &ether_type);
 546
 547                 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) {
 548 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 549                         mode6_debug("RX ARP:", eth_h, bufs[i]->port, &burstnumberRX);
 550 #endif
 551                         bond_mode_alb_arp_recv(eth_h, offset, internals);
 552                 }
 553 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 554                 else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4))
 555                         mode6_debug("RX IPv4:", eth_h, bufs[i]->port, &burstnumberRX);
 556 #endif
 557         }
 558
 559         return nb_recv_pkts;
 560 }
 561
 562 static uint16_t
 563 bond_ethdev_tx_burst_round_robin(void *queue, struct rte_mbuf **bufs,
 564                 uint16_t nb_pkts)
 565 {
 566         struct bond_dev_private *internals;
 567         struct bond_tx_queue *bd_tx_q;
 568
 569         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_pkts];
 570         uint16_t slave_nb_pkts[RTE_MAX_ETHPORTS] = { 0 };
 571
 572         uint16_t num_of_slaves;
 573         uint16_t slaves[RTE_MAX_ETHPORTS];
 574
 575         uint16_t num_tx_total = 0, num_tx_slave;
 576
 577         static int slave_idx = 0;
 578         int i, cslave_idx = 0, tx_fail_total = 0;
 579
 580         bd_tx_q = (struct bond_tx_queue *)queue;
 581         internals = bd_tx_q->dev_private;
 582
 583         /* Copy slave list to protect against slave up/down changes during tx
 584          * bursting */
 585         num_of_slaves = internals->active_slave_count;
 586         memcpy(slaves, internals->active_slaves,
 587                         sizeof(internals->active_slaves[0]) * num_of_slaves);
 588
 589         if (num_of_slaves < 1)
 590                 return num_tx_total;
 591
 592         /* Populate slaves mbuf with which packets are to be sent on it  */
 593         for (i = 0; i < nb_pkts; i++) {
 594                 cslave_idx = (slave_idx + i) % num_of_slaves;
 595                 slave_bufs[cslave_idx][(slave_nb_pkts[cslave_idx])++] = bufs[i];
 596         }
 597
 598         /* increment current slave index so the next call to tx burst starts on the
 599          * next slave */
 600         slave_idx = ++cslave_idx;
 601
 602         /* Send packet burst on each slave device */
 603         for (i = 0; i < num_of_slaves; i++) {
 604                 if (slave_nb_pkts[i] > 0) {
 605                         num_tx_slave = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
 606                                         slave_bufs[i], slave_nb_pkts[i]);
 607
 608                         /* if tx burst fails move packets to end of bufs */
 609                         if (unlikely(num_tx_slave < slave_nb_pkts[i])) {
 610                                 int tx_fail_slave = slave_nb_pkts[i] - num_tx_slave;
 611
 612                                 tx_fail_total += tx_fail_slave;
 613
 614                                 memcpy(&bufs[nb_pkts - tx_fail_total],
 615                                        &slave_bufs[i][num_tx_slave],
 616                                        tx_fail_slave * sizeof(bufs[0]));
 617                         }
 618                         num_tx_total += num_tx_slave;
 619                 }
 620         }
 621
 622         return num_tx_total;
 623 }
 624
 625 static uint16_t
 626 bond_ethdev_tx_burst_active_backup(void *queue,
 627                 struct rte_mbuf **bufs, uint16_t nb_pkts)
 628 {
 629         struct bond_dev_private *internals;
 630         struct bond_tx_queue *bd_tx_q;
 631
 632         bd_tx_q = (struct bond_tx_queue *)queue;
 633         internals = bd_tx_q->dev_private;
 634
 635         if (internals->active_slave_count < 1)
 636                 return 0;
 637
 638         return rte_eth_tx_burst(internals->current_primary_port, bd_tx_q->queue_id,
 639                         bufs, nb_pkts);
 640 }
 641
 642 static inline uint16_t
 643 ether_hash(struct rte_ether_hdr *eth_hdr)
 644 {
 645         unaligned_uint16_t *word_src_addr =
 646                 (unaligned_uint16_t *)eth_hdr->src_addr.addr_bytes;
 647         unaligned_uint16_t *word_dst_addr =
 648                 (unaligned_uint16_t *)eth_hdr->dst_addr.addr_bytes;
 649
 650         return (word_src_addr[0] ^ word_dst_addr[0]) ^
 651                         (word_src_addr[1] ^ word_dst_addr[1]) ^
 652                         (word_src_addr[2] ^ word_dst_addr[2]);
 653 }
 654
 655 static inline uint32_t
 656 ipv4_hash(struct rte_ipv4_hdr *ipv4_hdr)
 657 {
 658         return ipv4_hdr->src_addr ^ ipv4_hdr->dst_addr;
 659 }
 660
 661 static inline uint32_t
 662 ipv6_hash(struct rte_ipv6_hdr *ipv6_hdr)
 663 {
 664         unaligned_uint32_t *word_src_addr =
 665                 (unaligned_uint32_t *)&(ipv6_hdr->src_addr[0]);
 666         unaligned_uint32_t *word_dst_addr =
 667                 (unaligned_uint32_t *)&(ipv6_hdr->dst_addr[0]);
 668
 669         return (word_src_addr[0] ^ word_dst_addr[0]) ^
 670                         (word_src_addr[1] ^ word_dst_addr[1]) ^
 671                         (word_src_addr[2] ^ word_dst_addr[2]) ^
 672                         (word_src_addr[3] ^ word_dst_addr[3]);
 673 }
 674
 675
 676 void
 677 burst_xmit_l2_hash(struct rte_mbuf **buf, uint16_t nb_pkts,
 678                 uint16_t slave_count, uint16_t *slaves)
 679 {
 680         struct rte_ether_hdr *eth_hdr;
 681         uint32_t hash;
 682         int i;
 683
 684         for (i = 0; i < nb_pkts; i++) {
 685                 eth_hdr = rte_pktmbuf_mtod(buf[i], struct rte_ether_hdr *);
 686
 687                 hash = ether_hash(eth_hdr);
 688
 689                 slaves[i] = (hash ^= hash >> 8) % slave_count;
 690         }
 691 }
 692
 693 void
 694 burst_xmit_l23_hash(struct rte_mbuf **buf, uint16_t nb_pkts,
 695                 uint16_t slave_count, uint16_t *slaves)
 696 {
 697         uint16_t i;
 698         struct rte_ether_hdr *eth_hdr;
 699         uint16_t proto;
 700         size_t vlan_offset;
 701         uint32_t hash, l3hash;
 702
 703         for (i = 0; i < nb_pkts; i++) {
 704                 eth_hdr = rte_pktmbuf_mtod(buf[i], struct rte_ether_hdr *);
 705                 l3hash = 0;
 706
 707                 proto = eth_hdr->ether_type;
 708                 hash = ether_hash(eth_hdr);
 709
 710                 vlan_offset = get_vlan_offset(eth_hdr, &proto);
 711
 712                 if (rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4) == proto) {
 713                         struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)
 714                                         ((char *)(eth_hdr + 1) + vlan_offset);
 715                         l3hash = ipv4_hash(ipv4_hdr);
 716
 717                 } else if (rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6) == proto) {
 718                         struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)
 719                                         ((char *)(eth_hdr + 1) + vlan_offset);
 720                         l3hash = ipv6_hash(ipv6_hdr);
 721                 }
 722
 723                 hash = hash ^ l3hash;
 724                 hash ^= hash >> 16;
 725                 hash ^= hash >> 8;
 726
 727                 slaves[i] = hash % slave_count;
 728         }
 729 }
 730
 731 void
 732 burst_xmit_l34_hash(struct rte_mbuf **buf, uint16_t nb_pkts,
 733                 uint16_t slave_count, uint16_t *slaves)
 734 {
 735         struct rte_ether_hdr *eth_hdr;
 736         uint16_t proto;
 737         size_t vlan_offset;
 738         int i;
 739
 740         struct rte_udp_hdr *udp_hdr;
 741         struct rte_tcp_hdr *tcp_hdr;
 742         uint32_t hash, l3hash, l4hash;
 743
 744         for (i = 0; i < nb_pkts; i++) {
 745                 eth_hdr = rte_pktmbuf_mtod(buf[i], struct rte_ether_hdr *);
 746                 size_t pkt_end = (size_t)eth_hdr + rte_pktmbuf_data_len(buf[i]);
 747                 proto = eth_hdr->ether_type;
 748                 vlan_offset = get_vlan_offset(eth_hdr, &proto);
 749                 l3hash = 0;
 750                 l4hash = 0;
 751
 752                 if (rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4) == proto) {
 753                         struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)
 754                                         ((char *)(eth_hdr + 1) + vlan_offset);
 755                         size_t ip_hdr_offset;
 756
 757                         l3hash = ipv4_hash(ipv4_hdr);
 758
 759                         /* there is no L4 header in fragmented packet */
 760                         if (likely(rte_ipv4_frag_pkt_is_fragmented(ipv4_hdr)
 761                                                                 == 0)) {
 762                                 ip_hdr_offset = (ipv4_hdr->version_ihl
 763                                         & RTE_IPV4_HDR_IHL_MASK) *
 764                                         RTE_IPV4_IHL_MULTIPLIER;
 765
 766                                 if (ipv4_hdr->next_proto_id == IPPROTO_TCP) {
 767                                         tcp_hdr = (struct rte_tcp_hdr *)
 768                                                 ((char *)ipv4_hdr +
 769                                                         ip_hdr_offset);
 770                                         if ((size_t)tcp_hdr + sizeof(*tcp_hdr)
 771                                                         < pkt_end)
 772                                                 l4hash = HASH_L4_PORTS(tcp_hdr);
 773                                 } else if (ipv4_hdr->next_proto_id ==
 774                                                                 IPPROTO_UDP) {
 775                                         udp_hdr = (struct rte_udp_hdr *)
 776                                                 ((char *)ipv4_hdr +
 777                                                         ip_hdr_offset);
 778                                         if ((size_t)udp_hdr + sizeof(*udp_hdr)
 779                                                         < pkt_end)
 780                                                 l4hash = HASH_L4_PORTS(udp_hdr);
 781                                 }
 782                         }
 783                 } else if  (rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6) == proto) {
 784                         struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)
 785                                         ((char *)(eth_hdr + 1) + vlan_offset);
 786                         l3hash = ipv6_hash(ipv6_hdr);
 787
 788                         if (ipv6_hdr->proto == IPPROTO_TCP) {
 789                                 tcp_hdr = (struct rte_tcp_hdr *)(ipv6_hdr + 1);
 790                                 l4hash = HASH_L4_PORTS(tcp_hdr);
 791                         } else if (ipv6_hdr->proto == IPPROTO_UDP) {
 792                                 udp_hdr = (struct rte_udp_hdr *)(ipv6_hdr + 1);
 793                                 l4hash = HASH_L4_PORTS(udp_hdr);
 794                         }
 795                 }
 796
 797                 hash = l3hash ^ l4hash;
 798                 hash ^= hash >> 16;
 799                 hash ^= hash >> 8;
 800
 801                 slaves[i] = hash % slave_count;
 802         }
 803 }
 804
 805 struct bwg_slave {
 806         uint64_t bwg_left_int;
 807         uint64_t bwg_left_remainder;
 808         uint16_t slave;
 809 };
 810
 811 void
 812 bond_tlb_activate_slave(struct bond_dev_private *internals) {
 813         int i;
 814
 815         for (i = 0; i < internals->active_slave_count; i++) {
 816                 tlb_last_obytets[internals->active_slaves[i]] = 0;
 817         }
 818 }
 819
 820 static int
 821 bandwidth_cmp(const void *a, const void *b)
 822 {
 823         const struct bwg_slave *bwg_a = a;
 824         const struct bwg_slave *bwg_b = b;
 825         int64_t diff = (int64_t)bwg_b->bwg_left_int - (int64_t)bwg_a->bwg_left_int;
 826         int64_t diff2 = (int64_t)bwg_b->bwg_left_remainder -
 827                         (int64_t)bwg_a->bwg_left_remainder;
 828         if (diff > 0)
 829                 return 1;
 830         else if (diff < 0)
 831                 return -1;
 832         else if (diff2 > 0)
 833                 return 1;
 834         else if (diff2 < 0)
 835                 return -1;
 836         else
 837                 return 0;
 838 }
 839
 840 static void
 841 bandwidth_left(uint16_t port_id, uint64_t load, uint8_t update_idx,
 842                 struct bwg_slave *bwg_slave)
 843 {
 844         struct rte_eth_link link_status;
 845         int ret;
 846
 847         ret = rte_eth_link_get_nowait(port_id, &link_status);
 848         if (ret < 0) {
 849                 RTE_BOND_LOG(ERR, "Slave (port %u) link get failed: %s",
 850                              port_id, rte_strerror(-ret));
 851                 return;
 852         }
 853         uint64_t link_bwg = link_status.link_speed * 1000000ULL / 8;
 854         if (link_bwg == 0)
 855                 return;
 856         link_bwg = link_bwg * (update_idx+1) * REORDER_PERIOD_MS;
 857         bwg_slave->bwg_left_int = (link_bwg - 1000*load) / link_bwg;
 858         bwg_slave->bwg_left_remainder = (link_bwg - 1000*load) % link_bwg;
 859 }
 860
 861 static void
 862 bond_ethdev_update_tlb_slave_cb(void *arg)
 863 {
 864         struct bond_dev_private *internals = arg;
 865         struct rte_eth_stats slave_stats;
 866         struct bwg_slave bwg_array[RTE_MAX_ETHPORTS];
 867         uint16_t slave_count;
 868         uint64_t tx_bytes;
 869
 870         uint8_t update_stats = 0;
 871         uint16_t slave_id;
 872         uint16_t i;
 873
 874         internals->slave_update_idx++;
 875
 876
 877         if (internals->slave_update_idx >= REORDER_PERIOD_MS)
 878                 update_stats = 1;
 879
 880         for (i = 0; i < internals->active_slave_count; i++) {
 881                 slave_id = internals->active_slaves[i];
 882                 rte_eth_stats_get(slave_id, &slave_stats);
 883                 tx_bytes = slave_stats.obytes - tlb_last_obytets[slave_id];
 884                 bandwidth_left(slave_id, tx_bytes,
 885                                 internals->slave_update_idx, &bwg_array[i]);
 886                 bwg_array[i].slave = slave_id;
 887
 888                 if (update_stats) {
 889                         tlb_last_obytets[slave_id] = slave_stats.obytes;
 890                 }
 891         }
 892
 893         if (update_stats == 1)
 894                 internals->slave_update_idx = 0;
 895
 896         slave_count = i;
 897         qsort(bwg_array, slave_count, sizeof(bwg_array[0]), bandwidth_cmp);
 898         for (i = 0; i < slave_count; i++)
 899                 internals->tlb_slaves_order[i] = bwg_array[i].slave;
 900
 901         rte_eal_alarm_set(REORDER_PERIOD_MS * 1000, bond_ethdev_update_tlb_slave_cb,
 902                         (struct bond_dev_private *)internals);
 903 }
 904
 905 static uint16_t
 906 bond_ethdev_tx_burst_tlb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 907 {
 908         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
 909         struct bond_dev_private *internals = bd_tx_q->dev_private;
 910
 911         struct rte_eth_dev *primary_port =
 912                         &rte_eth_devices[internals->primary_port];
 913         uint16_t num_tx_total = 0;
 914         uint16_t i, j;
 915
 916         uint16_t num_of_slaves = internals->active_slave_count;
 917         uint16_t slaves[RTE_MAX_ETHPORTS];
 918
 919         struct rte_ether_hdr *ether_hdr;
 920         struct rte_ether_addr primary_slave_addr;
 921         struct rte_ether_addr active_slave_addr;
 922
 923         if (num_of_slaves < 1)
 924                 return num_tx_total;
 925
 926         memcpy(slaves, internals->tlb_slaves_order,
 927                                 sizeof(internals->tlb_slaves_order[0]) * num_of_slaves);
 928
 929
 930         rte_ether_addr_copy(primary_port->data->mac_addrs, &primary_slave_addr);
 931
 932         if (nb_pkts > 3) {
 933                 for (i = 0; i < 3; i++)
 934                         rte_prefetch0(rte_pktmbuf_mtod(bufs[i], void*));
 935         }
 936
 937         for (i = 0; i < num_of_slaves; i++) {
 938                 rte_eth_macaddr_get(slaves[i], &active_slave_addr);
 939                 for (j = num_tx_total; j < nb_pkts; j++) {
 940                         if (j + 3 < nb_pkts)
 941                                 rte_prefetch0(rte_pktmbuf_mtod(bufs[j+3], void*));
 942
 943                         ether_hdr = rte_pktmbuf_mtod(bufs[j],
 944                                                 struct rte_ether_hdr *);
 945                         if (rte_is_same_ether_addr(&ether_hdr->src_addr,
 946                                                         &primary_slave_addr))
 947                                 rte_ether_addr_copy(&active_slave_addr,
 948                                                 &ether_hdr->src_addr);
 949 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
 950                                         mode6_debug("TX IPv4:", ether_hdr, slaves[i], &burstnumberTX);
 951 #endif
 952                 }
 953
 954                 num_tx_total += rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
 955                                 bufs + num_tx_total, nb_pkts - num_tx_total);
 956
 957                 if (num_tx_total == nb_pkts)
 958                         break;
 959         }
 960
 961         return num_tx_total;
 962 }
 963
 964 void
 965 bond_tlb_disable(struct bond_dev_private *internals)
 966 {
 967         rte_eal_alarm_cancel(bond_ethdev_update_tlb_slave_cb, internals);
 968 }
 969
 970 void
 971 bond_tlb_enable(struct bond_dev_private *internals)
 972 {
 973         bond_ethdev_update_tlb_slave_cb(internals);
 974 }
 975
 976 static uint16_t
 977 bond_ethdev_tx_burst_alb(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 978 {
 979         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
 980         struct bond_dev_private *internals = bd_tx_q->dev_private;
 981
 982         struct rte_ether_hdr *eth_h;
 983         uint16_t ether_type, offset;
 984
 985         struct client_data *client_info;
 986
 987         /*
 988          * We create transmit buffers for every slave and one additional to send
 989          * through tlb. In worst case every packet will be send on one port.
 990          */
 991         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS + 1][nb_pkts];
 992         uint16_t slave_bufs_pkts[RTE_MAX_ETHPORTS + 1] = { 0 };
 993
 994         /*
 995          * We create separate transmit buffers for update packets as they won't
 996          * be counted in num_tx_total.
 997          */
 998         struct rte_mbuf *update_bufs[RTE_MAX_ETHPORTS][ALB_HASH_TABLE_SIZE];
 999         uint16_t update_bufs_pkts[RTE_MAX_ETHPORTS] = { 0 };
1000
1001         struct rte_mbuf *upd_pkt;
1002         size_t pkt_size;
1003
1004         uint16_t num_send, num_not_send = 0;
1005         uint16_t num_tx_total = 0;
1006         uint16_t slave_idx;
1007
1008         int i, j;
1009
1010         /* Search tx buffer for ARP packets and forward them to alb */
1011         for (i = 0; i < nb_pkts; i++) {
1012                 eth_h = rte_pktmbuf_mtod(bufs[i], struct rte_ether_hdr *);
1013                 ether_type = eth_h->ether_type;
1014                 offset = get_vlan_offset(eth_h, &ether_type);
1015
1016                 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) {
1017                         slave_idx = bond_mode_alb_arp_xmit(eth_h, offset, internals);
1018
1019                         /* Change src mac in eth header */
1020                         rte_eth_macaddr_get(slave_idx, &eth_h->src_addr);
1021
1022                         /* Add packet to slave tx buffer */
1023                         slave_bufs[slave_idx][slave_bufs_pkts[slave_idx]] = bufs[i];
1024                         slave_bufs_pkts[slave_idx]++;
1025                 } else {
1026                         /* If packet is not ARP, send it with TLB policy */
1027                         slave_bufs[RTE_MAX_ETHPORTS][slave_bufs_pkts[RTE_MAX_ETHPORTS]] =
1028                                         bufs[i];
1029                         slave_bufs_pkts[RTE_MAX_ETHPORTS]++;
1030                 }
1031         }
1032
1033         /* Update connected client ARP tables */
1034         if (internals->mode6.ntt) {
1035                 for (i = 0; i < ALB_HASH_TABLE_SIZE; i++) {
1036                         client_info = &internals->mode6.client_table[i];
1037
1038                         if (client_info->in_use) {
1039                                 /* Allocate new packet to send ARP update on current slave */
1040                                 upd_pkt = rte_pktmbuf_alloc(internals->mode6.mempool);
1041                                 if (upd_pkt == NULL) {
1042                                         RTE_BOND_LOG(ERR,
1043                                                      "Failed to allocate ARP packet from pool");
1044                                         continue;
1045                                 }
1046                                 pkt_size = sizeof(struct rte_ether_hdr) +
1047                                         sizeof(struct rte_arp_hdr) +
1048                                         client_info->vlan_count *
1049                                         sizeof(struct rte_vlan_hdr);
1050                                 upd_pkt->data_len = pkt_size;
1051                                 upd_pkt->pkt_len = pkt_size;
1052
1053                                 slave_idx = bond_mode_alb_arp_upd(client_info, upd_pkt,
1054                                                 internals);
1055
1056                                 /* Add packet to update tx buffer */
1057                                 update_bufs[slave_idx][update_bufs_pkts[slave_idx]] = upd_pkt;
1058                                 update_bufs_pkts[slave_idx]++;
1059                         }
1060                 }
1061                 internals->mode6.ntt = 0;
1062         }
1063
1064         /* Send ARP packets on proper slaves */
1065         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
1066                 if (slave_bufs_pkts[i] > 0) {
1067                         num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id,
1068                                         slave_bufs[i], slave_bufs_pkts[i]);
1069                         for (j = 0; j < slave_bufs_pkts[i] - num_send; j++) {
1070                                 bufs[nb_pkts - 1 - num_not_send - j] =
1071                                                 slave_bufs[i][nb_pkts - 1 - j];
1072                         }
1073
1074                         num_tx_total += num_send;
1075                         num_not_send += slave_bufs_pkts[i] - num_send;
1076
1077 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
1078         /* Print TX stats including update packets */
1079                         for (j = 0; j < slave_bufs_pkts[i]; j++) {
1080                                 eth_h = rte_pktmbuf_mtod(slave_bufs[i][j],
1081                                                         struct rte_ether_hdr *);
1082                                 mode6_debug("TX ARP:", eth_h, i, &burstnumberTX);
1083                         }
1084 #endif
1085                 }
1086         }
1087
1088         /* Send update packets on proper slaves */
1089         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
1090                 if (update_bufs_pkts[i] > 0) {
1091                         num_send = rte_eth_tx_burst(i, bd_tx_q->queue_id, update_bufs[i],
1092                                         update_bufs_pkts[i]);
1093                         for (j = num_send; j < update_bufs_pkts[i]; j++) {
1094                                 rte_pktmbuf_free(update_bufs[i][j]);
1095                         }
1096 #if defined(RTE_LIBRTE_BOND_DEBUG_ALB) || defined(RTE_LIBRTE_BOND_DEBUG_ALB_L1)
1097                         for (j = 0; j < update_bufs_pkts[i]; j++) {
1098                                 eth_h = rte_pktmbuf_mtod(update_bufs[i][j],
1099                                                         struct rte_ether_hdr *);
1100                                 mode6_debug("TX ARPupd:", eth_h, i, &burstnumberTX);
1101                         }
1102 #endif
1103                 }
1104         }
1105
1106         /* Send non-ARP packets using tlb policy */
1107         if (slave_bufs_pkts[RTE_MAX_ETHPORTS] > 0) {
1108                 num_send = bond_ethdev_tx_burst_tlb(queue,
1109                                 slave_bufs[RTE_MAX_ETHPORTS],
1110                                 slave_bufs_pkts[RTE_MAX_ETHPORTS]);
1111
1112                 for (j = 0; j < slave_bufs_pkts[RTE_MAX_ETHPORTS]; j++) {
1113                         bufs[nb_pkts - 1 - num_not_send - j] =
1114                                         slave_bufs[RTE_MAX_ETHPORTS][nb_pkts - 1 - j];
1115                 }
1116
1117                 num_tx_total += num_send;
1118         }
1119
1120         return num_tx_total;
1121 }
1122
1123 static inline uint16_t
1124 tx_burst_balance(void *queue, struct rte_mbuf **bufs, uint16_t nb_bufs,
1125                  uint16_t *slave_port_ids, uint16_t slave_count)
1126 {
1127         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
1128         struct bond_dev_private *internals = bd_tx_q->dev_private;
1129
1130         /* Array to sort mbufs for transmission on each slave into */
1131         struct rte_mbuf *slave_bufs[RTE_MAX_ETHPORTS][nb_bufs];
1132         /* Number of mbufs for transmission on each slave */
1133         uint16_t slave_nb_bufs[RTE_MAX_ETHPORTS] = { 0 };
1134         /* Mapping array generated by hash function to map mbufs to slaves */
1135         uint16_t bufs_slave_port_idxs[nb_bufs];
1136
1137         uint16_t slave_tx_count;
1138         uint16_t total_tx_count = 0, total_tx_fail_count = 0;
1139
1140         uint16_t i;
1141
1142         /*
1143          * Populate slaves mbuf with the packets which are to be sent on it
1144          * selecting output slave using hash based on xmit policy
1145          */
1146         internals->burst_xmit_hash(bufs, nb_bufs, slave_count,
1147                         bufs_slave_port_idxs);
1148
1149         for (i = 0; i < nb_bufs; i++) {
1150                 /* Populate slave mbuf arrays with mbufs for that slave. */
1151                 uint16_t slave_idx = bufs_slave_port_idxs[i];
1152
1153                 slave_bufs[slave_idx][slave_nb_bufs[slave_idx]++] = bufs[i];
1154         }
1155
1156         /* Send packet burst on each slave device */
1157         for (i = 0; i < slave_count; i++) {
1158                 if (slave_nb_bufs[i] == 0)
1159                         continue;
1160
1161                 slave_tx_count = rte_eth_tx_burst(slave_port_ids[i],
1162                                 bd_tx_q->queue_id, slave_bufs[i],
1163                                 slave_nb_bufs[i]);
1164
1165                 total_tx_count += slave_tx_count;
1166
1167                 /* If tx burst fails move packets to end of bufs */
1168                 if (unlikely(slave_tx_count < slave_nb_bufs[i])) {
1169                         int slave_tx_fail_count = slave_nb_bufs[i] -
1170                                         slave_tx_count;
1171                         total_tx_fail_count += slave_tx_fail_count;
1172                         memcpy(&bufs[nb_bufs - total_tx_fail_count],
1173                                &slave_bufs[i][slave_tx_count],
1174                                slave_tx_fail_count * sizeof(bufs[0]));
1175                 }
1176         }
1177
1178         return total_tx_count;
1179 }
1180
1181 static uint16_t
1182 bond_ethdev_tx_burst_balance(void *queue, struct rte_mbuf **bufs,
1183                 uint16_t nb_bufs)
1184 {
1185         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
1186         struct bond_dev_private *internals = bd_tx_q->dev_private;
1187
1188         uint16_t slave_port_ids[RTE_MAX_ETHPORTS];
1189         uint16_t slave_count;
1190
1191         if (unlikely(nb_bufs == 0))
1192                 return 0;
1193
1194         /* Copy slave list to protect against slave up/down changes during tx
1195          * bursting
1196          */
1197         slave_count = internals->active_slave_count;
1198         if (unlikely(slave_count < 1))
1199                 return 0;
1200
1201         memcpy(slave_port_ids, internals->active_slaves,
1202                         sizeof(slave_port_ids[0]) * slave_count);
1203         return tx_burst_balance(queue, bufs, nb_bufs, slave_port_ids,
1204                                 slave_count);
1205 }
1206
1207 static inline uint16_t
1208 tx_burst_8023ad(void *queue, struct rte_mbuf **bufs, uint16_t nb_bufs,
1209                 bool dedicated_txq)
1210 {
1211         struct bond_tx_queue *bd_tx_q = (struct bond_tx_queue *)queue;
1212         struct bond_dev_private *internals = bd_tx_q->dev_private;
1213
1214         uint16_t slave_port_ids[RTE_MAX_ETHPORTS];
1215         uint16_t slave_count;
1216
1217         uint16_t dist_slave_port_ids[RTE_MAX_ETHPORTS];
1218         uint16_t dist_slave_count;
1219
1220         uint16_t slave_tx_count;
1221
1222         uint16_t i;
1223
1224         /* Copy slave list to protect against slave up/down changes during tx
1225          * bursting */
1226         slave_count = internals->active_slave_count;
1227         if (unlikely(slave_count < 1))
1228                 return 0;
1229
1230         memcpy(slave_port_ids, internals->active_slaves,
1231                         sizeof(slave_port_ids[0]) * slave_count);
1232
1233         if (dedicated_txq)
1234                 goto skip_tx_ring;
1235
1236         /* Check for LACP control packets and send if available */
1237         for (i = 0; i < slave_count; i++) {
1238                 struct port *port = &bond_mode_8023ad_ports[slave_port_ids[i]];
1239                 struct rte_mbuf *ctrl_pkt = NULL;
1240
1241                 if (likely(rte_ring_empty(port->tx_ring)))
1242                         continue;
1243
1244                 if (rte_ring_dequeue(port->tx_ring,
1245                                      (void **)&ctrl_pkt) != -ENOENT) {
1246                         slave_tx_count = rte_eth_tx_burst(slave_port_ids[i],
1247                                         bd_tx_q->queue_id, &ctrl_pkt, 1);
1248                         /*
1249                          * re-enqueue LAG control plane packets to buffering
1250                          * ring if transmission fails so the packet isn't lost.
1251                          */
1252                         if (slave_tx_count != 1)
1253                                 rte_ring_enqueue(port->tx_ring, ctrl_pkt);
1254                 }
1255         }
1256
1257 skip_tx_ring:
1258         if (unlikely(nb_bufs == 0))
1259                 return 0;
1260
1261         dist_slave_count = 0;
1262         for (i = 0; i < slave_count; i++) {
1263                 struct port *port = &bond_mode_8023ad_ports[slave_port_ids[i]];
1264
1265                 if (ACTOR_STATE(port, DISTRIBUTING))
1266                         dist_slave_port_ids[dist_slave_count++] =
1267                                         slave_port_ids[i];
1268         }
1269
1270         if (unlikely(dist_slave_count < 1))
1271                 return 0;
1272
1273         return tx_burst_balance(queue, bufs, nb_bufs, dist_slave_port_ids,
1274                                 dist_slave_count);
1275 }
1276
1277 static uint16_t
1278 bond_ethdev_tx_burst_8023ad(void *queue, struct rte_mbuf **bufs,
1279                 uint16_t nb_bufs)
1280 {
1281         return tx_burst_8023ad(queue, bufs, nb_bufs, false);
1282 }
1283
1284 static uint16_t
1285 bond_ethdev_tx_burst_8023ad_fast_queue(void *queue, struct rte_mbuf **bufs,
1286                 uint16_t nb_bufs)
1287 {
1288         return tx_burst_8023ad(queue, bufs, nb_bufs, true);
1289 }
1290
1291 static uint16_t
1292 bond_ethdev_tx_burst_broadcast(void *queue, struct rte_mbuf **bufs,
1293                 uint16_t nb_pkts)
1294 {
1295         struct bond_dev_private *internals;
1296         struct bond_tx_queue *bd_tx_q;
1297
1298         uint16_t slaves[RTE_MAX_ETHPORTS];
1299         uint8_t tx_failed_flag = 0;
1300         uint16_t num_of_slaves;
1301
1302         uint16_t max_nb_of_tx_pkts = 0;
1303
1304         int slave_tx_total[RTE_MAX_ETHPORTS];
1305         int i, most_successful_tx_slave = -1;
1306
1307         bd_tx_q = (struct bond_tx_queue *)queue;
1308         internals = bd_tx_q->dev_private;
1309
1310         /* Copy slave list to protect against slave up/down changes during tx
1311          * bursting */
1312         num_of_slaves = internals->active_slave_count;
1313         memcpy(slaves, internals->active_slaves,
1314                         sizeof(internals->active_slaves[0]) * num_of_slaves);
1315
1316         if (num_of_slaves < 1)
1317                 return 0;
1318
1319         /* Increment reference count on mbufs */
1320         for (i = 0; i < nb_pkts; i++)
1321                 rte_pktmbuf_refcnt_update(bufs[i], num_of_slaves - 1);
1322
1323         /* Transmit burst on each active slave */
1324         for (i = 0; i < num_of_slaves; i++) {
1325                 slave_tx_total[i] = rte_eth_tx_burst(slaves[i], bd_tx_q->queue_id,
1326                                         bufs, nb_pkts);
1327
1328                 if (unlikely(slave_tx_total[i] < nb_pkts))
1329                         tx_failed_flag = 1;
1330
1331                 /* record the value and slave index for the slave which transmits the
1332                  * maximum number of packets */
1333                 if (slave_tx_total[i] > max_nb_of_tx_pkts) {
1334                         max_nb_of_tx_pkts = slave_tx_total[i];
1335                         most_successful_tx_slave = i;
1336                 }
1337         }
1338
1339         /* if slaves fail to transmit packets from burst, the calling application
1340          * is not expected to know about multiple references to packets so we must
1341          * handle failures of all packets except those of the most successful slave
1342          */
1343         if (unlikely(tx_failed_flag))
1344                 for (i = 0; i < num_of_slaves; i++)
1345                         if (i != most_successful_tx_slave)
1346                                 while (slave_tx_total[i] < nb_pkts)
1347                                         rte_pktmbuf_free(bufs[slave_tx_total[i]++]);
1348
1349         return max_nb_of_tx_pkts;
1350 }
1351
1352 static void
1353 link_properties_set(struct rte_eth_dev *ethdev, struct rte_eth_link *slave_link)
1354 {
1355         struct bond_dev_private *bond_ctx = ethdev->data->dev_private;
1356
1357         if (bond_ctx->mode == BONDING_MODE_8023AD) {
1358                 /**
1359                  * If in mode 4 then save the link properties of the first
1360                  * slave, all subsequent slaves must match these properties
1361                  */
1362                 struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link;
1363
1364                 bond_link->link_autoneg = slave_link->link_autoneg;
1365                 bond_link->link_duplex = slave_link->link_duplex;
1366                 bond_link->link_speed = slave_link->link_speed;
1367         } else {
1368                 /**
1369                  * In any other mode the link properties are set to default
1370                  * values of AUTONEG/DUPLEX
1371                  */
1372                 ethdev->data->dev_link.link_autoneg = RTE_ETH_LINK_AUTONEG;
1373                 ethdev->data->dev_link.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1374         }
1375 }
1376
1377 static int
1378 link_properties_valid(struct rte_eth_dev *ethdev,
1379                 struct rte_eth_link *slave_link)
1380 {
1381         struct bond_dev_private *bond_ctx = ethdev->data->dev_private;
1382
1383         if (bond_ctx->mode == BONDING_MODE_8023AD) {
1384                 struct rte_eth_link *bond_link = &bond_ctx->mode4.slave_link;
1385
1386                 if (bond_link->link_duplex != slave_link->link_duplex ||
1387                         bond_link->link_autoneg != slave_link->link_autoneg ||
1388                         bond_link->link_speed != slave_link->link_speed)
1389                         return -1;
1390         }
1391
1392         return 0;
1393 }
1394
1395 int
1396 mac_address_get(struct rte_eth_dev *eth_dev,
1397                 struct rte_ether_addr *dst_mac_addr)
1398 {
1399         struct rte_ether_addr *mac_addr;
1400
1401         if (eth_dev == NULL) {
1402                 RTE_BOND_LOG(ERR, "NULL pointer eth_dev specified");
1403                 return -1;
1404         }
1405
1406         if (dst_mac_addr == NULL) {
1407                 RTE_BOND_LOG(ERR, "NULL pointer MAC specified");
1408                 return -1;
1409         }
1410
1411         mac_addr = eth_dev->data->mac_addrs;
1412
1413         rte_ether_addr_copy(mac_addr, dst_mac_addr);
1414         return 0;
1415 }
1416
1417 int
1418 mac_address_set(struct rte_eth_dev *eth_dev,
1419                 struct rte_ether_addr *new_mac_addr)
1420 {
1421         struct rte_ether_addr *mac_addr;
1422
1423         if (eth_dev == NULL) {
1424                 RTE_BOND_LOG(ERR, "NULL pointer eth_dev specified");
1425                 return -1;
1426         }
1427
1428         if (new_mac_addr == NULL) {
1429                 RTE_BOND_LOG(ERR, "NULL pointer MAC specified");
1430                 return -1;
1431         }
1432
1433         mac_addr = eth_dev->data->mac_addrs;
1434
1435         /* If new MAC is different to current MAC then update */
1436         if (memcmp(mac_addr, new_mac_addr, sizeof(*mac_addr)) != 0)
1437                 memcpy(mac_addr, new_mac_addr, sizeof(*mac_addr));
1438
1439         return 0;
1440 }
1441
1442 static const struct rte_ether_addr null_mac_addr;
1443
1444 /*
1445  * Add additional MAC addresses to the slave
1446  */
1447 int
1448 slave_add_mac_addresses(struct rte_eth_dev *bonded_eth_dev,
1449                 uint16_t slave_port_id)
1450 {
1451         int i, ret;
1452         struct rte_ether_addr *mac_addr;
1453
1454         for (i = 1; i < BOND_MAX_MAC_ADDRS; i++) {
1455                 mac_addr = &bonded_eth_dev->data->mac_addrs[i];
1456                 if (rte_is_same_ether_addr(mac_addr, &null_mac_addr))
1457                         break;
1458
1459                 ret = rte_eth_dev_mac_addr_add(slave_port_id, mac_addr, 0);
1460                 if (ret < 0) {
1461                         /* rollback */
1462                         for (i--; i > 0; i--)
1463                                 rte_eth_dev_mac_addr_remove(slave_port_id,
1464                                         &bonded_eth_dev->data->mac_addrs[i]);
1465                         return ret;
1466                 }
1467         }
1468
1469         return 0;
1470 }
1471
1472 /*
1473  * Remove additional MAC addresses from the slave
1474  */
1475 int
1476 slave_remove_mac_addresses(struct rte_eth_dev *bonded_eth_dev,
1477                 uint16_t slave_port_id)
1478 {
1479         int i, rc, ret;
1480         struct rte_ether_addr *mac_addr;
1481
1482         rc = 0;
1483         for (i = 1; i < BOND_MAX_MAC_ADDRS; i++) {
1484                 mac_addr = &bonded_eth_dev->data->mac_addrs[i];
1485                 if (rte_is_same_ether_addr(mac_addr, &null_mac_addr))
1486                         break;
1487
1488                 ret = rte_eth_dev_mac_addr_remove(slave_port_id, mac_addr);
1489                 /* save only the first error */
1490                 if (ret < 0 && rc == 0)
1491                         rc = ret;
1492         }
1493
1494         return rc;
1495 }
1496
1497 int
1498 mac_address_slaves_update(struct rte_eth_dev *bonded_eth_dev)
1499 {
1500         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1501         bool set;
1502         int i;
1503
1504         /* Update slave devices MAC addresses */
1505         if (internals->slave_count < 1)
1506                 return -1;
1507
1508         switch (internals->mode) {
1509         case BONDING_MODE_ROUND_ROBIN:
1510         case BONDING_MODE_BALANCE:
1511         case BONDING_MODE_BROADCAST:
1512                 for (i = 0; i < internals->slave_count; i++) {
1513                         if (rte_eth_dev_default_mac_addr_set(
1514                                         internals->slaves[i].port_id,
1515                                         bonded_eth_dev->data->mac_addrs)) {
1516                                 RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1517                                                 internals->slaves[i].port_id);
1518                                 return -1;
1519                         }
1520                 }
1521                 break;
1522         case BONDING_MODE_8023AD:
1523                 bond_mode_8023ad_mac_address_update(bonded_eth_dev);
1524                 break;
1525         case BONDING_MODE_ACTIVE_BACKUP:
1526         case BONDING_MODE_TLB:
1527         case BONDING_MODE_ALB:
1528         default:
1529                 set = true;
1530                 for (i = 0; i < internals->slave_count; i++) {
1531                         if (internals->slaves[i].port_id ==
1532                                         internals->current_primary_port) {
1533                                 if (rte_eth_dev_default_mac_addr_set(
1534                                                 internals->current_primary_port,
1535                                                 bonded_eth_dev->data->mac_addrs)) {
1536                                         RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1537                                                         internals->current_primary_port);
1538                                         set = false;
1539                                 }
1540                         } else {
1541                                 if (rte_eth_dev_default_mac_addr_set(
1542                                                 internals->slaves[i].port_id,
1543                                                 &internals->slaves[i].persisted_mac_addr)) {
1544                                         RTE_BOND_LOG(ERR, "Failed to update port Id %d MAC address",
1545                                                         internals->slaves[i].port_id);
1546                                 }
1547                         }
1548                 }
1549                 if (!set)
1550                         return -1;
1551         }
1552
1553         return 0;
1554 }
1555
1556 int
1557 bond_ethdev_mode_set(struct rte_eth_dev *eth_dev, uint8_t mode)
1558 {
1559         struct bond_dev_private *internals;
1560
1561         internals = eth_dev->data->dev_private;
1562
1563         switch (mode) {
1564         case BONDING_MODE_ROUND_ROBIN:
1565                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_round_robin;
1566                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1567                 break;
1568         case BONDING_MODE_ACTIVE_BACKUP:
1569                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_active_backup;
1570                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
1571                 break;
1572         case BONDING_MODE_BALANCE:
1573                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_balance;
1574                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1575                 break;
1576         case BONDING_MODE_BROADCAST:
1577                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_broadcast;
1578                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst;
1579                 break;
1580         case BONDING_MODE_8023AD:
1581                 if (bond_mode_8023ad_enable(eth_dev) != 0)
1582                         return -1;
1583
1584                 if (internals->mode4.dedicated_queues.enabled == 0) {
1585                         eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_8023ad;
1586                         eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_8023ad;
1587                         RTE_BOND_LOG(WARNING,
1588                                 "Using mode 4, it is necessary to do TX burst "
1589                                 "and RX burst at least every 100ms.");
1590                 } else {
1591                         /* Use flow director's optimization */
1592                         eth_dev->rx_pkt_burst =
1593                                         bond_ethdev_rx_burst_8023ad_fast_queue;
1594                         eth_dev->tx_pkt_burst =
1595                                         bond_ethdev_tx_burst_8023ad_fast_queue;
1596                 }
1597                 break;
1598         case BONDING_MODE_TLB:
1599                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_tlb;
1600                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_active_backup;
1601                 break;
1602         case BONDING_MODE_ALB:
1603                 if (bond_mode_alb_enable(eth_dev) != 0)
1604                         return -1;
1605
1606                 eth_dev->tx_pkt_burst = bond_ethdev_tx_burst_alb;
1607                 eth_dev->rx_pkt_burst = bond_ethdev_rx_burst_alb;
1608                 break;
1609         default:
1610                 return -1;
1611         }
1612
1613         internals->mode = mode;
1614
1615         return 0;
1616 }
1617
1618
1619 static int
1620 slave_configure_slow_queue(struct rte_eth_dev *bonded_eth_dev,
1621                 struct rte_eth_dev *slave_eth_dev)
1622 {
1623         int errval = 0;
1624         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1625         struct port *port = &bond_mode_8023ad_ports[slave_eth_dev->data->port_id];
1626
1627         if (port->slow_pool == NULL) {
1628                 char mem_name[256];
1629                 int slave_id = slave_eth_dev->data->port_id;
1630
1631                 snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_slow_pool",
1632                                 slave_id);
1633                 port->slow_pool = rte_pktmbuf_pool_create(mem_name, 8191,
1634                         250, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
1635                         slave_eth_dev->data->numa_node);
1636
1637                 /* Any memory allocation failure in initialization is critical because
1638                  * resources can't be free, so reinitialization is impossible. */
1639                 if (port->slow_pool == NULL) {
1640                         rte_panic("Slave %u: Failed to create memory pool '%s': %s\n",
1641                                 slave_id, mem_name, rte_strerror(rte_errno));
1642                 }
1643         }
1644
1645         if (internals->mode4.dedicated_queues.enabled == 1) {
1646                 /* Configure slow Rx queue */
1647
1648                 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id,
1649                                 internals->mode4.dedicated_queues.rx_qid, 128,
1650                                 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1651                                 NULL, port->slow_pool);
1652                 if (errval != 0) {
1653                         RTE_BOND_LOG(ERR,
1654                                         "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)",
1655                                         slave_eth_dev->data->port_id,
1656                                         internals->mode4.dedicated_queues.rx_qid,
1657                                         errval);
1658                         return errval;
1659                 }
1660
1661                 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id,
1662                                 internals->mode4.dedicated_queues.tx_qid, 512,
1663                                 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1664                                 NULL);
1665                 if (errval != 0) {
1666                         RTE_BOND_LOG(ERR,
1667                                 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)",
1668                                 slave_eth_dev->data->port_id,
1669                                 internals->mode4.dedicated_queues.tx_qid,
1670                                 errval);
1671                         return errval;
1672                 }
1673         }
1674         return 0;
1675 }
1676
1677 int
1678 slave_configure(struct rte_eth_dev *bonded_eth_dev,
1679                 struct rte_eth_dev *slave_eth_dev)
1680 {
1681         struct bond_rx_queue *bd_rx_q;
1682         struct bond_tx_queue *bd_tx_q;
1683         uint16_t nb_rx_queues;
1684         uint16_t nb_tx_queues;
1685
1686         int errval;
1687         uint16_t q_id;
1688         struct rte_flow_error flow_error;
1689
1690         struct bond_dev_private *internals = bonded_eth_dev->data->dev_private;
1691
1692         /* Stop slave */
1693         errval = rte_eth_dev_stop(slave_eth_dev->data->port_id);
1694         if (errval != 0)
1695                 RTE_BOND_LOG(ERR, "rte_eth_dev_stop: port %u, err (%d)",
1696                              slave_eth_dev->data->port_id, errval);
1697
1698         /* Enable interrupts on slave device if supported */
1699         if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1700                 slave_eth_dev->data->dev_conf.intr_conf.lsc = 1;
1701
1702         /* If RSS is enabled for bonding, try to enable it for slaves  */
1703         if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) {
1704                 /* rss_key won't be empty if RSS is configured in bonded dev */
1705                 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len =
1706                                         internals->rss_key_len;
1707                 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key =
1708                                         internals->rss_key;
1709
1710                 slave_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf =
1711                                 bonded_eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
1712                 slave_eth_dev->data->dev_conf.rxmode.mq_mode =
1713                                 bonded_eth_dev->data->dev_conf.rxmode.mq_mode;
1714         }
1715
1716         slave_eth_dev->data->dev_conf.rxmode.mtu =
1717                         bonded_eth_dev->data->dev_conf.rxmode.mtu;
1718
1719         slave_eth_dev->data->dev_conf.txmode.offloads |=
1720                 bonded_eth_dev->data->dev_conf.txmode.offloads;
1721
1722         slave_eth_dev->data->dev_conf.txmode.offloads &=
1723                 (bonded_eth_dev->data->dev_conf.txmode.offloads |
1724                 ~internals->tx_offload_capa);
1725
1726         slave_eth_dev->data->dev_conf.rxmode.offloads |=
1727                 bonded_eth_dev->data->dev_conf.rxmode.offloads;
1728
1729         slave_eth_dev->data->dev_conf.rxmode.offloads &=
1730                 (bonded_eth_dev->data->dev_conf.rxmode.offloads |
1731                 ~internals->rx_offload_capa);
1732
1733
1734         nb_rx_queues = bonded_eth_dev->data->nb_rx_queues;
1735         nb_tx_queues = bonded_eth_dev->data->nb_tx_queues;
1736
1737         if (internals->mode == BONDING_MODE_8023AD) {
1738                 if (internals->mode4.dedicated_queues.enabled == 1) {
1739                         nb_rx_queues++;
1740                         nb_tx_queues++;
1741                 }
1742         }
1743
1744         /* Configure device */
1745         errval = rte_eth_dev_configure(slave_eth_dev->data->port_id,
1746                         nb_rx_queues, nb_tx_queues,
1747                         &(slave_eth_dev->data->dev_conf));
1748         if (errval != 0) {
1749                 RTE_BOND_LOG(ERR, "Cannot configure slave device: port %u, err (%d)",
1750                                 slave_eth_dev->data->port_id, errval);
1751                 return errval;
1752         }
1753
1754         errval = rte_eth_dev_set_mtu(slave_eth_dev->data->port_id,
1755                                      bonded_eth_dev->data->mtu);
1756         if (errval != 0 && errval != -ENOTSUP) {
1757                 RTE_BOND_LOG(ERR, "rte_eth_dev_set_mtu: port %u, err (%d)",
1758                                 slave_eth_dev->data->port_id, errval);
1759                 return errval;
1760         }
1761
1762         /* Setup Rx Queues */
1763         for (q_id = 0; q_id < bonded_eth_dev->data->nb_rx_queues; q_id++) {
1764                 bd_rx_q = (struct bond_rx_queue *)bonded_eth_dev->data->rx_queues[q_id];
1765
1766                 errval = rte_eth_rx_queue_setup(slave_eth_dev->data->port_id, q_id,
1767                                 bd_rx_q->nb_rx_desc,
1768                                 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1769                                 &(bd_rx_q->rx_conf), bd_rx_q->mb_pool);
1770                 if (errval != 0) {
1771                         RTE_BOND_LOG(ERR,
1772                                         "rte_eth_rx_queue_setup: port=%d queue_id %d, err (%d)",
1773                                         slave_eth_dev->data->port_id, q_id, errval);
1774                         return errval;
1775                 }
1776         }
1777
1778         /* Setup Tx Queues */
1779         for (q_id = 0; q_id < bonded_eth_dev->data->nb_tx_queues; q_id++) {
1780                 bd_tx_q = (struct bond_tx_queue *)bonded_eth_dev->data->tx_queues[q_id];
1781
1782                 errval = rte_eth_tx_queue_setup(slave_eth_dev->data->port_id, q_id,
1783                                 bd_tx_q->nb_tx_desc,
1784                                 rte_eth_dev_socket_id(slave_eth_dev->data->port_id),
1785                                 &bd_tx_q->tx_conf);
1786                 if (errval != 0) {
1787                         RTE_BOND_LOG(ERR,
1788                                 "rte_eth_tx_queue_setup: port=%d queue_id %d, err (%d)",
1789                                 slave_eth_dev->data->port_id, q_id, errval);
1790                         return errval;
1791                 }
1792         }
1793
1794         if (internals->mode == BONDING_MODE_8023AD &&
1795                         internals->mode4.dedicated_queues.enabled == 1) {
1796                 if (slave_configure_slow_queue(bonded_eth_dev, slave_eth_dev)
1797                                 != 0)
1798                         return errval;
1799
1800                 errval = bond_ethdev_8023ad_flow_verify(bonded_eth_dev,
1801                                 slave_eth_dev->data->port_id);
1802                 if (errval != 0) {
1803                         RTE_BOND_LOG(ERR,
1804                                 "bond_ethdev_8023ad_flow_verify: port=%d, err (%d)",
1805                                 slave_eth_dev->data->port_id, errval);
1806                         return errval;
1807                 }
1808
1809                 if (internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id] != NULL)
1810                         rte_flow_destroy(slave_eth_dev->data->port_id,
1811                                         internals->mode4.dedicated_queues.flow[slave_eth_dev->data->port_id],
1812                                         &flow_error);
1813
1814                 errval = bond_ethdev_8023ad_flow_set(bonded_eth_dev,
1815                                 slave_eth_dev->data->port_id);
1816                 if (errval != 0) {
1817                         RTE_BOND_LOG(ERR,
1818                                 "bond_ethdev_8023ad_flow_set: port=%d, err (%d)",
1819                                 slave_eth_dev->data->port_id, errval);
1820                         return errval;
1821                 }
1822         }
1823
1824         /* Start device */
1825         errval = rte_eth_dev_start(slave_eth_dev->data->port_id);
1826         if (errval != 0) {
1827                 RTE_BOND_LOG(ERR, "rte_eth_dev_start: port=%u, err (%d)",
1828                                 slave_eth_dev->data->port_id, errval);
1829                 return -1;
1830         }
1831
1832         /* If RSS is enabled for bonding, synchronize RETA */
1833         if (bonded_eth_dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS) {
1834                 int i;
1835                 struct bond_dev_private *internals;
1836
1837                 internals = bonded_eth_dev->data->dev_private;
1838
1839                 for (i = 0; i < internals->slave_count; i++) {
1840                         if (internals->slaves[i].port_id == slave_eth_dev->data->port_id) {
1841                                 errval = rte_eth_dev_rss_reta_update(
1842                                                 slave_eth_dev->data->port_id,
1843                                                 &internals->reta_conf[0],
1844                                                 internals->slaves[i].reta_size);
1845                                 if (errval != 0) {
1846                                         RTE_BOND_LOG(WARNING,
1847                                                      "rte_eth_dev_rss_reta_update on slave port %d fails (err %d)."
1848                                                      " RSS Configuration for bonding may be inconsistent.",
1849                                                      slave_eth_dev->data->port_id, errval);
1850                                 }
1851                                 break;
1852                         }
1853                 }
1854         }
1855
1856         /* If lsc interrupt is set, check initial slave's link status */
1857         if (slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
1858                 slave_eth_dev->dev_ops->link_update(slave_eth_dev, 0);
1859                 bond_ethdev_lsc_event_callback(slave_eth_dev->data->port_id,
1860                         RTE_ETH_EVENT_INTR_LSC, &bonded_eth_dev->data->port_id,
1861                         NULL);
1862         }
1863
1864         return 0;
1865 }
1866
1867 void
1868 slave_remove(struct bond_dev_private *internals,
1869                 struct rte_eth_dev *slave_eth_dev)
1870 {
1871         uint16_t i;
1872
1873         for (i = 0; i < internals->slave_count; i++)
1874                 if (internals->slaves[i].port_id ==
1875                                 slave_eth_dev->data->port_id)
1876                         break;
1877
1878         if (i < (internals->slave_count - 1)) {
1879                 struct rte_flow *flow;
1880
1881                 memmove(&internals->slaves[i], &internals->slaves[i + 1],
1882                                 sizeof(internals->slaves[0]) *
1883                                 (internals->slave_count - i - 1));
1884                 TAILQ_FOREACH(flow, &internals->flow_list, next) {
1885                         memmove(&flow->flows[i], &flow->flows[i + 1],
1886                                 sizeof(flow->flows[0]) *
1887                                 (internals->slave_count - i - 1));
1888                         flow->flows[internals->slave_count - 1] = NULL;
1889                 }
1890         }
1891
1892         internals->slave_count--;
1893
1894         /* force reconfiguration of slave interfaces */
1895         rte_eth_dev_internal_reset(slave_eth_dev);
1896 }
1897
1898 static void
1899 bond_ethdev_slave_link_status_change_monitor(void *cb_arg);
1900
1901 void
1902 slave_add(struct bond_dev_private *internals,
1903                 struct rte_eth_dev *slave_eth_dev)
1904 {
1905         struct bond_slave_details *slave_details =
1906                         &internals->slaves[internals->slave_count];
1907
1908         slave_details->port_id = slave_eth_dev->data->port_id;
1909         slave_details->last_link_status = 0;
1910
1911         /* Mark slave devices that don't support interrupts so we can
1912          * compensate when we start the bond
1913          */
1914         if (!(slave_eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
1915                 slave_details->link_status_poll_enabled = 1;
1916         }
1917
1918         slave_details->link_status_wait_to_complete = 0;
1919         /* clean tlb_last_obytes when adding port for bonding device */
1920         memcpy(&(slave_details->persisted_mac_addr), slave_eth_dev->data->mac_addrs,
1921                         sizeof(struct rte_ether_addr));
1922 }
1923
1924 void
1925 bond_ethdev_primary_set(struct bond_dev_private *internals,
1926                 uint16_t slave_port_id)
1927 {
1928         int i;
1929
1930         if (internals->active_slave_count < 1)
1931                 internals->current_primary_port = slave_port_id;
1932         else
1933                 /* Search bonded device slave ports for new proposed primary port */
1934                 for (i = 0; i < internals->active_slave_count; i++) {
1935                         if (internals->active_slaves[i] == slave_port_id)
1936                                 internals->current_primary_port = slave_port_id;
1937                 }
1938 }
1939
1940 static int
1941 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev);
1942
1943 static int
1944 bond_ethdev_start(struct rte_eth_dev *eth_dev)
1945 {
1946         struct bond_dev_private *internals;
1947         int i;
1948
1949         /* slave eth dev will be started by bonded device */
1950         if (check_for_bonded_ethdev(eth_dev)) {
1951                 RTE_BOND_LOG(ERR, "User tried to explicitly start a slave eth_dev (%d)",
1952                                 eth_dev->data->port_id);
1953                 return -1;
1954         }
1955
1956         eth_dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
1957         eth_dev->data->dev_started = 1;
1958
1959         internals = eth_dev->data->dev_private;
1960
1961         if (internals->slave_count == 0) {
1962                 RTE_BOND_LOG(ERR, "Cannot start port since there are no slave devices");
1963                 goto out_err;
1964         }
1965
1966         if (internals->user_defined_mac == 0) {
1967                 struct rte_ether_addr *new_mac_addr = NULL;
1968
1969                 for (i = 0; i < internals->slave_count; i++)
1970                         if (internals->slaves[i].port_id == internals->primary_port)
1971                                 new_mac_addr = &internals->slaves[i].persisted_mac_addr;
1972
1973                 if (new_mac_addr == NULL)
1974                         goto out_err;
1975
1976                 if (mac_address_set(eth_dev, new_mac_addr) != 0) {
1977                         RTE_BOND_LOG(ERR, "bonded port (%d) failed to update MAC address",
1978                                         eth_dev->data->port_id);
1979                         goto out_err;
1980                 }
1981         }
1982
1983         if (internals->mode == BONDING_MODE_8023AD) {
1984                 if (internals->mode4.dedicated_queues.enabled == 1) {
1985                         internals->mode4.dedicated_queues.rx_qid =
1986                                         eth_dev->data->nb_rx_queues;
1987                         internals->mode4.dedicated_queues.tx_qid =
1988                                         eth_dev->data->nb_tx_queues;
1989                 }
1990         }
1991
1992
1993         /* Reconfigure each slave device if starting bonded device */
1994         for (i = 0; i < internals->slave_count; i++) {
1995                 struct rte_eth_dev *slave_ethdev =
1996                                 &(rte_eth_devices[internals->slaves[i].port_id]);
1997                 if (slave_configure(eth_dev, slave_ethdev) != 0) {
1998                         RTE_BOND_LOG(ERR,
1999                                 "bonded port (%d) failed to reconfigure slave device (%d)",
2000                                 eth_dev->data->port_id,
2001                                 internals->slaves[i].port_id);
2002                         goto out_err;
2003                 }
2004                 /* We will need to poll for link status if any slave doesn't
2005                  * support interrupts
2006                  */
2007                 if (internals->slaves[i].link_status_poll_enabled)
2008                         internals->link_status_polling_enabled = 1;
2009         }
2010
2011         /* start polling if needed */
2012         if (internals->link_status_polling_enabled) {
2013                 rte_eal_alarm_set(
2014                         internals->link_status_polling_interval_ms * 1000,
2015                         bond_ethdev_slave_link_status_change_monitor,
2016                         (void *)&rte_eth_devices[internals->port_id]);
2017         }
2018
2019         /* Update all slave devices MACs*/
2020         if (mac_address_slaves_update(eth_dev) != 0)
2021                 goto out_err;
2022
2023         if (internals->user_defined_primary_port)
2024                 bond_ethdev_primary_set(internals, internals->primary_port);
2025
2026         if (internals->mode == BONDING_MODE_8023AD)
2027                 bond_mode_8023ad_start(eth_dev);
2028
2029         if (internals->mode == BONDING_MODE_TLB ||
2030                         internals->mode == BONDING_MODE_ALB)
2031                 bond_tlb_enable(internals);
2032
2033         return 0;
2034
2035 out_err:
2036         eth_dev->data->dev_started = 0;
2037         return -1;
2038 }
2039
2040 static void
2041 bond_ethdev_free_queues(struct rte_eth_dev *dev)
2042 {
2043         uint16_t i;
2044
2045         if (dev->data->rx_queues != NULL) {
2046                 for (i = 0; i < dev->data->nb_rx_queues; i++) {
2047                         rte_free(dev->data->rx_queues[i]);
2048                         dev->data->rx_queues[i] = NULL;
2049                 }
2050                 dev->data->nb_rx_queues = 0;
2051         }
2052
2053         if (dev->data->tx_queues != NULL) {
2054                 for (i = 0; i < dev->data->nb_tx_queues; i++) {
2055                         rte_free(dev->data->tx_queues[i]);
2056                         dev->data->tx_queues[i] = NULL;
2057                 }
2058                 dev->data->nb_tx_queues = 0;
2059         }
2060 }
2061
2062 int
2063 bond_ethdev_stop(struct rte_eth_dev *eth_dev)
2064 {
2065         struct bond_dev_private *internals = eth_dev->data->dev_private;
2066         uint16_t i;
2067         int ret;
2068
2069         if (internals->mode == BONDING_MODE_8023AD) {
2070                 struct port *port;
2071                 void *pkt = NULL;
2072
2073                 bond_mode_8023ad_stop(eth_dev);
2074
2075                 /* Discard all messages to/from mode 4 state machines */
2076                 for (i = 0; i < internals->active_slave_count; i++) {
2077                         port = &bond_mode_8023ad_ports[internals->active_slaves[i]];
2078
2079                         RTE_ASSERT(port->rx_ring != NULL);
2080                         while (rte_ring_dequeue(port->rx_ring, &pkt) != -ENOENT)
2081                                 rte_pktmbuf_free(pkt);
2082
2083                         RTE_ASSERT(port->tx_ring != NULL);
2084                         while (rte_ring_dequeue(port->tx_ring, &pkt) != -ENOENT)
2085                                 rte_pktmbuf_free(pkt);
2086                 }
2087         }
2088
2089         if (internals->mode == BONDING_MODE_TLB ||
2090                         internals->mode == BONDING_MODE_ALB) {
2091                 bond_tlb_disable(internals);
2092                 for (i = 0; i < internals->active_slave_count; i++)
2093                         tlb_last_obytets[internals->active_slaves[i]] = 0;
2094         }
2095
2096         eth_dev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
2097         eth_dev->data->dev_started = 0;
2098
2099         internals->link_status_polling_enabled = 0;
2100         for (i = 0; i < internals->slave_count; i++) {
2101                 uint16_t slave_id = internals->slaves[i].port_id;
2102                 if (find_slave_by_id(internals->active_slaves,
2103                                 internals->active_slave_count, slave_id) !=
2104                                                 internals->active_slave_count) {
2105                         internals->slaves[i].last_link_status = 0;
2106                         ret = rte_eth_dev_stop(slave_id);
2107                         if (ret != 0) {
2108                                 RTE_BOND_LOG(ERR, "Failed to stop device on port %u",
2109                                              slave_id);
2110                                 return ret;
2111                         }
2112                         deactivate_slave(eth_dev, slave_id);
2113                 }
2114         }
2115
2116         return 0;
2117 }
2118
2119 int
2120 bond_ethdev_close(struct rte_eth_dev *dev)
2121 {
2122         struct bond_dev_private *internals = dev->data->dev_private;
2123         uint16_t bond_port_id = internals->port_id;
2124         int skipped = 0;
2125         struct rte_flow_error ferror;
2126
2127         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2128                 return 0;
2129
2130         RTE_BOND_LOG(INFO, "Closing bonded device %s", dev->device->name);
2131         while (internals->slave_count != skipped) {
2132                 uint16_t port_id = internals->slaves[skipped].port_id;
2133
2134                 if (rte_eth_dev_stop(port_id) != 0) {
2135                         RTE_BOND_LOG(ERR, "Failed to stop device on port %u",
2136                                      port_id);
2137                         skipped++;
2138                 }
2139
2140                 if (rte_eth_bond_slave_remove(bond_port_id, port_id) != 0) {
2141                         RTE_BOND_LOG(ERR,
2142                                      "Failed to remove port %d from bonded device %s",
2143                                      port_id, dev->device->name);
2144                         skipped++;
2145                 }
2146         }
2147         bond_flow_ops.flush(dev, &ferror);
2148         bond_ethdev_free_queues(dev);
2149         rte_bitmap_reset(internals->vlan_filter_bmp);
2150         rte_bitmap_free(internals->vlan_filter_bmp);
2151         rte_free(internals->vlan_filter_bmpmem);
2152
2153         /* Try to release mempool used in mode6. If the bond
2154          * device is not mode6, free the NULL is not problem.
2155          */
2156         rte_mempool_free(internals->mode6.mempool);
2157
2158         rte_kvargs_free(internals->kvlist);
2159
2160         return 0;
2161 }
2162
2163 /* forward declaration */
2164 static int bond_ethdev_configure(struct rte_eth_dev *dev);
2165
2166 static int
2167 bond_ethdev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2168 {
2169         struct bond_dev_private *internals = dev->data->dev_private;
2170         struct bond_slave_details slave;
2171         int ret;
2172
2173         uint16_t max_nb_rx_queues = UINT16_MAX;
2174         uint16_t max_nb_tx_queues = UINT16_MAX;
2175         uint16_t max_rx_desc_lim = UINT16_MAX;
2176         uint16_t max_tx_desc_lim = UINT16_MAX;
2177
2178         dev_info->max_mac_addrs = BOND_MAX_MAC_ADDRS;
2179
2180         dev_info->max_rx_pktlen = internals->candidate_max_rx_pktlen ?
2181                         internals->candidate_max_rx_pktlen :
2182                         RTE_ETHER_MAX_JUMBO_FRAME_LEN;
2183
2184         /* Max number of tx/rx queues that the bonded device can support is the
2185          * minimum values of the bonded slaves, as all slaves must be capable
2186          * of supporting the same number of tx/rx queues.
2187          */
2188         if (internals->slave_count > 0) {
2189                 struct rte_eth_dev_info slave_info;
2190                 uint16_t idx;
2191
2192                 for (idx = 0; idx < internals->slave_count; idx++) {
2193                         slave = internals->slaves[idx];
2194                         ret = rte_eth_dev_info_get(slave.port_id, &slave_info);
2195                         if (ret != 0) {
2196                                 RTE_BOND_LOG(ERR,
2197                                         "%s: Error during getting device (port %u) info: %s\n",
2198                                         __func__,
2199                                         slave.port_id,
2200                                         strerror(-ret));
2201
2202                                 return ret;
2203                         }
2204
2205                         if (slave_info.max_rx_queues < max_nb_rx_queues)
2206                                 max_nb_rx_queues = slave_info.max_rx_queues;
2207
2208                         if (slave_info.max_tx_queues < max_nb_tx_queues)
2209                                 max_nb_tx_queues = slave_info.max_tx_queues;
2210
2211                         if (slave_info.rx_desc_lim.nb_max < max_rx_desc_lim)
2212                                 max_rx_desc_lim = slave_info.rx_desc_lim.nb_max;
2213
2214                         if (slave_info.tx_desc_lim.nb_max < max_tx_desc_lim)
2215                                 max_tx_desc_lim = slave_info.tx_desc_lim.nb_max;
2216                 }
2217         }
2218
2219         dev_info->max_rx_queues = max_nb_rx_queues;
2220         dev_info->max_tx_queues = max_nb_tx_queues;
2221
2222         memcpy(&dev_info->default_rxconf, &internals->default_rxconf,
2223                sizeof(dev_info->default_rxconf));
2224         memcpy(&dev_info->default_txconf, &internals->default_txconf,
2225                sizeof(dev_info->default_txconf));
2226
2227         dev_info->rx_desc_lim.nb_max = max_rx_desc_lim;
2228         dev_info->tx_desc_lim.nb_max = max_tx_desc_lim;
2229
2230         /**
2231          * If dedicated hw queues enabled for link bonding device in LACP mode
2232          * then we need to reduce the maximum number of data path queues by 1.
2233          */
2234         if (internals->mode == BONDING_MODE_8023AD &&
2235                 internals->mode4.dedicated_queues.enabled == 1) {
2236                 dev_info->max_rx_queues--;
2237                 dev_info->max_tx_queues--;
2238         }
2239
2240         dev_info->min_rx_bufsize = 0;
2241
2242         dev_info->rx_offload_capa = internals->rx_offload_capa;
2243         dev_info->tx_offload_capa = internals->tx_offload_capa;
2244         dev_info->rx_queue_offload_capa = internals->rx_queue_offload_capa;
2245         dev_info->tx_queue_offload_capa = internals->tx_queue_offload_capa;
2246         dev_info->flow_type_rss_offloads = internals->flow_type_rss_offloads;
2247
2248         dev_info->reta_size = internals->reta_size;
2249         dev_info->hash_key_size = internals->rss_key_len;
2250
2251         return 0;
2252 }
2253
2254 static int
2255 bond_ethdev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
2256 {
2257         int res;
2258         uint16_t i;
2259         struct bond_dev_private *internals = dev->data->dev_private;
2260
2261         /* don't do this while a slave is being added */
2262         rte_spinlock_lock(&internals->lock);
2263
2264         if (on)
2265                 rte_bitmap_set(internals->vlan_filter_bmp, vlan_id);
2266         else
2267                 rte_bitmap_clear(internals->vlan_filter_bmp, vlan_id);
2268
2269         for (i = 0; i < internals->slave_count; i++) {
2270                 uint16_t port_id = internals->slaves[i].port_id;
2271
2272                 res = rte_eth_dev_vlan_filter(port_id, vlan_id, on);
2273                 if (res == ENOTSUP)
2274                         RTE_BOND_LOG(WARNING,
2275                                      "Setting VLAN filter on slave port %u not supported.",
2276                                      port_id);
2277         }
2278
2279         rte_spinlock_unlock(&internals->lock);
2280         return 0;
2281 }
2282
2283 static int
2284 bond_ethdev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
2285                 uint16_t nb_rx_desc, unsigned int socket_id __rte_unused,
2286                 const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool)
2287 {
2288         struct bond_rx_queue *bd_rx_q = (struct bond_rx_queue *)
2289                         rte_zmalloc_socket(NULL, sizeof(struct bond_rx_queue),
2290                                         0, dev->data->numa_node);
2291         if (bd_rx_q == NULL)
2292                 return -1;
2293
2294         bd_rx_q->queue_id = rx_queue_id;
2295         bd_rx_q->dev_private = dev->data->dev_private;
2296
2297         bd_rx_q->nb_rx_desc = nb_rx_desc;
2298
2299         memcpy(&(bd_rx_q->rx_conf), rx_conf, sizeof(struct rte_eth_rxconf));
2300         bd_rx_q->mb_pool = mb_pool;
2301
2302         dev->data->rx_queues[rx_queue_id] = bd_rx_q;
2303
2304         return 0;
2305 }
2306
2307 static int
2308 bond_ethdev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
2309                 uint16_t nb_tx_desc, unsigned int socket_id __rte_unused,
2310                 const struct rte_eth_txconf *tx_conf)
2311 {
2312         struct bond_tx_queue *bd_tx_q  = (struct bond_tx_queue *)
2313                         rte_zmalloc_socket(NULL, sizeof(struct bond_tx_queue),
2314                                         0, dev->data->numa_node);
2315
2316         if (bd_tx_q == NULL)
2317                 return -1;
2318
2319         bd_tx_q->queue_id = tx_queue_id;
2320         bd_tx_q->dev_private = dev->data->dev_private;
2321
2322         bd_tx_q->nb_tx_desc = nb_tx_desc;
2323         memcpy(&(bd_tx_q->tx_conf), tx_conf, sizeof(bd_tx_q->tx_conf));
2324
2325         dev->data->tx_queues[tx_queue_id] = bd_tx_q;
2326
2327         return 0;
2328 }
2329
2330 static void
2331 bond_ethdev_rx_queue_release(struct rte_eth_dev *dev, uint16_t queue_id)
2332 {
2333         void *queue = dev->data->rx_queues[queue_id];
2334
2335         if (queue == NULL)
2336                 return;
2337
2338         rte_free(queue);
2339 }
2340
2341 static void
2342 bond_ethdev_tx_queue_release(struct rte_eth_dev *dev, uint16_t queue_id)
2343 {
2344         void *queue = dev->data->tx_queues[queue_id];
2345
2346         if (queue == NULL)
2347                 return;
2348
2349         rte_free(queue);
2350 }
2351
2352 static void
2353 bond_ethdev_slave_link_status_change_monitor(void *cb_arg)
2354 {
2355         struct rte_eth_dev *bonded_ethdev, *slave_ethdev;
2356         struct bond_dev_private *internals;
2357
2358         /* Default value for polling slave found is true as we don't want to
2359          * disable the polling thread if we cannot get the lock */
2360         int i, polling_slave_found = 1;
2361
2362         if (cb_arg == NULL)
2363                 return;
2364
2365         bonded_ethdev = cb_arg;
2366         internals = bonded_ethdev->data->dev_private;
2367
2368         if (!bonded_ethdev->data->dev_started ||
2369                 !internals->link_status_polling_enabled)
2370                 return;
2371
2372         /* If device is currently being configured then don't check slaves link
2373          * status, wait until next period */
2374         if (rte_spinlock_trylock(&internals->lock)) {
2375                 if (internals->slave_count > 0)
2376                         polling_slave_found = 0;
2377
2378                 for (i = 0; i < internals->slave_count; i++) {
2379                         if (!internals->slaves[i].link_status_poll_enabled)
2380                                 continue;
2381
2382                         slave_ethdev = &rte_eth_devices[internals->slaves[i].port_id];
2383                         polling_slave_found = 1;
2384
2385                         /* Update slave link status */
2386                         (*slave_ethdev->dev_ops->link_update)(slave_ethdev,
2387                                         internals->slaves[i].link_status_wait_to_complete);
2388
2389                         /* if link status has changed since last checked then call lsc
2390                          * event callback */
2391                         if (slave_ethdev->data->dev_link.link_status !=
2392                                         internals->slaves[i].last_link_status) {
2393                                 internals->slaves[i].last_link_status =
2394                                                 slave_ethdev->data->dev_link.link_status;
2395
2396                                 bond_ethdev_lsc_event_callback(internals->slaves[i].port_id,
2397                                                 RTE_ETH_EVENT_INTR_LSC,
2398                                                 &bonded_ethdev->data->port_id,
2399                                                 NULL);
2400                         }
2401                 }
2402                 rte_spinlock_unlock(&internals->lock);
2403         }
2404
2405         if (polling_slave_found)
2406                 /* Set alarm to continue monitoring link status of slave ethdev's */
2407                 rte_eal_alarm_set(internals->link_status_polling_interval_ms * 1000,
2408                                 bond_ethdev_slave_link_status_change_monitor, cb_arg);
2409 }
2410
2411 static int
2412 bond_ethdev_link_update(struct rte_eth_dev *ethdev, int wait_to_complete)
2413 {
2414         int (*link_update)(uint16_t port_id, struct rte_eth_link *eth_link);
2415
2416         struct bond_dev_private *bond_ctx;
2417         struct rte_eth_link slave_link;
2418
2419         bool one_link_update_succeeded;
2420         uint32_t idx;
2421         int ret;
2422
2423         bond_ctx = ethdev->data->dev_private;
2424
2425         ethdev->data->dev_link.link_speed = RTE_ETH_SPEED_NUM_NONE;
2426
2427         if (ethdev->data->dev_started == 0 ||
2428                         bond_ctx->active_slave_count == 0) {
2429                 ethdev->data->dev_link.link_status = RTE_ETH_LINK_DOWN;
2430                 return 0;
2431         }
2432
2433         ethdev->data->dev_link.link_status = RTE_ETH_LINK_UP;
2434
2435         if (wait_to_complete)
2436                 link_update = rte_eth_link_get;
2437         else
2438                 link_update = rte_eth_link_get_nowait;
2439
2440         switch (bond_ctx->mode) {
2441         case BONDING_MODE_BROADCAST:
2442                 /**
2443                  * Setting link speed to UINT32_MAX to ensure we pick up the
2444                  * value of the first active slave
2445                  */
2446                 ethdev->data->dev_link.link_speed = UINT32_MAX;
2447
2448                 /**
2449                  * link speed is minimum value of all the slaves link speed as
2450                  * packet loss will occur on this slave if transmission at rates
2451                  * greater than this are attempted
2452                  */
2453                 for (idx = 0; idx < bond_ctx->active_slave_count; idx++) {
2454                         ret = link_update(bond_ctx->active_slaves[idx],
2455                                           &slave_link);
2456                         if (ret < 0) {
2457                                 ethdev->data->dev_link.link_speed =
2458                                         RTE_ETH_SPEED_NUM_NONE;
2459                                 RTE_BOND_LOG(ERR,
2460                                         "Slave (port %u) link get failed: %s",
2461                                         bond_ctx->active_slaves[idx],
2462                                         rte_strerror(-ret));
2463                                 return 0;
2464                         }
2465
2466                         if (slave_link.link_speed <
2467                                         ethdev->data->dev_link.link_speed)
2468                                 ethdev->data->dev_link.link_speed =
2469                                                 slave_link.link_speed;
2470                 }
2471                 break;
2472         case BONDING_MODE_ACTIVE_BACKUP:
2473                 /* Current primary slave */
2474                 ret = link_update(bond_ctx->current_primary_port, &slave_link);
2475                 if (ret < 0) {
2476                         RTE_BOND_LOG(ERR, "Slave (port %u) link get failed: %s",
2477                                 bond_ctx->current_primary_port,
2478                                 rte_strerror(-ret));
2479                         return 0;
2480                 }
2481
2482                 ethdev->data->dev_link.link_speed = slave_link.link_speed;
2483                 break;
2484         case BONDING_MODE_8023AD:
2485                 ethdev->data->dev_link.link_autoneg =
2486                                 bond_ctx->mode4.slave_link.link_autoneg;
2487                 ethdev->data->dev_link.link_duplex =
2488                                 bond_ctx->mode4.slave_link.link_duplex;
2489                 /* fall through */
2490                 /* to update link speed */
2491         case BONDING_MODE_ROUND_ROBIN:
2492         case BONDING_MODE_BALANCE:
2493         case BONDING_MODE_TLB:
2494         case BONDING_MODE_ALB:
2495         default:
2496                 /**
2497                  * In theses mode the maximum theoretical link speed is the sum
2498                  * of all the slaves
2499                  */
2500                 ethdev->data->dev_link.link_speed = RTE_ETH_SPEED_NUM_NONE;
2501                 one_link_update_succeeded = false;
2502
2503                 for (idx = 0; idx < bond_ctx->active_slave_count; idx++) {
2504                         ret = link_update(bond_ctx->active_slaves[idx],
2505                                         &slave_link);
2506                         if (ret < 0) {
2507                                 RTE_BOND_LOG(ERR,
2508                                         "Slave (port %u) link get failed: %s",
2509                                         bond_ctx->active_slaves[idx],
2510                                         rte_strerror(-ret));
2511                                 continue;
2512                         }
2513
2514                         one_link_update_succeeded = true;
2515                         ethdev->data->dev_link.link_speed +=
2516                                         slave_link.link_speed;
2517                 }
2518
2519                 if (!one_link_update_succeeded) {
2520                         RTE_BOND_LOG(ERR, "All slaves link get failed");
2521                         return 0;
2522                 }
2523         }
2524
2525
2526         return 0;
2527 }
2528
2529
2530 static int
2531 bond_ethdev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2532 {
2533         struct bond_dev_private *internals = dev->data->dev_private;
2534         struct rte_eth_stats slave_stats;
2535         int i, j;
2536
2537         for (i = 0; i < internals->slave_count; i++) {
2538                 rte_eth_stats_get(internals->slaves[i].port_id, &slave_stats);
2539
2540                 stats->ipackets += slave_stats.ipackets;
2541                 stats->opackets += slave_stats.opackets;
2542                 stats->ibytes += slave_stats.ibytes;
2543                 stats->obytes += slave_stats.obytes;
2544                 stats->imissed += slave_stats.imissed;
2545                 stats->ierrors += slave_stats.ierrors;
2546                 stats->oerrors += slave_stats.oerrors;
2547                 stats->rx_nombuf += slave_stats.rx_nombuf;
2548
2549                 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) {
2550                         stats->q_ipackets[j] += slave_stats.q_ipackets[j];
2551                         stats->q_opackets[j] += slave_stats.q_opackets[j];
2552                         stats->q_ibytes[j] += slave_stats.q_ibytes[j];
2553                         stats->q_obytes[j] += slave_stats.q_obytes[j];
2554                         stats->q_errors[j] += slave_stats.q_errors[j];
2555                 }
2556
2557         }
2558
2559         return 0;
2560 }
2561
2562 static int
2563 bond_ethdev_stats_reset(struct rte_eth_dev *dev)
2564 {
2565         struct bond_dev_private *internals = dev->data->dev_private;
2566         int i;
2567         int err;
2568         int ret;
2569
2570         for (i = 0, err = 0; i < internals->slave_count; i++) {
2571                 ret = rte_eth_stats_reset(internals->slaves[i].port_id);
2572                 if (ret != 0)
2573                         err = ret;
2574         }
2575
2576         return err;
2577 }
2578
2579 static int
2580 bond_ethdev_promiscuous_enable(struct rte_eth_dev *eth_dev)
2581 {
2582         struct bond_dev_private *internals = eth_dev->data->dev_private;
2583         int i;
2584         int ret = 0;
2585         uint16_t port_id;
2586
2587         switch (internals->mode) {
2588         /* Promiscuous mode is propagated to all slaves */
2589         case BONDING_MODE_ROUND_ROBIN:
2590         case BONDING_MODE_BALANCE:
2591         case BONDING_MODE_BROADCAST:
2592         case BONDING_MODE_8023AD: {
2593                 unsigned int slave_ok = 0;
2594
2595                 for (i = 0; i < internals->slave_count; i++) {
2596                         port_id = internals->slaves[i].port_id;
2597
2598                         ret = rte_eth_promiscuous_enable(port_id);
2599                         if (ret != 0)
2600                                 RTE_BOND_LOG(ERR,
2601                                         "Failed to enable promiscuous mode for port %u: %s",
2602                                         port_id, rte_strerror(-ret));
2603                         else
2604                                 slave_ok++;
2605                 }
2606                 /*
2607                  * Report success if operation is successful on at least
2608                  * on one slave. Otherwise return last error code.
2609                  */
2610                 if (slave_ok > 0)
2611                         ret = 0;
2612                 break;
2613         }
2614         /* Promiscuous mode is propagated only to primary slave */
2615         case BONDING_MODE_ACTIVE_BACKUP:
2616         case BONDING_MODE_TLB:
2617         case BONDING_MODE_ALB:
2618         default:
2619                 /* Do not touch promisc when there cannot be primary ports */
2620                 if (internals->slave_count == 0)
2621                         break;
2622                 port_id = internals->current_primary_port;
2623                 ret = rte_eth_promiscuous_enable(port_id);
2624                 if (ret != 0)
2625                         RTE_BOND_LOG(ERR,
2626                                 "Failed to enable promiscuous mode for port %u: %s",
2627                                 port_id, rte_strerror(-ret));
2628         }
2629
2630         return ret;
2631 }
2632
2633 static int
2634 bond_ethdev_promiscuous_disable(struct rte_eth_dev *dev)
2635 {
2636         struct bond_dev_private *internals = dev->data->dev_private;
2637         int i;
2638         int ret = 0;
2639         uint16_t port_id;
2640
2641         switch (internals->mode) {
2642         /* Promiscuous mode is propagated to all slaves */
2643         case BONDING_MODE_ROUND_ROBIN:
2644         case BONDING_MODE_BALANCE:
2645         case BONDING_MODE_BROADCAST:
2646         case BONDING_MODE_8023AD: {
2647                 unsigned int slave_ok = 0;
2648
2649                 for (i = 0; i < internals->slave_count; i++) {
2650                         port_id = internals->slaves[i].port_id;
2651
2652                         if (internals->mode == BONDING_MODE_8023AD &&
2653                             bond_mode_8023ad_ports[port_id].forced_rx_flags ==
2654                                         BOND_8023AD_FORCED_PROMISC) {
2655                                 slave_ok++;
2656                                 continue;
2657                         }
2658                         ret = rte_eth_promiscuous_disable(port_id);
2659                         if (ret != 0)
2660                                 RTE_BOND_LOG(ERR,
2661                                         "Failed to disable promiscuous mode for port %u: %s",
2662                                         port_id, rte_strerror(-ret));
2663                         else
2664                                 slave_ok++;
2665                 }
2666                 /*
2667                  * Report success if operation is successful on at least
2668                  * on one slave. Otherwise return last error code.
2669                  */
2670                 if (slave_ok > 0)
2671                         ret = 0;
2672                 break;
2673         }
2674         /* Promiscuous mode is propagated only to primary slave */
2675         case BONDING_MODE_ACTIVE_BACKUP:
2676         case BONDING_MODE_TLB:
2677         case BONDING_MODE_ALB:
2678         default:
2679                 /* Do not touch promisc when there cannot be primary ports */
2680                 if (internals->slave_count == 0)
2681                         break;
2682                 port_id = internals->current_primary_port;
2683                 ret = rte_eth_promiscuous_disable(port_id);
2684                 if (ret != 0)
2685                         RTE_BOND_LOG(ERR,
2686                                 "Failed to disable promiscuous mode for port %u: %s",
2687                                 port_id, rte_strerror(-ret));
2688         }
2689
2690         return ret;
2691 }
2692
2693 static int
2694 bond_ethdev_promiscuous_update(struct rte_eth_dev *dev)
2695 {
2696         struct bond_dev_private *internals = dev->data->dev_private;
2697         uint16_t port_id = internals->current_primary_port;
2698
2699         switch (internals->mode) {
2700         case BONDING_MODE_ROUND_ROBIN:
2701         case BONDING_MODE_BALANCE:
2702         case BONDING_MODE_BROADCAST:
2703         case BONDING_MODE_8023AD:
2704                 /* As promiscuous mode is propagated to all slaves for these
2705                  * mode, no need to update for bonding device.
2706                  */
2707                 break;
2708         case BONDING_MODE_ACTIVE_BACKUP:
2709         case BONDING_MODE_TLB:
2710         case BONDING_MODE_ALB:
2711         default:
2712                 /* As promiscuous mode is propagated only to primary slave
2713                  * for these mode. When active/standby switchover, promiscuous
2714                  * mode should be set to new primary slave according to bonding
2715                  * device.
2716                  */
2717                 if (rte_eth_promiscuous_get(internals->port_id) == 1)
2718                         rte_eth_promiscuous_enable(port_id);
2719                 else
2720                         rte_eth_promiscuous_disable(port_id);
2721         }
2722
2723         return 0;
2724 }
2725
2726 static int
2727 bond_ethdev_allmulticast_enable(struct rte_eth_dev *eth_dev)
2728 {
2729         struct bond_dev_private *internals = eth_dev->data->dev_private;
2730         int i;
2731         int ret = 0;
2732         uint16_t port_id;
2733
2734         switch (internals->mode) {
2735         /* allmulti mode is propagated to all slaves */
2736         case BONDING_MODE_ROUND_ROBIN:
2737         case BONDING_MODE_BALANCE:
2738         case BONDING_MODE_BROADCAST:
2739         case BONDING_MODE_8023AD: {
2740                 unsigned int slave_ok = 0;
2741
2742                 for (i = 0; i < internals->slave_count; i++) {
2743                         port_id = internals->slaves[i].port_id;
2744
2745                         ret = rte_eth_allmulticast_enable(port_id);
2746                         if (ret != 0)
2747                                 RTE_BOND_LOG(ERR,
2748                                         "Failed to enable allmulti mode for port %u: %s",
2749                                         port_id, rte_strerror(-ret));
2750                         else
2751                                 slave_ok++;
2752                 }
2753                 /*
2754                  * Report success if operation is successful on at least
2755                  * on one slave. Otherwise return last error code.
2756                  */
2757                 if (slave_ok > 0)
2758                         ret = 0;
2759                 break;
2760         }
2761         /* allmulti mode is propagated only to primary slave */
2762         case BONDING_MODE_ACTIVE_BACKUP:
2763         case BONDING_MODE_TLB:
2764         case BONDING_MODE_ALB:
2765         default:
2766                 /* Do not touch allmulti when there cannot be primary ports */
2767                 if (internals->slave_count == 0)
2768                         break;
2769                 port_id = internals->current_primary_port;
2770                 ret = rte_eth_allmulticast_enable(port_id);
2771                 if (ret != 0)
2772                         RTE_BOND_LOG(ERR,
2773                                 "Failed to enable allmulti mode for port %u: %s",
2774                                 port_id, rte_strerror(-ret));
2775         }
2776
2777         return ret;
2778 }
2779
2780 static int
2781 bond_ethdev_allmulticast_disable(struct rte_eth_dev *eth_dev)
2782 {
2783         struct bond_dev_private *internals = eth_dev->data->dev_private;
2784         int i;
2785         int ret = 0;
2786         uint16_t port_id;
2787
2788         switch (internals->mode) {
2789         /* allmulti mode is propagated to all slaves */
2790         case BONDING_MODE_ROUND_ROBIN:
2791         case BONDING_MODE_BALANCE:
2792         case BONDING_MODE_BROADCAST:
2793         case BONDING_MODE_8023AD: {
2794                 unsigned int slave_ok = 0;
2795
2796                 for (i = 0; i < internals->slave_count; i++) {
2797                         uint16_t port_id = internals->slaves[i].port_id;
2798
2799                         if (internals->mode == BONDING_MODE_8023AD &&
2800                             bond_mode_8023ad_ports[port_id].forced_rx_flags ==
2801                                         BOND_8023AD_FORCED_ALLMULTI)
2802                                 continue;
2803
2804                         ret = rte_eth_allmulticast_disable(port_id);
2805                         if (ret != 0)
2806                                 RTE_BOND_LOG(ERR,
2807                                         "Failed to disable allmulti mode for port %u: %s",
2808                                         port_id, rte_strerror(-ret));
2809                         else
2810                                 slave_ok++;
2811                 }
2812                 /*
2813                  * Report success if operation is successful on at least
2814                  * on one slave. Otherwise return last error code.
2815                  */
2816                 if (slave_ok > 0)
2817                         ret = 0;
2818                 break;
2819         }
2820         /* allmulti mode is propagated only to primary slave */
2821         case BONDING_MODE_ACTIVE_BACKUP:
2822         case BONDING_MODE_TLB:
2823         case BONDING_MODE_ALB:
2824         default:
2825                 /* Do not touch allmulti when there cannot be primary ports */
2826                 if (internals->slave_count == 0)
2827                         break;
2828                 port_id = internals->current_primary_port;
2829                 ret = rte_eth_allmulticast_disable(port_id);
2830                 if (ret != 0)
2831                         RTE_BOND_LOG(ERR,
2832                                 "Failed to disable allmulti mode for port %u: %s",
2833                                 port_id, rte_strerror(-ret));
2834         }
2835
2836         return ret;
2837 }
2838
2839 static int
2840 bond_ethdev_allmulticast_update(struct rte_eth_dev *dev)
2841 {
2842         struct bond_dev_private *internals = dev->data->dev_private;
2843         uint16_t port_id = internals->current_primary_port;
2844
2845         switch (internals->mode) {
2846         case BONDING_MODE_ROUND_ROBIN:
2847         case BONDING_MODE_BALANCE:
2848         case BONDING_MODE_BROADCAST:
2849         case BONDING_MODE_8023AD:
2850                 /* As allmulticast mode is propagated to all slaves for these
2851                  * mode, no need to update for bonding device.
2852                  */
2853                 break;
2854         case BONDING_MODE_ACTIVE_BACKUP:
2855         case BONDING_MODE_TLB:
2856         case BONDING_MODE_ALB:
2857         default:
2858                 /* As allmulticast mode is propagated only to primary slave
2859                  * for these mode. When active/standby switchover, allmulticast
2860                  * mode should be set to new primary slave according to bonding
2861                  * device.
2862                  */
2863                 if (rte_eth_allmulticast_get(internals->port_id) == 1)
2864                         rte_eth_allmulticast_enable(port_id);
2865                 else
2866                         rte_eth_allmulticast_disable(port_id);
2867         }
2868
2869         return 0;
2870 }
2871
2872 static void
2873 bond_ethdev_delayed_lsc_propagation(void *arg)
2874 {
2875         if (arg == NULL)
2876                 return;
2877
2878         rte_eth_dev_callback_process((struct rte_eth_dev *)arg,
2879                         RTE_ETH_EVENT_INTR_LSC, NULL);
2880 }
2881
2882 int
2883 bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type,
2884                 void *param, void *ret_param __rte_unused)
2885 {
2886         struct rte_eth_dev *bonded_eth_dev;
2887         struct bond_dev_private *internals;
2888         struct rte_eth_link link;
2889         int rc = -1;
2890         int ret;
2891
2892         uint8_t lsc_flag = 0;
2893         int valid_slave = 0;
2894         uint16_t active_pos;
2895         uint16_t i;
2896
2897         if (type != RTE_ETH_EVENT_INTR_LSC || param == NULL)
2898                 return rc;
2899
2900         bonded_eth_dev = &rte_eth_devices[*(uint16_t *)param];
2901
2902         if (check_for_bonded_ethdev(bonded_eth_dev))
2903                 return rc;
2904
2905         internals = bonded_eth_dev->data->dev_private;
2906
2907         /* If the device isn't started don't handle interrupts */
2908         if (!bonded_eth_dev->data->dev_started)
2909                 return rc;
2910
2911         /* verify that port_id is a valid slave of bonded port */
2912         for (i = 0; i < internals->slave_count; i++) {
2913                 if (internals->slaves[i].port_id == port_id) {
2914                         valid_slave = 1;
2915                         break;
2916                 }
2917         }
2918
2919         if (!valid_slave)
2920                 return rc;
2921
2922         /* Synchronize lsc callback parallel calls either by real link event
2923          * from the slaves PMDs or by the bonding PMD itself.
2924          */
2925         rte_spinlock_lock(&internals->lsc_lock);
2926
2927         /* Search for port in active port list */
2928         active_pos = find_slave_by_id(internals->active_slaves,
2929                         internals->active_slave_count, port_id);
2930
2931         ret = rte_eth_link_get_nowait(port_id, &link);
2932         if (ret < 0)
2933                 RTE_BOND_LOG(ERR, "Slave (port %u) link get failed", port_id);
2934
2935         if (ret == 0 && link.link_status) {
2936                 if (active_pos < internals->active_slave_count)
2937                         goto link_update;
2938
2939                 /* check link state properties if bonded link is up*/
2940                 if (bonded_eth_dev->data->dev_link.link_status == RTE_ETH_LINK_UP) {
2941                         if (link_properties_valid(bonded_eth_dev, &link) != 0)
2942                                 RTE_BOND_LOG(ERR, "Invalid link properties "
2943                                              "for slave %d in bonding mode %d",
2944                                              port_id, internals->mode);
2945                 } else {
2946                         /* inherit slave link properties */
2947                         link_properties_set(bonded_eth_dev, &link);
2948                 }
2949
2950                 /* If no active slave ports then set this port to be
2951                  * the primary port.
2952                  */
2953                 if (internals->active_slave_count < 1) {
2954                         /* If first active slave, then change link status */
2955                         bonded_eth_dev->data->dev_link.link_status =
2956                                                                 RTE_ETH_LINK_UP;
2957                         internals->current_primary_port = port_id;
2958                         lsc_flag = 1;
2959
2960                         mac_address_slaves_update(bonded_eth_dev);
2961                         bond_ethdev_promiscuous_update(bonded_eth_dev);
2962                         bond_ethdev_allmulticast_update(bonded_eth_dev);
2963                 }
2964
2965                 activate_slave(bonded_eth_dev, port_id);
2966
2967                 /* If the user has defined the primary port then default to
2968                  * using it.
2969                  */
2970                 if (internals->user_defined_primary_port &&
2971                                 internals->primary_port == port_id)
2972                         bond_ethdev_primary_set(internals, port_id);
2973         } else {
2974                 if (active_pos == internals->active_slave_count)
2975                         goto link_update;
2976
2977                 /* Remove from active slave list */
2978                 deactivate_slave(bonded_eth_dev, port_id);
2979
2980                 if (internals->active_slave_count < 1)
2981                         lsc_flag = 1;
2982
2983                 /* Update primary id, take first active slave from list or if none
2984                  * available set to -1 */
2985                 if (port_id == internals->current_primary_port) {
2986                         if (internals->active_slave_count > 0)
2987                                 bond_ethdev_primary_set(internals,
2988                                                 internals->active_slaves[0]);
2989                         else
2990                                 internals->current_primary_port = internals->primary_port;
2991                         mac_address_slaves_update(bonded_eth_dev);
2992                         bond_ethdev_promiscuous_update(bonded_eth_dev);
2993                         bond_ethdev_allmulticast_update(bonded_eth_dev);
2994                 }
2995         }
2996
2997 link_update:
2998         /**
2999          * Update bonded device link properties after any change to active
3000          * slaves
3001          */
3002         bond_ethdev_link_update(bonded_eth_dev, 0);
3003
3004         if (lsc_flag) {
3005                 /* Cancel any possible outstanding interrupts if delays are enabled */
3006                 if (internals->link_up_delay_ms > 0 ||
3007                         internals->link_down_delay_ms > 0)
3008                         rte_eal_alarm_cancel(bond_ethdev_delayed_lsc_propagation,
3009                                         bonded_eth_dev);
3010
3011                 if (bonded_eth_dev->data->dev_link.link_status) {
3012                         if (internals->link_up_delay_ms > 0)
3013                                 rte_eal_alarm_set(internals->link_up_delay_ms * 1000,
3014                                                 bond_ethdev_delayed_lsc_propagation,
3015                                                 (void *)bonded_eth_dev);
3016                         else
3017                                 rte_eth_dev_callback_process(bonded_eth_dev,
3018                                                 RTE_ETH_EVENT_INTR_LSC,
3019                                                 NULL);
3020
3021                 } else {
3022                         if (internals->link_down_delay_ms > 0)
3023                                 rte_eal_alarm_set(internals->link_down_delay_ms * 1000,
3024                                                 bond_ethdev_delayed_lsc_propagation,
3025                                                 (void *)bonded_eth_dev);
3026                         else
3027                                 rte_eth_dev_callback_process(bonded_eth_dev,
3028                                                 RTE_ETH_EVENT_INTR_LSC,
3029                                                 NULL);
3030                 }
3031         }
3032
3033         rte_spinlock_unlock(&internals->lsc_lock);
3034
3035         return rc;
3036 }
3037
3038 static int
3039 bond_ethdev_rss_reta_update(struct rte_eth_dev *dev,
3040                 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size)
3041 {
3042         unsigned i, j;
3043         int result = 0;
3044         int slave_reta_size;
3045         unsigned reta_count;
3046         struct bond_dev_private *internals = dev->data->dev_private;
3047
3048         if (reta_size != internals->reta_size)
3049                 return -EINVAL;
3050
3051          /* Copy RETA table */
3052         reta_count = (reta_size + RTE_ETH_RETA_GROUP_SIZE - 1) /
3053                         RTE_ETH_RETA_GROUP_SIZE;
3054
3055         for (i = 0; i < reta_count; i++) {
3056                 internals->reta_conf[i].mask = reta_conf[i].mask;
3057                 for (j = 0; j < RTE_ETH_RETA_GROUP_SIZE; j++)
3058                         if ((reta_conf[i].mask >> j) & 0x01)
3059                                 internals->reta_conf[i].reta[j] = reta_conf[i].reta[j];
3060         }
3061
3062         /* Fill rest of array */
3063         for (; i < RTE_DIM(internals->reta_conf); i += reta_count)
3064                 memcpy(&internals->reta_conf[i], &internals->reta_conf[0],
3065                                 sizeof(internals->reta_conf[0]) * reta_count);
3066
3067         /* Propagate RETA over slaves */
3068         for (i = 0; i < internals->slave_count; i++) {
3069                 slave_reta_size = internals->slaves[i].reta_size;
3070                 result = rte_eth_dev_rss_reta_update(internals->slaves[i].port_id,
3071                                 &internals->reta_conf[0], slave_reta_size);
3072                 if (result < 0)
3073                         return result;
3074         }
3075
3076         return 0;
3077 }
3078
3079 static int
3080 bond_ethdev_rss_reta_query(struct rte_eth_dev *dev,
3081                 struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size)
3082 {
3083         int i, j;
3084         struct bond_dev_private *internals = dev->data->dev_private;
3085
3086         if (reta_size != internals->reta_size)
3087                 return -EINVAL;
3088
3089          /* Copy RETA table */
3090         for (i = 0; i < reta_size / RTE_ETH_RETA_GROUP_SIZE; i++)
3091                 for (j = 0; j < RTE_ETH_RETA_GROUP_SIZE; j++)
3092                         if ((reta_conf[i].mask >> j) & 0x01)
3093                                 reta_conf[i].reta[j] = internals->reta_conf[i].reta[j];
3094
3095         return 0;
3096 }
3097
3098 static int
3099 bond_ethdev_rss_hash_update(struct rte_eth_dev *dev,
3100                 struct rte_eth_rss_conf *rss_conf)
3101 {
3102         int i, result = 0;
3103         struct bond_dev_private *internals = dev->data->dev_private;
3104         struct rte_eth_rss_conf bond_rss_conf;
3105
3106         memcpy(&bond_rss_conf, rss_conf, sizeof(struct rte_eth_rss_conf));
3107
3108         bond_rss_conf.rss_hf &= internals->flow_type_rss_offloads;
3109
3110         if (bond_rss_conf.rss_hf != 0)
3111                 dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf = bond_rss_conf.rss_hf;
3112
3113         if (bond_rss_conf.rss_key) {
3114                 if (bond_rss_conf.rss_key_len < internals->rss_key_len)
3115                         return -EINVAL;
3116                 else if (bond_rss_conf.rss_key_len > internals->rss_key_len)
3117                         RTE_BOND_LOG(WARNING, "rss_key will be truncated");
3118
3119                 memcpy(internals->rss_key, bond_rss_conf.rss_key,
3120                                 internals->rss_key_len);
3121                 bond_rss_conf.rss_key_len = internals->rss_key_len;
3122         }
3123
3124         for (i = 0; i < internals->slave_count; i++) {
3125                 result = rte_eth_dev_rss_hash_update(internals->slaves[i].port_id,
3126                                 &bond_rss_conf);
3127                 if (result < 0)
3128                         return result;
3129         }
3130
3131         return 0;
3132 }
3133
3134 static int
3135 bond_ethdev_rss_hash_conf_get(struct rte_eth_dev *dev,
3136                 struct rte_eth_rss_conf *rss_conf)
3137 {
3138         struct bond_dev_private *internals = dev->data->dev_private;
3139
3140         rss_conf->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
3141         rss_conf->rss_key_len = internals->rss_key_len;
3142         if (rss_conf->rss_key)
3143                 memcpy(rss_conf->rss_key, internals->rss_key, internals->rss_key_len);
3144
3145         return 0;
3146 }
3147
3148 static int
3149 bond_ethdev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
3150 {
3151         struct rte_eth_dev *slave_eth_dev;
3152         struct bond_dev_private *internals = dev->data->dev_private;
3153         int ret, i;
3154
3155         rte_spinlock_lock(&internals->lock);
3156
3157         for (i = 0; i < internals->slave_count; i++) {
3158                 slave_eth_dev = &rte_eth_devices[internals->slaves[i].port_id];
3159                 if (*slave_eth_dev->dev_ops->mtu_set == NULL) {
3160                         rte_spinlock_unlock(&internals->lock);
3161                         return -ENOTSUP;
3162                 }
3163         }
3164         for (i = 0; i < internals->slave_count; i++) {
3165                 ret = rte_eth_dev_set_mtu(internals->slaves[i].port_id, mtu);
3166                 if (ret < 0) {
3167                         rte_spinlock_unlock(&internals->lock);
3168                         return ret;
3169                 }
3170         }
3171
3172         rte_spinlock_unlock(&internals->lock);
3173         return 0;
3174 }
3175
3176 static int
3177 bond_ethdev_mac_address_set(struct rte_eth_dev *dev,
3178                         struct rte_ether_addr *addr)
3179 {
3180         if (mac_address_set(dev, addr)) {
3181                 RTE_BOND_LOG(ERR, "Failed to update MAC address");
3182                 return -EINVAL;
3183         }
3184
3185         return 0;
3186 }
3187
3188 static int
3189 bond_flow_ops_get(struct rte_eth_dev *dev __rte_unused,
3190                   const struct rte_flow_ops **ops)
3191 {
3192         *ops = &bond_flow_ops;
3193         return 0;
3194 }
3195
3196 static int
3197 bond_ethdev_mac_addr_add(struct rte_eth_dev *dev,
3198                         struct rte_ether_addr *mac_addr,
3199                         __rte_unused uint32_t index, uint32_t vmdq)
3200 {
3201         struct rte_eth_dev *slave_eth_dev;
3202         struct bond_dev_private *internals = dev->data->dev_private;
3203         int ret, i;
3204
3205         rte_spinlock_lock(&internals->lock);
3206
3207         for (i = 0; i < internals->slave_count; i++) {
3208                 slave_eth_dev = &rte_eth_devices[internals->slaves[i].port_id];
3209                 if (*slave_eth_dev->dev_ops->mac_addr_add == NULL ||
3210                          *slave_eth_dev->dev_ops->mac_addr_remove == NULL) {
3211                         ret = -ENOTSUP;
3212                         goto end;
3213                 }
3214         }
3215
3216         for (i = 0; i < internals->slave_count; i++) {
3217                 ret = rte_eth_dev_mac_addr_add(internals->slaves[i].port_id,
3218                                 mac_addr, vmdq);
3219                 if (ret < 0) {
3220                         /* rollback */
3221                         for (i--; i >= 0; i--)
3222                                 rte_eth_dev_mac_addr_remove(
3223                                         internals->slaves[i].port_id, mac_addr);
3224                         goto end;
3225                 }
3226         }
3227
3228         ret = 0;
3229 end:
3230         rte_spinlock_unlock(&internals->lock);
3231         return ret;
3232 }
3233
3234 static void
3235 bond_ethdev_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
3236 {
3237         struct rte_eth_dev *slave_eth_dev;
3238         struct bond_dev_private *internals = dev->data->dev_private;
3239         int i;
3240
3241         rte_spinlock_lock(&internals->lock);
3242
3243         for (i = 0; i < internals->slave_count; i++) {
3244                 slave_eth_dev = &rte_eth_devices[internals->slaves[i].port_id];
3245                 if (*slave_eth_dev->dev_ops->mac_addr_remove == NULL)
3246                         goto end;
3247         }
3248
3249         struct rte_ether_addr *mac_addr = &dev->data->mac_addrs[index];
3250
3251         for (i = 0; i < internals->slave_count; i++)
3252                 rte_eth_dev_mac_addr_remove(internals->slaves[i].port_id,
3253                                 mac_addr);
3254
3255 end:
3256         rte_spinlock_unlock(&internals->lock);
3257 }
3258
3259 const struct eth_dev_ops default_dev_ops = {
3260         .dev_start            = bond_ethdev_start,
3261         .dev_stop             = bond_ethdev_stop,
3262         .dev_close            = bond_ethdev_close,
3263         .dev_configure        = bond_ethdev_configure,
3264         .dev_infos_get        = bond_ethdev_info,
3265         .vlan_filter_set      = bond_ethdev_vlan_filter_set,
3266         .rx_queue_setup       = bond_ethdev_rx_queue_setup,
3267         .tx_queue_setup       = bond_ethdev_tx_queue_setup,
3268         .rx_queue_release     = bond_ethdev_rx_queue_release,
3269         .tx_queue_release     = bond_ethdev_tx_queue_release,
3270         .link_update          = bond_ethdev_link_update,
3271         .stats_get            = bond_ethdev_stats_get,
3272         .stats_reset          = bond_ethdev_stats_reset,
3273         .promiscuous_enable   = bond_ethdev_promiscuous_enable,
3274         .promiscuous_disable  = bond_ethdev_promiscuous_disable,
3275         .allmulticast_enable  = bond_ethdev_allmulticast_enable,
3276         .allmulticast_disable = bond_ethdev_allmulticast_disable,
3277         .reta_update          = bond_ethdev_rss_reta_update,
3278         .reta_query           = bond_ethdev_rss_reta_query,
3279         .rss_hash_update      = bond_ethdev_rss_hash_update,
3280         .rss_hash_conf_get    = bond_ethdev_rss_hash_conf_get,
3281         .mtu_set              = bond_ethdev_mtu_set,
3282         .mac_addr_set         = bond_ethdev_mac_address_set,
3283         .mac_addr_add         = bond_ethdev_mac_addr_add,
3284         .mac_addr_remove      = bond_ethdev_mac_addr_remove,
3285         .flow_ops_get         = bond_flow_ops_get
3286 };
3287
3288 static int
3289 bond_alloc(struct rte_vdev_device *dev, uint8_t mode)
3290 {
3291         const char *name = rte_vdev_device_name(dev);
3292         uint8_t socket_id = dev->device.numa_node;
3293         struct bond_dev_private *internals = NULL;
3294         struct rte_eth_dev *eth_dev = NULL;
3295         uint32_t vlan_filter_bmp_size;
3296
3297         /* now do all data allocation - for eth_dev structure, dummy pci driver
3298          * and internal (private) data
3299          */
3300
3301         /* reserve an ethdev entry */
3302         eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internals));
3303         if (eth_dev == NULL) {
3304                 RTE_BOND_LOG(ERR, "Unable to allocate rte_eth_dev");
3305                 goto err;
3306         }
3307
3308         internals = eth_dev->data->dev_private;
3309         eth_dev->data->nb_rx_queues = (uint16_t)1;
3310         eth_dev->data->nb_tx_queues = (uint16_t)1;
3311
3312         /* Allocate memory for storing MAC addresses */
3313         eth_dev->data->mac_addrs = rte_zmalloc_socket(name, RTE_ETHER_ADDR_LEN *
3314                         BOND_MAX_MAC_ADDRS, 0, socket_id);
3315         if (eth_dev->data->mac_addrs == NULL) {
3316                 RTE_BOND_LOG(ERR,
3317                              "Failed to allocate %u bytes needed to store MAC addresses",
3318                              RTE_ETHER_ADDR_LEN * BOND_MAX_MAC_ADDRS);
3319                 goto err;
3320         }
3321
3322         eth_dev->dev_ops = &default_dev_ops;
3323         eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC |
3324                                         RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
3325
3326         rte_spinlock_init(&internals->lock);
3327         rte_spinlock_init(&internals->lsc_lock);
3328
3329         internals->port_id = eth_dev->data->port_id;
3330         internals->mode = BONDING_MODE_INVALID;
3331         internals->current_primary_port = RTE_MAX_ETHPORTS + 1;
3332         internals->balance_xmit_policy = BALANCE_XMIT_POLICY_LAYER2;
3333         internals->burst_xmit_hash = burst_xmit_l2_hash;
3334         internals->user_defined_mac = 0;
3335
3336         internals->link_status_polling_enabled = 0;
3337
3338         internals->link_status_polling_interval_ms =
3339                 DEFAULT_POLLING_INTERVAL_10_MS;
3340         internals->link_down_delay_ms = 0;
3341         internals->link_up_delay_ms = 0;
3342
3343         internals->slave_count = 0;
3344         internals->active_slave_count = 0;
3345         internals->rx_offload_capa = 0;
3346         internals->tx_offload_capa = 0;
3347         internals->rx_queue_offload_capa = 0;
3348         internals->tx_queue_offload_capa = 0;
3349         internals->candidate_max_rx_pktlen = 0;
3350         internals->max_rx_pktlen = 0;
3351
3352         /* Initially allow to choose any offload type */
3353         internals->flow_type_rss_offloads = RTE_ETH_RSS_PROTO_MASK;
3354
3355         memset(&internals->default_rxconf, 0,
3356                sizeof(internals->default_rxconf));
3357         memset(&internals->default_txconf, 0,
3358                sizeof(internals->default_txconf));
3359
3360         memset(&internals->rx_desc_lim, 0, sizeof(internals->rx_desc_lim));
3361         memset(&internals->tx_desc_lim, 0, sizeof(internals->tx_desc_lim));
3362
3363         memset(internals->active_slaves, 0, sizeof(internals->active_slaves));
3364         memset(internals->slaves, 0, sizeof(internals->slaves));
3365
3366         TAILQ_INIT(&internals->flow_list);
3367         internals->flow_isolated_valid = 0;
3368
3369         /* Set mode 4 default configuration */
3370         bond_mode_8023ad_setup(eth_dev, NULL);
3371         if (bond_ethdev_mode_set(eth_dev, mode)) {
3372                 RTE_BOND_LOG(ERR, "Failed to set bonded device %u mode to %u",
3373                                  eth_dev->data->port_id, mode);
3374                 goto err;
3375         }
3376
3377         vlan_filter_bmp_size =
3378                 rte_bitmap_get_memory_footprint(RTE_ETHER_MAX_VLAN_ID + 1);
3379         internals->vlan_filter_bmpmem = rte_malloc(name, vlan_filter_bmp_size,
3380                                                    RTE_CACHE_LINE_SIZE);
3381         if (internals->vlan_filter_bmpmem == NULL) {
3382                 RTE_BOND_LOG(ERR,
3383                              "Failed to allocate vlan bitmap for bonded device %u",
3384                              eth_dev->data->port_id);
3385                 goto err;
3386         }
3387
3388         internals->vlan_filter_bmp = rte_bitmap_init(RTE_ETHER_MAX_VLAN_ID + 1,
3389                         internals->vlan_filter_bmpmem, vlan_filter_bmp_size);
3390         if (internals->vlan_filter_bmp == NULL) {
3391                 RTE_BOND_LOG(ERR,
3392                              "Failed to init vlan bitmap for bonded device %u",
3393                              eth_dev->data->port_id);
3394                 rte_free(internals->vlan_filter_bmpmem);
3395                 goto err;
3396         }
3397
3398         return eth_dev->data->port_id;
3399
3400 err:
3401         rte_free(internals);
3402         if (eth_dev != NULL)
3403                 eth_dev->data->dev_private = NULL;
3404         rte_eth_dev_release_port(eth_dev);
3405         return -1;
3406 }
3407
3408 static int
3409 bond_probe(struct rte_vdev_device *dev)
3410 {
3411         const char *name;
3412         struct bond_dev_private *internals;
3413         struct rte_kvargs *kvlist;
3414         uint8_t bonding_mode;
3415         int arg_count, port_id;
3416         int socket_id;
3417         uint8_t agg_mode;
3418         struct rte_eth_dev *eth_dev;
3419
3420         if (!dev)
3421                 return -EINVAL;
3422
3423         name = rte_vdev_device_name(dev);
3424         RTE_BOND_LOG(INFO, "Initializing pmd_bond for %s", name);
3425
3426         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
3427                 eth_dev = rte_eth_dev_attach_secondary(name);
3428                 if (!eth_dev) {
3429                         RTE_BOND_LOG(ERR, "Failed to probe %s", name);
3430                         return -1;
3431                 }
3432                 /* TODO: request info from primary to set up Rx and Tx */
3433                 eth_dev->dev_ops = &default_dev_ops;
3434                 eth_dev->device = &dev->device;
3435                 rte_eth_dev_probing_finish(eth_dev);
3436                 return 0;
3437         }
3438
3439         kvlist = rte_kvargs_parse(rte_vdev_device_args(dev),
3440                 pmd_bond_init_valid_arguments);
3441         if (kvlist == NULL)
3442                 return -1;
3443
3444         /* Parse link bonding mode */
3445         if (rte_kvargs_count(kvlist, PMD_BOND_MODE_KVARG) == 1) {
3446                 if (rte_kvargs_process(kvlist, PMD_BOND_MODE_KVARG,
3447                                 &bond_ethdev_parse_slave_mode_kvarg,
3448                                 &bonding_mode) != 0) {
3449                         RTE_BOND_LOG(ERR, "Invalid mode for bonded device %s",
3450                                         name);
3451                         goto parse_error;
3452                 }
3453         } else {
3454                 RTE_BOND_LOG(ERR, "Mode must be specified only once for bonded "
3455                                 "device %s", name);
3456                 goto parse_error;
3457         }
3458
3459         /* Parse socket id to create bonding device on */
3460         arg_count = rte_kvargs_count(kvlist, PMD_BOND_SOCKET_ID_KVARG);
3461         if (arg_count == 1) {
3462                 if (rte_kvargs_process(kvlist, PMD_BOND_SOCKET_ID_KVARG,
3463                                 &bond_ethdev_parse_socket_id_kvarg, &socket_id)
3464                                 != 0) {
3465                         RTE_BOND_LOG(ERR, "Invalid socket Id specified for "
3466                                         "bonded device %s", name);
3467                         goto parse_error;
3468                 }
3469         } else if (arg_count > 1) {
3470                 RTE_BOND_LOG(ERR, "Socket Id can be specified only once for "
3471                                 "bonded device %s", name);
3472                 goto parse_error;
3473         } else {
3474                 socket_id = rte_socket_id();
3475         }
3476
3477         dev->device.numa_node = socket_id;
3478
3479         /* Create link bonding eth device */
3480         port_id = bond_alloc(dev, bonding_mode);
3481         if (port_id < 0) {
3482                 RTE_BOND_LOG(ERR, "Failed to create socket %s in mode %u on "
3483                                 "socket %u.",   name, bonding_mode, socket_id);
3484                 goto parse_error;
3485         }
3486         internals = rte_eth_devices[port_id].data->dev_private;
3487         internals->kvlist = kvlist;
3488
3489         if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) {
3490                 if (rte_kvargs_process(kvlist,
3491                                 PMD_BOND_AGG_MODE_KVARG,
3492                                 &bond_ethdev_parse_slave_agg_mode_kvarg,
3493                                 &agg_mode) != 0) {
3494                         RTE_BOND_LOG(ERR,
3495                                         "Failed to parse agg selection mode for bonded device %s",
3496                                         name);
3497                         goto parse_error;
3498                 }
3499
3500                 if (internals->mode == BONDING_MODE_8023AD)
3501                         internals->mode4.agg_selection = agg_mode;
3502         } else {
3503                 internals->mode4.agg_selection = AGG_STABLE;
3504         }
3505
3506         rte_eth_dev_probing_finish(&rte_eth_devices[port_id]);
3507         RTE_BOND_LOG(INFO, "Create bonded device %s on port %d in mode %u on "
3508                         "socket %u.",   name, port_id, bonding_mode, socket_id);
3509         return 0;
3510
3511 parse_error:
3512         rte_kvargs_free(kvlist);
3513
3514         return -1;
3515 }
3516
3517 static int
3518 bond_remove(struct rte_vdev_device *dev)
3519 {
3520         struct rte_eth_dev *eth_dev;
3521         struct bond_dev_private *internals;
3522         const char *name;
3523         int ret = 0;
3524
3525         if (!dev)
3526                 return -EINVAL;
3527
3528         name = rte_vdev_device_name(dev);
3529         RTE_BOND_LOG(INFO, "Uninitializing pmd_bond for %s", name);
3530
3531         /* find an ethdev entry */
3532         eth_dev = rte_eth_dev_allocated(name);
3533         if (eth_dev == NULL)
3534                 return 0; /* port already released */
3535
3536         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3537                 return rte_eth_dev_release_port(eth_dev);
3538
3539         RTE_ASSERT(eth_dev->device == &dev->device);
3540
3541         internals = eth_dev->data->dev_private;
3542         if (internals->slave_count != 0)
3543                 return -EBUSY;
3544
3545         if (eth_dev->data->dev_started == 1) {
3546                 ret = bond_ethdev_stop(eth_dev);
3547                 bond_ethdev_close(eth_dev);
3548         }
3549         rte_eth_dev_release_port(eth_dev);
3550
3551         return ret;
3552 }
3553
3554 /* this part will resolve the slave portids after all the other pdev and vdev
3555  * have been allocated */
3556 static int
3557 bond_ethdev_configure(struct rte_eth_dev *dev)
3558 {
3559         const char *name = dev->device->name;
3560         struct bond_dev_private *internals = dev->data->dev_private;
3561         struct rte_kvargs *kvlist = internals->kvlist;
3562         int arg_count;
3563         uint16_t port_id = dev - rte_eth_devices;
3564         uint8_t agg_mode;
3565
3566         static const uint8_t default_rss_key[40] = {
3567                 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2, 0x41, 0x67, 0x25, 0x3D,
3568                 0x43, 0xA3, 0x8F, 0xB0, 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
3569                 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C, 0x6A, 0x42, 0xB7, 0x3B,
3570                 0xBE, 0xAC, 0x01, 0xFA
3571         };
3572
3573         unsigned i, j;
3574
3575         /*
3576          * If RSS is enabled, fill table with default values and
3577          * set key to the value specified in port RSS configuration.
3578          * Fall back to default RSS key if the key is not specified
3579          */
3580         if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS) {
3581                 struct rte_eth_rss_conf *rss_conf =
3582                         &dev->data->dev_conf.rx_adv_conf.rss_conf;
3583
3584                 if (internals->rss_key_len == 0) {
3585                         internals->rss_key_len = sizeof(default_rss_key);
3586                 }
3587
3588                 if (rss_conf->rss_key != NULL) {
3589                         if (internals->rss_key_len > rss_conf->rss_key_len) {
3590                                 RTE_BOND_LOG(ERR, "Invalid rss key length(%u)",
3591                                                 rss_conf->rss_key_len);
3592                                 return -EINVAL;
3593                         }
3594
3595                         memcpy(internals->rss_key, rss_conf->rss_key,
3596                                internals->rss_key_len);
3597                 } else {
3598                         if (internals->rss_key_len > sizeof(default_rss_key)) {
3599                                 RTE_BOND_LOG(ERR,
3600                                        "There is no suitable default hash key");
3601                                 return -EINVAL;
3602                         }
3603
3604                         memcpy(internals->rss_key, default_rss_key,
3605                                internals->rss_key_len);
3606                 }
3607
3608                 for (i = 0; i < RTE_DIM(internals->reta_conf); i++) {
3609                         internals->reta_conf[i].mask = ~0LL;
3610                         for (j = 0; j < RTE_ETH_RETA_GROUP_SIZE; j++)
3611                                 internals->reta_conf[i].reta[j] =
3612                                                 (i * RTE_ETH_RETA_GROUP_SIZE + j) %
3613                                                 dev->data->nb_rx_queues;
3614                 }
3615         }
3616
3617         /* set the max_rx_pktlen */
3618         internals->max_rx_pktlen = internals->candidate_max_rx_pktlen;
3619
3620         /*
3621          * if no kvlist, it means that this bonded device has been created
3622          * through the bonding api.
3623          */
3624         if (!kvlist)
3625                 return 0;
3626
3627         /* Parse MAC address for bonded device */
3628         arg_count = rte_kvargs_count(kvlist, PMD_BOND_MAC_ADDR_KVARG);
3629         if (arg_count == 1) {
3630                 struct rte_ether_addr bond_mac;
3631
3632                 if (rte_kvargs_process(kvlist, PMD_BOND_MAC_ADDR_KVARG,
3633                                        &bond_ethdev_parse_bond_mac_addr_kvarg, &bond_mac) < 0) {
3634                         RTE_BOND_LOG(INFO, "Invalid mac address for bonded device %s",
3635                                      name);
3636                         return -1;
3637                 }
3638
3639                 /* Set MAC address */
3640                 if (rte_eth_bond_mac_address_set(port_id, &bond_mac) != 0) {
3641                         RTE_BOND_LOG(ERR,
3642                                      "Failed to set mac address on bonded device %s",
3643                                      name);
3644                         return -1;
3645                 }
3646         } else if (arg_count > 1) {
3647                 RTE_BOND_LOG(ERR,
3648                              "MAC address can be specified only once for bonded device %s",
3649                              name);
3650                 return -1;
3651         }
3652
3653         /* Parse/set balance mode transmit policy */
3654         arg_count = rte_kvargs_count(kvlist, PMD_BOND_XMIT_POLICY_KVARG);
3655         if (arg_count == 1) {
3656                 uint8_t xmit_policy;
3657
3658                 if (rte_kvargs_process(kvlist, PMD_BOND_XMIT_POLICY_KVARG,
3659                                        &bond_ethdev_parse_balance_xmit_policy_kvarg, &xmit_policy) !=
3660                     0) {
3661                         RTE_BOND_LOG(INFO,
3662                                      "Invalid xmit policy specified for bonded device %s",
3663                                      name);
3664                         return -1;
3665                 }
3666
3667                 /* Set balance mode transmit policy*/
3668                 if (rte_eth_bond_xmit_policy_set(port_id, xmit_policy) != 0) {
3669                         RTE_BOND_LOG(ERR,
3670                                      "Failed to set balance xmit policy on bonded device %s",
3671                                      name);
3672                         return -1;
3673                 }
3674         } else if (arg_count > 1) {
3675                 RTE_BOND_LOG(ERR,
3676                              "Transmit policy can be specified only once for bonded device %s",
3677                              name);
3678                 return -1;
3679         }
3680
3681         if (rte_kvargs_count(kvlist, PMD_BOND_AGG_MODE_KVARG) == 1) {
3682                 if (rte_kvargs_process(kvlist,
3683                                        PMD_BOND_AGG_MODE_KVARG,
3684                                        &bond_ethdev_parse_slave_agg_mode_kvarg,
3685                                        &agg_mode) != 0) {
3686                         RTE_BOND_LOG(ERR,
3687                                      "Failed to parse agg selection mode for bonded device %s",
3688                                      name);
3689                 }
3690                 if (internals->mode == BONDING_MODE_8023AD) {
3691                         int ret = rte_eth_bond_8023ad_agg_selection_set(port_id,
3692                                         agg_mode);
3693                         if (ret < 0) {
3694                                 RTE_BOND_LOG(ERR,
3695                                         "Invalid args for agg selection set for bonded device %s",
3696                                         name);
3697                                 return -1;
3698                         }
3699                 }
3700         }
3701
3702         /* Parse/add slave ports to bonded device */
3703         if (rte_kvargs_count(kvlist, PMD_BOND_SLAVE_PORT_KVARG) > 0) {
3704                 struct bond_ethdev_slave_ports slave_ports;
3705                 unsigned i;
3706
3707                 memset(&slave_ports, 0, sizeof(slave_ports));
3708
3709                 if (rte_kvargs_process(kvlist, PMD_BOND_SLAVE_PORT_KVARG,
3710                                        &bond_ethdev_parse_slave_port_kvarg, &slave_ports) != 0) {
3711                         RTE_BOND_LOG(ERR,
3712                                      "Failed to parse slave ports for bonded device %s",
3713                                      name);
3714                         return -1;
3715                 }
3716
3717                 for (i = 0; i < slave_ports.slave_count; i++) {
3718                         if (rte_eth_bond_slave_add(port_id, slave_ports.slaves[i]) != 0) {
3719                                 RTE_BOND_LOG(ERR,
3720                                              "Failed to add port %d as slave to bonded device %s",
3721                                              slave_ports.slaves[i], name);
3722                         }
3723                 }
3724
3725         } else {
3726                 RTE_BOND_LOG(INFO, "No slaves specified for bonded device %s", name);
3727                 return -1;
3728         }
3729
3730         /* Parse/set primary slave port id*/
3731         arg_count = rte_kvargs_count(kvlist, PMD_BOND_PRIMARY_SLAVE_KVARG);
3732         if (arg_count == 1) {
3733                 uint16_t primary_slave_port_id;
3734
3735                 if (rte_kvargs_process(kvlist,
3736                                        PMD_BOND_PRIMARY_SLAVE_KVARG,
3737                                        &bond_ethdev_parse_primary_slave_port_id_kvarg,
3738                                        &primary_slave_port_id) < 0) {
3739                         RTE_BOND_LOG(INFO,
3740                                      "Invalid primary slave port id specified for bonded device %s",
3741                                      name);
3742                         return -1;
3743                 }
3744
3745                 /* Set balance mode transmit policy*/
3746                 if (rte_eth_bond_primary_set(port_id, primary_slave_port_id)
3747                     != 0) {
3748                         RTE_BOND_LOG(ERR,
3749                                      "Failed to set primary slave port %d on bonded device %s",
3750                                      primary_slave_port_id, name);
3751                         return -1;
3752                 }
3753         } else if (arg_count > 1) {
3754                 RTE_BOND_LOG(INFO,
3755                              "Primary slave can be specified only once for bonded device %s",
3756                              name);
3757                 return -1;
3758         }
3759
3760         /* Parse link status monitor polling interval */
3761         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LSC_POLL_PERIOD_KVARG);
3762         if (arg_count == 1) {
3763                 uint32_t lsc_poll_interval_ms;
3764
3765                 if (rte_kvargs_process(kvlist,
3766                                        PMD_BOND_LSC_POLL_PERIOD_KVARG,
3767                                        &bond_ethdev_parse_time_ms_kvarg,
3768                                        &lsc_poll_interval_ms) < 0) {
3769                         RTE_BOND_LOG(INFO,
3770                                      "Invalid lsc polling interval value specified for bonded"
3771                                      " device %s", name);
3772                         return -1;
3773                 }
3774
3775                 if (rte_eth_bond_link_monitoring_set(port_id, lsc_poll_interval_ms)
3776                     != 0) {
3777                         RTE_BOND_LOG(ERR,
3778                                      "Failed to set lsc monitor polling interval (%u ms) on bonded device %s",
3779                                      lsc_poll_interval_ms, name);
3780                         return -1;
3781                 }
3782         } else if (arg_count > 1) {
3783                 RTE_BOND_LOG(INFO,
3784                              "LSC polling interval can be specified only once for bonded"
3785                              " device %s", name);
3786                 return -1;
3787         }
3788
3789         /* Parse link up interrupt propagation delay */
3790         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_UP_PROP_DELAY_KVARG);
3791         if (arg_count == 1) {
3792                 uint32_t link_up_delay_ms;
3793
3794                 if (rte_kvargs_process(kvlist,
3795                                        PMD_BOND_LINK_UP_PROP_DELAY_KVARG,
3796                                        &bond_ethdev_parse_time_ms_kvarg,
3797                                        &link_up_delay_ms) < 0) {
3798                         RTE_BOND_LOG(INFO,
3799                                      "Invalid link up propagation delay value specified for"
3800                                      " bonded device %s", name);
3801                         return -1;
3802                 }
3803
3804                 /* Set balance mode transmit policy*/
3805                 if (rte_eth_bond_link_up_prop_delay_set(port_id, link_up_delay_ms)
3806                     != 0) {
3807                         RTE_BOND_LOG(ERR,
3808                                      "Failed to set link up propagation delay (%u ms) on bonded"
3809                                      " device %s", link_up_delay_ms, name);
3810                         return -1;
3811                 }
3812         } else if (arg_count > 1) {
3813                 RTE_BOND_LOG(INFO,
3814                              "Link up propagation delay can be specified only once for"
3815                              " bonded device %s", name);
3816                 return -1;
3817         }
3818
3819         /* Parse link down interrupt propagation delay */
3820         arg_count = rte_kvargs_count(kvlist, PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG);
3821         if (arg_count == 1) {
3822                 uint32_t link_down_delay_ms;
3823
3824                 if (rte_kvargs_process(kvlist,
3825                                        PMD_BOND_LINK_DOWN_PROP_DELAY_KVARG,
3826                                        &bond_ethdev_parse_time_ms_kvarg,
3827                                        &link_down_delay_ms) < 0) {
3828                         RTE_BOND_LOG(INFO,
3829                                      "Invalid link down propagation delay value specified for"
3830                                      " bonded device %s", name);
3831                         return -1;
3832                 }
3833
3834                 /* Set balance mode transmit policy*/
3835                 if (rte_eth_bond_link_down_prop_delay_set(port_id, link_down_delay_ms)
3836                     != 0) {
3837                         RTE_BOND_LOG(ERR,
3838                                      "Failed to set link down propagation delay (%u ms) on bonded device %s",
3839                                      link_down_delay_ms, name);
3840                         return -1;
3841                 }
3842         } else if (arg_count > 1) {
3843                 RTE_BOND_LOG(INFO,
3844                              "Link down propagation delay can be specified only once for  bonded device %s",
3845                              name);
3846                 return -1;
3847         }
3848
3849         return 0;
3850 }
3851
3852 struct rte_vdev_driver pmd_bond_drv = {
3853         .probe = bond_probe,
3854         .remove = bond_remove,
3855 };
3856
3857 RTE_PMD_REGISTER_VDEV(net_bonding, pmd_bond_drv);
3858 RTE_PMD_REGISTER_ALIAS(net_bonding, eth_bond);
3859
3860 RTE_PMD_REGISTER_PARAM_STRING(net_bonding,
3861         "slave=<ifc> "
3862         "primary=<ifc> "
3863         "mode=[0-6] "
3864         "xmit_policy=[l2 | l23 | l34] "
3865         "agg_mode=[count | stable | bandwidth] "
3866         "socket_id=<int> "
3867         "mac=<mac addr> "
3868         "lsc_poll_period_ms=<int> "
3869         "up_delay=<int> "
3870         "down_delay=<int>");
3871
3872 /* We can't use RTE_LOG_REGISTER_DEFAULT because of the forced name for
3873  * this library, see meson.build.
3874  */
3875 RTE_LOG_REGISTER(bond_logtype, pmd.net.bonding, NOTICE);