drivers/net/bonding/rte_eth_bond_8023ad.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <stddef.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_alarm.h>
  10 #include <rte_malloc.h>
  11 #include <rte_errno.h>
  12 #include <rte_cycles.h>
  13 #include <rte_compat.h>
  14
  15 #include "eth_bond_private.h"
  16
  17 static void bond_mode_8023ad_ext_periodic_cb(void *arg);
  18 #ifdef RTE_LIBRTE_BOND_DEBUG_8023AD
  19
  20 #define MODE4_DEBUG(fmt, ...)                           \
  21         rte_log(RTE_LOG_DEBUG, bond_logtype,            \
  22                 "%6u [Port %u: %s] " fmt,               \
  23                 bond_dbg_get_time_diff_ms(), slave_id,  \
  24                 __func__, ##__VA_ARGS__)
  25
  26 static uint64_t start_time;
  27
  28 static unsigned
  29 bond_dbg_get_time_diff_ms(void)
  30 {
  31         uint64_t now;
  32
  33         now = rte_rdtsc();
  34         if (start_time == 0)
  35                 start_time = now;
  36
  37         return ((now - start_time) * 1000) / rte_get_tsc_hz();
  38 }
  39
  40 static void
  41 bond_print_lacp(struct lacpdu *l)
  42 {
  43         char a_address[18];
  44         char p_address[18];
  45         char a_state[256] = { 0 };
  46         char p_state[256] = { 0 };
  47
  48         static const char * const state_labels[] = {
  49                 "ACT", "TIMEOUT", "AGG", "SYNC", "COL", "DIST", "DEF", "EXP"
  50         };
  51
  52         int a_len = 0;
  53         int p_len = 0;
  54         uint8_t i;
  55         uint8_t *addr;
  56
  57         addr = l->actor.port_params.system.addr_bytes;
  58         snprintf(a_address, sizeof(a_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  59                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  60
  61         addr = l->partner.port_params.system.addr_bytes;
  62         snprintf(p_address, sizeof(p_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  63                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  64
  65         for (i = 0; i < 8; i++) {
  66                 if ((l->actor.state >> i) & 1) {
  67                         a_len += snprintf(&a_state[a_len], RTE_DIM(a_state) - a_len, "%s ",
  68                                 state_labels[i]);
  69                 }
  70
  71                 if ((l->partner.state >> i) & 1) {
  72                         p_len += snprintf(&p_state[p_len], RTE_DIM(p_state) - p_len, "%s ",
  73                                 state_labels[i]);
  74                 }
  75         }
  76
  77         if (a_len && a_state[a_len-1] == ' ')
  78                 a_state[a_len-1] = '\0';
  79
  80         if (p_len && p_state[p_len-1] == ' ')
  81                 p_state[p_len-1] = '\0';
  82
  83         RTE_BOND_LOG(DEBUG,
  84                      "LACP: {\n"
  85                      "  subtype= %02X\n"
  86                      "  ver_num=%02X\n"
  87                      "  actor={ tlv=%02X, len=%02X\n"
  88                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  89                      "       state={ %s }\n"
  90                      "  }\n"
  91                      "  partner={ tlv=%02X, len=%02X\n"
  92                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  93                      "       state={ %s }\n"
  94                      "  }\n"
  95                      "  collector={info=%02X, length=%02X, max_delay=%04X\n, "
  96                      "type_term=%02X, terminator_length = %02X }",
  97                      l->subtype,
  98                      l->version_number,
  99                      l->actor.tlv_type_info,
 100                      l->actor.info_length,
 101                      l->actor.port_params.system_priority,
 102                      a_address,
 103                      l->actor.port_params.key,
 104                      l->actor.port_params.port_priority,
 105                      l->actor.port_params.port_number,
 106                      a_state,
 107                      l->partner.tlv_type_info,
 108                      l->partner.info_length,
 109                      l->partner.port_params.system_priority,
 110                      p_address,
 111                      l->partner.port_params.key,
 112                      l->partner.port_params.port_priority,
 113                      l->partner.port_params.port_number,
 114                      p_state,
 115                      l->tlv_type_collector_info,
 116                      l->collector_info_length,
 117                      l->collector_max_delay,
 118                      l->tlv_type_terminator,
 119                      l->terminator_length);
 120
 121 }
 122
 123 #define BOND_PRINT_LACP(lacpdu) bond_print_lacp(lacpdu)
 124 #else
 125 #define BOND_PRINT_LACP(lacpdu) do { } while (0)
 126 #define MODE4_DEBUG(fmt, ...) do { } while (0)
 127 #endif
 128
 129 static const struct rte_ether_addr lacp_mac_addr = {
 130         .addr_bytes = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 }
 131 };
 132
 133 struct port bond_mode_8023ad_ports[RTE_MAX_ETHPORTS];
 134
 135 static void
 136 timer_cancel(uint64_t *timer)
 137 {
 138         *timer = 0;
 139 }
 140
 141 static void
 142 timer_set(uint64_t *timer, uint64_t timeout)
 143 {
 144         *timer = rte_rdtsc() + timeout;
 145 }
 146
 147 /* Forces given timer to be in expired state. */
 148 static void
 149 timer_force_expired(uint64_t *timer)
 150 {
 151         *timer = rte_rdtsc();
 152 }
 153
 154 static bool
 155 timer_is_stopped(uint64_t *timer)
 156 {
 157         return *timer == 0;
 158 }
 159
 160 static bool
 161 timer_is_expired(uint64_t *timer)
 162 {
 163         return *timer < rte_rdtsc();
 164 }
 165
 166 /* Timer is in running state if it is not stopped nor expired */
 167 static bool
 168 timer_is_running(uint64_t *timer)
 169 {
 170         return !timer_is_stopped(timer) && !timer_is_expired(timer);
 171 }
 172
 173 static void
 174 set_warning_flags(struct port *port, uint16_t flags)
 175 {
 176         int retval;
 177         uint16_t old;
 178         uint16_t new_flag = 0;
 179
 180         do {
 181                 old = port->warnings_to_show;
 182                 new_flag = old | flags;
 183                 retval = rte_atomic16_cmpset(&port->warnings_to_show, old, new_flag);
 184         } while (unlikely(retval == 0));
 185 }
 186
 187 static void
 188 show_warnings(uint16_t slave_id)
 189 {
 190         struct port *port = &bond_mode_8023ad_ports[slave_id];
 191         uint8_t warnings;
 192
 193         do {
 194                 warnings = port->warnings_to_show;
 195         } while (rte_atomic16_cmpset(&port->warnings_to_show, warnings, 0) == 0);
 196
 197         if (!warnings)
 198                 return;
 199
 200         if (!timer_is_expired(&port->warning_timer))
 201                 return;
 202
 203
 204         timer_set(&port->warning_timer, BOND_8023AD_WARNINGS_PERIOD_MS *
 205                         rte_get_tsc_hz() / 1000);
 206
 207         if (warnings & WRN_RX_QUEUE_FULL) {
 208                 RTE_BOND_LOG(DEBUG,
 209                              "Slave %u: failed to enqueue LACP packet into RX ring.\n"
 210                              "Receive and transmit functions must be invoked on bonded"
 211                              "interface at least 10 times per second or LACP will notwork correctly",
 212                              slave_id);
 213         }
 214
 215         if (warnings & WRN_TX_QUEUE_FULL) {
 216                 RTE_BOND_LOG(DEBUG,
 217                              "Slave %u: failed to enqueue LACP packet into TX ring.\n"
 218                              "Receive and transmit functions must be invoked on bonded"
 219                              "interface at least 10 times per second or LACP will not work correctly",
 220                              slave_id);
 221         }
 222
 223         if (warnings & WRN_RX_MARKER_TO_FAST)
 224                 RTE_BOND_LOG(INFO, "Slave %u: marker to early - ignoring.",
 225                              slave_id);
 226
 227         if (warnings & WRN_UNKNOWN_SLOW_TYPE) {
 228                 RTE_BOND_LOG(INFO,
 229                         "Slave %u: ignoring unknown slow protocol frame type",
 230                              slave_id);
 231         }
 232
 233         if (warnings & WRN_UNKNOWN_MARKER_TYPE)
 234                 RTE_BOND_LOG(INFO, "Slave %u: ignoring unknown marker type",
 235                              slave_id);
 236
 237         if (warnings & WRN_NOT_LACP_CAPABLE)
 238                 MODE4_DEBUG("Port %u is not LACP capable!\n", slave_id);
 239 }
 240
 241 static void
 242 record_default(struct port *port)
 243 {
 244         /* Record default parameters for partner. Partner admin parameters
 245          * are not implemented so set them to arbitrary default (last known) and
 246          * mark actor that parner is in defaulted state. */
 247         port->partner_state = STATE_LACP_ACTIVE;
 248         ACTOR_STATE_SET(port, DEFAULTED);
 249 }
 250
 251 /** Function handles rx state machine.
 252  *
 253  * This function implements Receive State Machine from point 5.4.12 in
 254  * 802.1AX documentation. It should be called periodically.
 255  *
 256  * @param lacpdu                LACPDU received.
 257  * @param port                  Port on which LACPDU was received.
 258  */
 259 static void
 260 rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 261                 struct lacpdu *lacp)
 262 {
 263         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 264         uint64_t timeout;
 265
 266         if (SM_FLAG(port, BEGIN)) {
 267                 /* Initialize stuff */
 268                 MODE4_DEBUG("-> INITIALIZE\n");
 269                 SM_FLAG_CLR(port, MOVED);
 270                 port->selected = UNSELECTED;
 271
 272                 record_default(port);
 273
 274                 ACTOR_STATE_CLR(port, EXPIRED);
 275                 timer_cancel(&port->current_while_timer);
 276
 277                 /* DISABLED: On initialization partner is out of sync */
 278                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 279
 280                 /* LACP DISABLED stuff if LACP not enabled on this port */
 281                 if (!SM_FLAG(port, LACP_ENABLED))
 282                         PARTNER_STATE_CLR(port, AGGREGATION);
 283                 else
 284                         PARTNER_STATE_SET(port, AGGREGATION);
 285         }
 286
 287         if (!SM_FLAG(port, LACP_ENABLED)) {
 288                 /* Update parameters only if state changed */
 289                 if (!timer_is_stopped(&port->current_while_timer)) {
 290                         port->selected = UNSELECTED;
 291                         record_default(port);
 292                         PARTNER_STATE_CLR(port, AGGREGATION);
 293                         ACTOR_STATE_CLR(port, EXPIRED);
 294                         timer_cancel(&port->current_while_timer);
 295                 }
 296                 return;
 297         }
 298
 299         if (lacp) {
 300                 MODE4_DEBUG("LACP -> CURRENT\n");
 301                 BOND_PRINT_LACP(lacp);
 302                 /* Update selected flag. If partner parameters are defaulted assume they
 303                  * are match. If not defaulted  compare LACP actor with ports parner
 304                  * params. */
 305                 if (!ACTOR_STATE(port, DEFAULTED) &&
 306                         (ACTOR_STATE(port, AGGREGATION) != PARTNER_STATE(port, AGGREGATION)
 307                         || memcmp(&port->partner, &lacp->actor.port_params,
 308                                 sizeof(port->partner)) != 0)) {
 309                         MODE4_DEBUG("selected <- UNSELECTED\n");
 310                         port->selected = UNSELECTED;
 311                 }
 312
 313                 /* Record this PDU actor params as partner params */
 314                 memcpy(&port->partner, &lacp->actor.port_params,
 315                         sizeof(struct port_params));
 316                 port->partner_state = lacp->actor.state;
 317
 318                 /* Partner parameters are not defaulted any more */
 319                 ACTOR_STATE_CLR(port, DEFAULTED);
 320
 321                 /* If LACP partner params match this port actor params */
 322                 agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 323                 bool match = port->actor.system_priority ==
 324                         lacp->partner.port_params.system_priority &&
 325                         rte_is_same_ether_addr(&agg->actor.system,
 326                         &lacp->partner.port_params.system) &&
 327                         port->actor.port_priority ==
 328                         lacp->partner.port_params.port_priority &&
 329                         port->actor.port_number ==
 330                         lacp->partner.port_params.port_number;
 331
 332                 /* Update NTT if partners information are outdated (xored and masked
 333                  * bits are set)*/
 334                 uint8_t state_mask = STATE_LACP_ACTIVE | STATE_LACP_SHORT_TIMEOUT |
 335                         STATE_SYNCHRONIZATION | STATE_AGGREGATION;
 336
 337                 if (((port->actor_state ^ lacp->partner.state) & state_mask) ||
 338                                 match == false) {
 339                         SM_FLAG_SET(port, NTT);
 340                 }
 341
 342                 /* If LACP partner params match this port actor params */
 343                 if (match == true && ACTOR_STATE(port, AGGREGATION) ==
 344                                 PARTNER_STATE(port,     AGGREGATION))
 345                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 346                 else if (!PARTNER_STATE(port, AGGREGATION) && ACTOR_STATE(port,
 347                                 AGGREGATION))
 348                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 349                 else
 350                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 351
 352                 if (ACTOR_STATE(port, LACP_SHORT_TIMEOUT))
 353                         timeout = internals->mode4.short_timeout;
 354                 else
 355                         timeout = internals->mode4.long_timeout;
 356
 357                 timer_set(&port->current_while_timer, timeout);
 358                 ACTOR_STATE_CLR(port, EXPIRED);
 359                 SM_FLAG_CLR(port, EXPIRED);
 360                 return; /* No state change */
 361         }
 362
 363         /* If CURRENT state timer is not running (stopped or expired)
 364          * transit to EXPIRED state from DISABLED or CURRENT */
 365         if (!timer_is_running(&port->current_while_timer)) {
 366                 if (SM_FLAG(port, EXPIRED)) {
 367                         port->selected = UNSELECTED;
 368                         memcpy(&port->partner, &port->partner_admin,
 369                                 sizeof(struct port_params));
 370                         record_default(port);
 371                         ACTOR_STATE_CLR(port, EXPIRED);
 372                         timer_cancel(&port->current_while_timer);
 373                 } else {
 374                         SM_FLAG_SET(port, EXPIRED);
 375                         ACTOR_STATE_SET(port, EXPIRED);
 376                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 377                         PARTNER_STATE_SET(port, LACP_SHORT_TIMEOUT);
 378                         timer_set(&port->current_while_timer,
 379                                 internals->mode4.short_timeout);
 380                 }
 381         }
 382 }
 383
 384 /**
 385  * Function handles periodic tx state machine.
 386  *
 387  * Function implements Periodic Transmission state machine from point 5.4.13
 388  * in 802.1AX documentation. It should be called periodically.
 389  *
 390  * @param port                  Port to handle state machine.
 391  */
 392 static void
 393 periodic_machine(struct bond_dev_private *internals, uint16_t slave_id)
 394 {
 395         struct port *port = &bond_mode_8023ad_ports[slave_id];
 396         /* Calculate if either site is LACP enabled */
 397         uint64_t timeout;
 398         uint8_t active = ACTOR_STATE(port, LACP_ACTIVE) ||
 399                 PARTNER_STATE(port, LACP_ACTIVE);
 400
 401         uint8_t is_partner_fast, was_partner_fast;
 402         /* No periodic is on BEGIN, LACP DISABLE or when both sides are pasive */
 403         if (SM_FLAG(port, BEGIN) || !SM_FLAG(port, LACP_ENABLED) || !active) {
 404                 timer_cancel(&port->periodic_timer);
 405                 timer_force_expired(&port->tx_machine_timer);
 406                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 407
 408                 MODE4_DEBUG("-> NO_PERIODIC ( %s%s%s)\n",
 409                         SM_FLAG(port, BEGIN) ? "begind " : "",
 410                         SM_FLAG(port, LACP_ENABLED) ? "" : "LACP disabled ",
 411                         active ? "LACP active " : "LACP pasive ");
 412                 return;
 413         }
 414
 415         is_partner_fast = PARTNER_STATE(port, LACP_SHORT_TIMEOUT);
 416         was_partner_fast = SM_FLAG(port, PARTNER_SHORT_TIMEOUT);
 417
 418         /* If periodic timer is not started, transit from NO PERIODIC to FAST/SLOW.
 419          * Other case: check if timer expire or partners settings changed. */
 420         if (!timer_is_stopped(&port->periodic_timer)) {
 421                 if (timer_is_expired(&port->periodic_timer)) {
 422                         SM_FLAG_SET(port, NTT);
 423                 } else if (is_partner_fast != was_partner_fast) {
 424                         /* Partners timeout  was slow and now it is fast -> send LACP.
 425                          * In other case (was fast and now it is slow) just switch
 426                          * timeout to slow without forcing send of LACP (because standard
 427                          * say so)*/
 428                         if (is_partner_fast)
 429                                 SM_FLAG_SET(port, NTT);
 430                 } else
 431                         return; /* Nothing changed */
 432         }
 433
 434         /* Handle state transition to FAST/SLOW LACP timeout */
 435         if (is_partner_fast) {
 436                 timeout = internals->mode4.fast_periodic_timeout;
 437                 SM_FLAG_SET(port, PARTNER_SHORT_TIMEOUT);
 438         } else {
 439                 timeout = internals->mode4.slow_periodic_timeout;
 440                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 441         }
 442
 443         timer_set(&port->periodic_timer, timeout);
 444 }
 445
 446 /**
 447  * Function handles mux state machine.
 448  *
 449  * Function implements Mux Machine from point 5.4.15 in 802.1AX documentation.
 450  * It should be called periodically.
 451  *
 452  * @param port                  Port to handle state machine.
 453  */
 454 static void
 455 mux_machine(struct bond_dev_private *internals, uint16_t slave_id)
 456 {
 457         struct port *port = &bond_mode_8023ad_ports[slave_id];
 458
 459         /* Save current state for later use */
 460         const uint8_t state_mask = STATE_SYNCHRONIZATION | STATE_DISTRIBUTING |
 461                 STATE_COLLECTING;
 462
 463         /* Enter DETACHED state on BEGIN condition or from any other state if
 464          * port was unselected */
 465         if (SM_FLAG(port, BEGIN) ||
 466                         port->selected == UNSELECTED || (port->selected == STANDBY &&
 467                                 (port->actor_state & state_mask) != 0)) {
 468                 /* detach mux from aggregator */
 469                 port->actor_state &= ~state_mask;
 470                 /* Set ntt to true if BEGIN condition or transition from any other state
 471                  * which is indicated that wait_while_timer was started */
 472                 if (SM_FLAG(port, BEGIN) ||
 473                                 !timer_is_stopped(&port->wait_while_timer)) {
 474                         SM_FLAG_SET(port, NTT);
 475                         MODE4_DEBUG("-> DETACHED\n");
 476                 }
 477                 timer_cancel(&port->wait_while_timer);
 478         }
 479
 480         if (timer_is_stopped(&port->wait_while_timer)) {
 481                 if (port->selected == SELECTED || port->selected == STANDBY) {
 482                         timer_set(&port->wait_while_timer,
 483                                 internals->mode4.aggregate_wait_timeout);
 484
 485                         MODE4_DEBUG("DETACHED -> WAITING\n");
 486                 }
 487                 /* Waiting state entered */
 488                 return;
 489         }
 490
 491         /* Transit next state if port is ready */
 492         if (!timer_is_expired(&port->wait_while_timer))
 493                 return;
 494
 495         if ((ACTOR_STATE(port, DISTRIBUTING) || ACTOR_STATE(port, COLLECTING)) &&
 496                 !PARTNER_STATE(port, SYNCHRONIZATION)) {
 497                 /* If in COLLECTING or DISTRIBUTING state and partner becomes out of
 498                  * sync transit to ATACHED state.  */
 499                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 500                 ACTOR_STATE_CLR(port, COLLECTING);
 501                 /* Clear actor sync to activate transit ATACHED in condition bellow */
 502                 ACTOR_STATE_CLR(port, SYNCHRONIZATION);
 503                 MODE4_DEBUG("Out of sync -> ATTACHED\n");
 504         }
 505
 506         if (!ACTOR_STATE(port, SYNCHRONIZATION)) {
 507                 /* attach mux to aggregator */
 508                 RTE_ASSERT((port->actor_state & (STATE_COLLECTING |
 509                         STATE_DISTRIBUTING)) == 0);
 510
 511                 ACTOR_STATE_SET(port, SYNCHRONIZATION);
 512                 SM_FLAG_SET(port, NTT);
 513                 MODE4_DEBUG("ATTACHED Entered\n");
 514         } else if (!ACTOR_STATE(port, COLLECTING)) {
 515                 /* Start collecting if in sync */
 516                 if (PARTNER_STATE(port, SYNCHRONIZATION)) {
 517                         MODE4_DEBUG("ATTACHED -> COLLECTING\n");
 518                         ACTOR_STATE_SET(port, COLLECTING);
 519                         SM_FLAG_SET(port, NTT);
 520                 }
 521         } else if (ACTOR_STATE(port, COLLECTING)) {
 522                 /* Check if partner is in COLLECTING state. If so this port can
 523                  * distribute frames to it */
 524                 if (!ACTOR_STATE(port, DISTRIBUTING)) {
 525                         if (PARTNER_STATE(port, COLLECTING)) {
 526                                 /* Enable  DISTRIBUTING if partner is collecting */
 527                                 ACTOR_STATE_SET(port, DISTRIBUTING);
 528                                 SM_FLAG_SET(port, NTT);
 529                                 MODE4_DEBUG("COLLECTING -> DISTRIBUTING\n");
 530                                 RTE_BOND_LOG(INFO,
 531                                         "Bond %u: slave id %u distributing started.",
 532                                         internals->port_id, slave_id);
 533                         }
 534                 } else {
 535                         if (!PARTNER_STATE(port, COLLECTING)) {
 536                                 /* Disable DISTRIBUTING (enter COLLECTING state) if partner
 537                                  * is not collecting */
 538                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 539                                 SM_FLAG_SET(port, NTT);
 540                                 MODE4_DEBUG("DISTRIBUTING -> COLLECTING\n");
 541                                 RTE_BOND_LOG(INFO,
 542                                         "Bond %u: slave id %u distributing stopped.",
 543                                         internals->port_id, slave_id);
 544                         }
 545                 }
 546         }
 547 }
 548
 549 /**
 550  * Function handles transmit state machine.
 551  *
 552  * Function implements Transmit Machine from point 5.4.16 in 802.1AX
 553  * documentation.
 554  *
 555  * @param port
 556  */
 557 static void
 558 tx_machine(struct bond_dev_private *internals, uint16_t slave_id)
 559 {
 560         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 561
 562         struct rte_mbuf *lacp_pkt = NULL;
 563         struct lacpdu_header *hdr;
 564         struct lacpdu *lacpdu;
 565
 566         /* If periodic timer is not running periodic machine is in NO PERIODIC and
 567          * according to 802.3ax standard tx machine should not transmit any frames
 568          * and set ntt to false. */
 569         if (timer_is_stopped(&port->periodic_timer))
 570                 SM_FLAG_CLR(port, NTT);
 571
 572         if (!SM_FLAG(port, NTT))
 573                 return;
 574
 575         if (!timer_is_expired(&port->tx_machine_timer))
 576                 return;
 577
 578         lacp_pkt = rte_pktmbuf_alloc(port->mbuf_pool);
 579         if (lacp_pkt == NULL) {
 580                 RTE_BOND_LOG(ERR, "Failed to allocate LACP packet from pool");
 581                 return;
 582         }
 583
 584         lacp_pkt->data_len = sizeof(*hdr);
 585         lacp_pkt->pkt_len = sizeof(*hdr);
 586
 587         hdr = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 588
 589         /* Source and destination MAC */
 590         rte_ether_addr_copy(&lacp_mac_addr, &hdr->eth_hdr.d_addr);
 591         rte_eth_macaddr_get(slave_id, &hdr->eth_hdr.s_addr);
 592         hdr->eth_hdr.ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_SLOW);
 593
 594         lacpdu = &hdr->lacpdu;
 595         memset(lacpdu, 0, sizeof(*lacpdu));
 596
 597         /* Initialize LACP part */
 598         lacpdu->subtype = SLOW_SUBTYPE_LACP;
 599         lacpdu->version_number = 1;
 600
 601         /* ACTOR */
 602         lacpdu->actor.tlv_type_info = TLV_TYPE_ACTOR_INFORMATION;
 603         lacpdu->actor.info_length = sizeof(struct lacpdu_actor_partner_params);
 604         memcpy(&hdr->lacpdu.actor.port_params, &port->actor,
 605                         sizeof(port->actor));
 606         agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 607         rte_ether_addr_copy(&agg->actor.system,
 608                         &hdr->lacpdu.actor.port_params.system);
 609         lacpdu->actor.state = port->actor_state;
 610
 611         /* PARTNER */
 612         lacpdu->partner.tlv_type_info = TLV_TYPE_PARTNER_INFORMATION;
 613         lacpdu->partner.info_length = sizeof(struct lacpdu_actor_partner_params);
 614         memcpy(&lacpdu->partner.port_params, &port->partner,
 615                         sizeof(struct port_params));
 616         lacpdu->partner.state = port->partner_state;
 617
 618         /* Other fields */
 619         lacpdu->tlv_type_collector_info = TLV_TYPE_COLLECTOR_INFORMATION;
 620         lacpdu->collector_info_length = 0x10;
 621         lacpdu->collector_max_delay = 0;
 622
 623         lacpdu->tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION;
 624         lacpdu->terminator_length = 0;
 625
 626         MODE4_DEBUG("Sending LACP frame\n");
 627         BOND_PRINT_LACP(lacpdu);
 628
 629         if (internals->mode4.dedicated_queues.enabled == 0) {
 630                 int retval = rte_ring_enqueue(port->tx_ring, lacp_pkt);
 631                 if (retval != 0) {
 632                         /* If TX ring full, drop packet and free message.
 633                            Retransmission will happen in next function call. */
 634                         rte_pktmbuf_free(lacp_pkt);
 635                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 636                         return;
 637                 }
 638         } else {
 639                 uint16_t pkts_sent = rte_eth_tx_burst(slave_id,
 640                                 internals->mode4.dedicated_queues.tx_qid,
 641                                 &lacp_pkt, 1);
 642                 if (pkts_sent != 1) {
 643                         rte_pktmbuf_free(lacp_pkt);
 644                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 645                         return;
 646                 }
 647         }
 648
 649
 650         timer_set(&port->tx_machine_timer, internals->mode4.tx_period_timeout);
 651         SM_FLAG_CLR(port, NTT);
 652 }
 653
 654 static uint16_t
 655 max_index(uint64_t *a, int n)
 656 {
 657         if (n <= 0)
 658                 return -1;
 659
 660         int i, max_i = 0;
 661         uint64_t max = a[0];
 662
 663         for (i = 1; i < n; ++i) {
 664                 if (a[i] > max) {
 665                         max = a[i];
 666                         max_i = i;
 667                 }
 668         }
 669
 670         return max_i;
 671 }
 672
 673 /**
 674  * Function assigns port to aggregator.
 675  *
 676  * @param bond_dev_private      Pointer to bond_dev_private structure.
 677  * @param port_pos                      Port to assign.
 678  */
 679 static void
 680 selection_logic(struct bond_dev_private *internals, uint16_t slave_id)
 681 {
 682         struct port *agg, *port;
 683         uint16_t slaves_count, new_agg_id, i, j = 0;
 684         uint16_t *slaves;
 685         uint64_t agg_bandwidth[RTE_MAX_ETHPORTS] = {0};
 686         uint64_t agg_count[RTE_MAX_ETHPORTS] = {0};
 687         uint16_t default_slave = 0;
 688         struct rte_eth_link link_info;
 689         uint16_t agg_new_idx = 0;
 690         int ret;
 691
 692         slaves = internals->active_slaves;
 693         slaves_count = internals->active_slave_count;
 694         port = &bond_mode_8023ad_ports[slave_id];
 695
 696         /* Search for aggregator suitable for this port */
 697         for (i = 0; i < slaves_count; ++i) {
 698                 agg = &bond_mode_8023ad_ports[slaves[i]];
 699                 /* Skip ports that are not aggreagators */
 700                 if (agg->aggregator_port_id != slaves[i])
 701                         continue;
 702
 703                 ret = rte_eth_link_get_nowait(slaves[i], &link_info);
 704                 if (ret < 0) {
 705                         RTE_BOND_LOG(ERR,
 706                                 "Slave (port %u) link get failed: %s\n",
 707                                 slaves[i], rte_strerror(-ret));
 708                         continue;
 709                 }
 710                 agg_count[i] += 1;
 711                 agg_bandwidth[i] += link_info.link_speed;
 712
 713                 /* Actors system ID is not checked since all slave device have the same
 714                  * ID (MAC address). */
 715                 if ((agg->actor.key == port->actor.key &&
 716                         agg->partner.system_priority == port->partner.system_priority &&
 717                         rte_is_same_ether_addr(&agg->partner.system,
 718                                         &port->partner.system) == 1
 719                         && (agg->partner.key == port->partner.key)) &&
 720                         rte_is_zero_ether_addr(&port->partner.system) != 1 &&
 721                         (agg->actor.key &
 722                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) != 0) {
 723
 724                         if (j == 0)
 725                                 default_slave = i;
 726                         j++;
 727                 }
 728         }
 729
 730         switch (internals->mode4.agg_selection) {
 731         case AGG_COUNT:
 732                 agg_new_idx = max_index(agg_count, slaves_count);
 733                 new_agg_id = slaves[agg_new_idx];
 734                 break;
 735         case AGG_BANDWIDTH:
 736                 agg_new_idx = max_index(agg_bandwidth, slaves_count);
 737                 new_agg_id = slaves[agg_new_idx];
 738                 break;
 739         case AGG_STABLE:
 740                 if (default_slave == slaves_count)
 741                         new_agg_id = slaves[slave_id];
 742                 else
 743                         new_agg_id = slaves[default_slave];
 744                 break;
 745         default:
 746                 if (default_slave == slaves_count)
 747                         new_agg_id = slaves[slave_id];
 748                 else
 749                         new_agg_id = slaves[default_slave];
 750                 break;
 751         }
 752
 753         if (new_agg_id != port->aggregator_port_id) {
 754                 port->aggregator_port_id = new_agg_id;
 755
 756                 MODE4_DEBUG("-> SELECTED: ID=%3u\n"
 757                         "\t%s aggregator ID=%3u\n",
 758                         port->aggregator_port_id,
 759                         port->aggregator_port_id == slave_id ?
 760                                 "aggregator not found, using default" : "aggregator found",
 761                         port->aggregator_port_id);
 762         }
 763
 764         port->selected = SELECTED;
 765 }
 766
 767 /* Function maps DPDK speed to bonding speed stored in key field */
 768 static uint16_t
 769 link_speed_key(uint16_t speed) {
 770         uint16_t key_speed;
 771
 772         switch (speed) {
 773         case ETH_SPEED_NUM_NONE:
 774                 key_speed = 0x00;
 775                 break;
 776         case ETH_SPEED_NUM_10M:
 777                 key_speed = BOND_LINK_SPEED_KEY_10M;
 778                 break;
 779         case ETH_SPEED_NUM_100M:
 780                 key_speed = BOND_LINK_SPEED_KEY_100M;
 781                 break;
 782         case ETH_SPEED_NUM_1G:
 783                 key_speed = BOND_LINK_SPEED_KEY_1000M;
 784                 break;
 785         case ETH_SPEED_NUM_10G:
 786                 key_speed = BOND_LINK_SPEED_KEY_10G;
 787                 break;
 788         case ETH_SPEED_NUM_20G:
 789                 key_speed = BOND_LINK_SPEED_KEY_20G;
 790                 break;
 791         case ETH_SPEED_NUM_40G:
 792                 key_speed = BOND_LINK_SPEED_KEY_40G;
 793                 break;
 794         default:
 795                 /* Unknown speed*/
 796                 key_speed = 0xFFFF;
 797         }
 798
 799         return key_speed;
 800 }
 801
 802 static void
 803 rx_machine_update(struct bond_dev_private *internals, uint16_t slave_id,
 804                 struct rte_mbuf *lacp_pkt) {
 805         struct lacpdu_header *lacp;
 806         struct lacpdu_actor_partner_params *partner;
 807
 808         if (lacp_pkt != NULL) {
 809                 lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 810                 RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
 811
 812                 partner = &lacp->lacpdu.partner;
 813                 if (rte_is_zero_ether_addr(&partner->port_params.system) ||
 814                         rte_is_same_ether_addr(&partner->port_params.system,
 815                         &internals->mode4.mac_addr)) {
 816                         /* This LACP frame is sending to the bonding port
 817                          * so pass it to rx_machine.
 818                          */
 819                         rx_machine(internals, slave_id, &lacp->lacpdu);
 820                 }
 821                 rte_pktmbuf_free(lacp_pkt);
 822         } else
 823                 rx_machine(internals, slave_id, NULL);
 824 }
 825
 826 static void
 827 bond_mode_8023ad_periodic_cb(void *arg)
 828 {
 829         struct rte_eth_dev *bond_dev = arg;
 830         struct bond_dev_private *internals = bond_dev->data->dev_private;
 831         struct port *port;
 832         struct rte_eth_link link_info;
 833         struct rte_ether_addr slave_addr;
 834         struct rte_mbuf *lacp_pkt = NULL;
 835         uint16_t slave_id;
 836         uint16_t i;
 837
 838
 839         /* Update link status on each port */
 840         for (i = 0; i < internals->active_slave_count; i++) {
 841                 uint16_t key;
 842                 int ret;
 843
 844                 slave_id = internals->active_slaves[i];
 845                 ret = rte_eth_link_get_nowait(slave_id, &link_info);
 846                 if (ret < 0) {
 847                         RTE_BOND_LOG(ERR,
 848                                 "Slave (port %u) link get failed: %s\n",
 849                                 slave_id, rte_strerror(-ret));
 850                 }
 851
 852                 if (ret >= 0 && link_info.link_status != 0) {
 853                         key = link_speed_key(link_info.link_speed) << 1;
 854                         if (link_info.link_duplex == ETH_LINK_FULL_DUPLEX)
 855                                 key |= BOND_LINK_FULL_DUPLEX_KEY;
 856                 } else {
 857                         key = 0;
 858                 }
 859
 860                 rte_eth_macaddr_get(slave_id, &slave_addr);
 861                 port = &bond_mode_8023ad_ports[slave_id];
 862
 863                 key = rte_cpu_to_be_16(key);
 864                 if (key != port->actor.key) {
 865                         if (!(key & rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)))
 866                                 set_warning_flags(port, WRN_NOT_LACP_CAPABLE);
 867
 868                         port->actor.key = key;
 869                         SM_FLAG_SET(port, NTT);
 870                 }
 871
 872                 if (!rte_is_same_ether_addr(&port->actor.system, &slave_addr)) {
 873                         rte_ether_addr_copy(&slave_addr, &port->actor.system);
 874                         if (port->aggregator_port_id == slave_id)
 875                                 SM_FLAG_SET(port, NTT);
 876                 }
 877         }
 878
 879         for (i = 0; i < internals->active_slave_count; i++) {
 880                 slave_id = internals->active_slaves[i];
 881                 port = &bond_mode_8023ad_ports[slave_id];
 882
 883                 if ((port->actor.key &
 884                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) == 0) {
 885
 886                         SM_FLAG_SET(port, BEGIN);
 887
 888                         /* LACP is disabled on half duples or link is down */
 889                         if (SM_FLAG(port, LACP_ENABLED)) {
 890                                 /* If port was enabled set it to BEGIN state */
 891                                 SM_FLAG_CLR(port, LACP_ENABLED);
 892                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 893                                 ACTOR_STATE_CLR(port, COLLECTING);
 894                         }
 895
 896                         /* Skip this port processing */
 897                         continue;
 898                 }
 899
 900                 SM_FLAG_SET(port, LACP_ENABLED);
 901
 902                 if (internals->mode4.dedicated_queues.enabled == 0) {
 903                         /* Find LACP packet to this port. Do not check subtype,
 904                          * it is done in function that queued packet
 905                          */
 906                         int retval = rte_ring_dequeue(port->rx_ring,
 907                                         (void **)&lacp_pkt);
 908
 909                         if (retval != 0)
 910                                 lacp_pkt = NULL;
 911
 912                         rx_machine_update(internals, slave_id, lacp_pkt);
 913                 } else {
 914                         uint16_t rx_count = rte_eth_rx_burst(slave_id,
 915                                         internals->mode4.dedicated_queues.rx_qid,
 916                                         &lacp_pkt, 1);
 917
 918                         if (rx_count == 1)
 919                                 bond_mode_8023ad_handle_slow_pkt(internals,
 920                                                 slave_id, lacp_pkt);
 921                         else
 922                                 rx_machine_update(internals, slave_id, NULL);
 923                 }
 924
 925                 periodic_machine(internals, slave_id);
 926                 mux_machine(internals, slave_id);
 927                 tx_machine(internals, slave_id);
 928                 selection_logic(internals, slave_id);
 929
 930                 SM_FLAG_CLR(port, BEGIN);
 931                 show_warnings(slave_id);
 932         }
 933
 934         rte_eal_alarm_set(internals->mode4.update_timeout_us,
 935                         bond_mode_8023ad_periodic_cb, arg);
 936 }
 937
 938 static int
 939 bond_mode_8023ad_register_lacp_mac(uint16_t slave_id)
 940 {
 941         int ret;
 942
 943         ret = rte_eth_allmulticast_enable(slave_id);
 944         if (ret != 0) {
 945                 RTE_BOND_LOG(ERR,
 946                         "failed to enable allmulti mode for port %u: %s",
 947                         slave_id, rte_strerror(-ret));
 948         }
 949         if (rte_eth_allmulticast_get(slave_id)) {
 950                 RTE_BOND_LOG(DEBUG, "forced allmulti for port %u",
 951                              slave_id);
 952                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 953                                 BOND_8023AD_FORCED_ALLMULTI;
 954                 return 0;
 955         }
 956
 957         ret = rte_eth_promiscuous_enable(slave_id);
 958         if (ret != 0) {
 959                 RTE_BOND_LOG(ERR,
 960                         "failed to enable promiscuous mode for port %u: %s",
 961                         slave_id, rte_strerror(-ret));
 962         }
 963         if (rte_eth_promiscuous_get(slave_id)) {
 964                 RTE_BOND_LOG(DEBUG, "forced promiscuous for port %u",
 965                              slave_id);
 966                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 967                                 BOND_8023AD_FORCED_PROMISC;
 968                 return 0;
 969         }
 970
 971         return -1;
 972 }
 973
 974 static void
 975 bond_mode_8023ad_unregister_lacp_mac(uint16_t slave_id)
 976 {
 977         int ret;
 978
 979         switch (bond_mode_8023ad_ports[slave_id].forced_rx_flags) {
 980         case BOND_8023AD_FORCED_ALLMULTI:
 981                 RTE_BOND_LOG(DEBUG, "unset allmulti for port %u", slave_id);
 982                 ret = rte_eth_allmulticast_disable(slave_id);
 983                 if (ret != 0)
 984                         RTE_BOND_LOG(ERR,
 985                                 "failed to disable allmulti mode for port %u: %s",
 986                                 slave_id, rte_strerror(-ret));
 987                 break;
 988
 989         case BOND_8023AD_FORCED_PROMISC:
 990                 RTE_BOND_LOG(DEBUG, "unset promisc for port %u", slave_id);
 991                 ret = rte_eth_promiscuous_disable(slave_id);
 992                 if (ret != 0)
 993                         RTE_BOND_LOG(ERR,
 994                                 "failed to disable promiscuous mode for port %u: %s",
 995                                 slave_id, rte_strerror(-ret));
 996                 break;
 997
 998         default:
 999                 break;
1000         }
1001 }
1002
1003 void
1004 bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev,
1005                                 uint16_t slave_id)
1006 {
1007         struct bond_dev_private *internals = bond_dev->data->dev_private;
1008
1009         struct port *port = &bond_mode_8023ad_ports[slave_id];
1010         struct port_params initial = {
1011                         .system = { { 0 } },
1012                         .system_priority = rte_cpu_to_be_16(0xFFFF),
1013                         .key = rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY),
1014                         .port_priority = rte_cpu_to_be_16(0x00FF),
1015                         .port_number = 0,
1016         };
1017
1018         char mem_name[RTE_ETH_NAME_MAX_LEN];
1019         int socket_id;
1020         unsigned element_size;
1021         uint32_t total_tx_desc;
1022         struct bond_tx_queue *bd_tx_q;
1023         uint16_t q_id;
1024
1025         /* Given slave mus not be in active list */
1026         RTE_ASSERT(find_slave_by_id(internals->active_slaves,
1027         internals->active_slave_count, slave_id) == internals->active_slave_count);
1028         RTE_SET_USED(internals); /* used only for assert when enabled */
1029
1030         memcpy(&port->actor, &initial, sizeof(struct port_params));
1031         /* Standard requires that port ID must be grater than 0.
1032          * Add 1 do get corresponding port_number */
1033         port->actor.port_number = rte_cpu_to_be_16(slave_id + 1);
1034
1035         memcpy(&port->partner, &initial, sizeof(struct port_params));
1036         memcpy(&port->partner_admin, &initial, sizeof(struct port_params));
1037
1038         /* default states */
1039         port->actor_state = STATE_AGGREGATION | STATE_LACP_ACTIVE | STATE_DEFAULTED;
1040         port->partner_state = STATE_LACP_ACTIVE | STATE_AGGREGATION;
1041         port->sm_flags = SM_FLAGS_BEGIN;
1042
1043         /* use this port as agregator */
1044         port->aggregator_port_id = slave_id;
1045
1046         if (bond_mode_8023ad_register_lacp_mac(slave_id) < 0) {
1047                 RTE_BOND_LOG(WARNING, "slave %u is most likely broken and won't receive LACP packets",
1048                              slave_id);
1049         }
1050
1051         timer_cancel(&port->warning_timer);
1052
1053         if (port->mbuf_pool != NULL)
1054                 return;
1055
1056         RTE_ASSERT(port->rx_ring == NULL);
1057         RTE_ASSERT(port->tx_ring == NULL);
1058
1059         socket_id = rte_eth_dev_socket_id(slave_id);
1060         if (socket_id == -1)
1061                 socket_id = rte_socket_id();
1062
1063         element_size = sizeof(struct slow_protocol_frame) +
1064                                 RTE_PKTMBUF_HEADROOM;
1065
1066         /* The size of the mempool should be at least:
1067          * the sum of the TX descriptors + BOND_MODE_8023AX_SLAVE_TX_PKTS */
1068         total_tx_desc = BOND_MODE_8023AX_SLAVE_TX_PKTS;
1069         for (q_id = 0; q_id < bond_dev->data->nb_tx_queues; q_id++) {
1070                 bd_tx_q = (struct bond_tx_queue*)bond_dev->data->tx_queues[q_id];
1071                 total_tx_desc += bd_tx_q->nb_tx_desc;
1072         }
1073
1074         snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_pool", slave_id);
1075         port->mbuf_pool = rte_pktmbuf_pool_create(mem_name, total_tx_desc,
1076                 RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ?
1077                         32 : RTE_MEMPOOL_CACHE_MAX_SIZE,
1078                 0, element_size, socket_id);
1079
1080         /* Any memory allocation failure in initialization is critical because
1081          * resources can't be free, so reinitialization is impossible. */
1082         if (port->mbuf_pool == NULL) {
1083                 rte_panic("Slave %u: Failed to create memory pool '%s': %s\n",
1084                         slave_id, mem_name, rte_strerror(rte_errno));
1085         }
1086
1087         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_rx", slave_id);
1088         port->rx_ring = rte_ring_create(mem_name,
1089                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_RX_PKTS), socket_id, 0);
1090
1091         if (port->rx_ring == NULL) {
1092                 rte_panic("Slave %u: Failed to create rx ring '%s': %s\n", slave_id,
1093                         mem_name, rte_strerror(rte_errno));
1094         }
1095
1096         /* TX ring is at least one pkt longer to make room for marker packet. */
1097         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_tx", slave_id);
1098         port->tx_ring = rte_ring_create(mem_name,
1099                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_TX_PKTS + 1), socket_id, 0);
1100
1101         if (port->tx_ring == NULL) {
1102                 rte_panic("Slave %u: Failed to create tx ring '%s': %s\n", slave_id,
1103                         mem_name, rte_strerror(rte_errno));
1104         }
1105 }
1106
1107 int
1108 bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev __rte_unused,
1109                 uint16_t slave_id)
1110 {
1111         void *pkt = NULL;
1112         struct port *port = NULL;
1113         uint8_t old_partner_state;
1114
1115         port = &bond_mode_8023ad_ports[slave_id];
1116
1117         ACTOR_STATE_CLR(port, AGGREGATION);
1118         port->selected = UNSELECTED;
1119
1120         old_partner_state = port->partner_state;
1121         record_default(port);
1122
1123         bond_mode_8023ad_unregister_lacp_mac(slave_id);
1124
1125         /* If partner timeout state changes then disable timer */
1126         if (!((old_partner_state ^ port->partner_state) &
1127                         STATE_LACP_SHORT_TIMEOUT))
1128                 timer_cancel(&port->current_while_timer);
1129
1130         PARTNER_STATE_CLR(port, AGGREGATION);
1131         ACTOR_STATE_CLR(port, EXPIRED);
1132
1133         /* flush rx/tx rings */
1134         while (rte_ring_dequeue(port->rx_ring, &pkt) == 0)
1135                 rte_pktmbuf_free((struct rte_mbuf *)pkt);
1136
1137         while (rte_ring_dequeue(port->tx_ring, &pkt) == 0)
1138                         rte_pktmbuf_free((struct rte_mbuf *)pkt);
1139         return 0;
1140 }
1141
1142 void
1143 bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev)
1144 {
1145         struct bond_dev_private *internals = bond_dev->data->dev_private;
1146         struct rte_ether_addr slave_addr;
1147         struct port *slave, *agg_slave;
1148         uint16_t slave_id, i, j;
1149
1150         bond_mode_8023ad_stop(bond_dev);
1151
1152         for (i = 0; i < internals->active_slave_count; i++) {
1153                 slave_id = internals->active_slaves[i];
1154                 slave = &bond_mode_8023ad_ports[slave_id];
1155                 rte_eth_macaddr_get(slave_id, &slave_addr);
1156
1157                 if (rte_is_same_ether_addr(&slave_addr, &slave->actor.system))
1158                         continue;
1159
1160                 rte_ether_addr_copy(&slave_addr, &slave->actor.system);
1161                 /* Do nothing if this port is not an aggregator. In other case
1162                  * Set NTT flag on every port that use this aggregator. */
1163                 if (slave->aggregator_port_id != slave_id)
1164                         continue;
1165
1166                 for (j = 0; j < internals->active_slave_count; j++) {
1167                         agg_slave = &bond_mode_8023ad_ports[internals->active_slaves[j]];
1168                         if (agg_slave->aggregator_port_id == slave_id)
1169                                 SM_FLAG_SET(agg_slave, NTT);
1170                 }
1171         }
1172
1173         if (bond_dev->data->dev_started)
1174                 bond_mode_8023ad_start(bond_dev);
1175 }
1176
1177 static void
1178 bond_mode_8023ad_conf_get(struct rte_eth_dev *dev,
1179                 struct rte_eth_bond_8023ad_conf *conf)
1180 {
1181         struct bond_dev_private *internals = dev->data->dev_private;
1182         struct mode8023ad_private *mode4 = &internals->mode4;
1183         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1184
1185         conf->fast_periodic_ms = mode4->fast_periodic_timeout / ms_ticks;
1186         conf->slow_periodic_ms = mode4->slow_periodic_timeout / ms_ticks;
1187         conf->short_timeout_ms = mode4->short_timeout / ms_ticks;
1188         conf->long_timeout_ms = mode4->long_timeout / ms_ticks;
1189         conf->aggregate_wait_timeout_ms = mode4->aggregate_wait_timeout / ms_ticks;
1190         conf->tx_period_ms = mode4->tx_period_timeout / ms_ticks;
1191         conf->update_timeout_ms = mode4->update_timeout_us / 1000;
1192         conf->rx_marker_period_ms = mode4->rx_marker_timeout / ms_ticks;
1193         conf->slowrx_cb = mode4->slowrx_cb;
1194         conf->agg_selection = mode4->agg_selection;
1195 }
1196
1197 static void
1198 bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf)
1199 {
1200         conf->fast_periodic_ms = BOND_8023AD_FAST_PERIODIC_MS;
1201         conf->slow_periodic_ms = BOND_8023AD_SLOW_PERIODIC_MS;
1202         conf->short_timeout_ms = BOND_8023AD_SHORT_TIMEOUT_MS;
1203         conf->long_timeout_ms = BOND_8023AD_LONG_TIMEOUT_MS;
1204         conf->aggregate_wait_timeout_ms = BOND_8023AD_AGGREGATE_WAIT_TIMEOUT_MS;
1205         conf->tx_period_ms = BOND_8023AD_TX_MACHINE_PERIOD_MS;
1206         conf->rx_marker_period_ms = BOND_8023AD_RX_MARKER_PERIOD_MS;
1207         conf->update_timeout_ms = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS;
1208         conf->slowrx_cb = NULL;
1209         conf->agg_selection = AGG_STABLE;
1210 }
1211
1212 static void
1213 bond_mode_8023ad_conf_assign(struct mode8023ad_private *mode4,
1214                 struct rte_eth_bond_8023ad_conf *conf)
1215 {
1216         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1217
1218         mode4->fast_periodic_timeout = conf->fast_periodic_ms * ms_ticks;
1219         mode4->slow_periodic_timeout = conf->slow_periodic_ms * ms_ticks;
1220         mode4->short_timeout = conf->short_timeout_ms * ms_ticks;
1221         mode4->long_timeout = conf->long_timeout_ms * ms_ticks;
1222         mode4->aggregate_wait_timeout = conf->aggregate_wait_timeout_ms * ms_ticks;
1223         mode4->tx_period_timeout = conf->tx_period_ms * ms_ticks;
1224         mode4->rx_marker_timeout = conf->rx_marker_period_ms * ms_ticks;
1225         mode4->update_timeout_us = conf->update_timeout_ms * 1000;
1226
1227         mode4->dedicated_queues.enabled = 0;
1228         mode4->dedicated_queues.rx_qid = UINT16_MAX;
1229         mode4->dedicated_queues.tx_qid = UINT16_MAX;
1230 }
1231
1232 void
1233 bond_mode_8023ad_setup(struct rte_eth_dev *dev,
1234                 struct rte_eth_bond_8023ad_conf *conf)
1235 {
1236         struct rte_eth_bond_8023ad_conf def_conf;
1237         struct bond_dev_private *internals = dev->data->dev_private;
1238         struct mode8023ad_private *mode4 = &internals->mode4;
1239
1240         if (conf == NULL) {
1241                 conf = &def_conf;
1242                 bond_mode_8023ad_conf_get_default(conf);
1243         }
1244
1245         bond_mode_8023ad_stop(dev);
1246         bond_mode_8023ad_conf_assign(mode4, conf);
1247         mode4->slowrx_cb = conf->slowrx_cb;
1248         mode4->agg_selection = AGG_STABLE;
1249
1250         if (dev->data->dev_started)
1251                 bond_mode_8023ad_start(dev);
1252 }
1253
1254 int
1255 bond_mode_8023ad_enable(struct rte_eth_dev *bond_dev)
1256 {
1257         struct bond_dev_private *internals = bond_dev->data->dev_private;
1258         uint16_t i;
1259
1260         for (i = 0; i < internals->active_slave_count; i++)
1261                 bond_mode_8023ad_activate_slave(bond_dev,
1262                                 internals->active_slaves[i]);
1263
1264         return 0;
1265 }
1266
1267 int
1268 bond_mode_8023ad_start(struct rte_eth_dev *bond_dev)
1269 {
1270         struct bond_dev_private *internals = bond_dev->data->dev_private;
1271         struct mode8023ad_private *mode4 = &internals->mode4;
1272         static const uint64_t us = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS * 1000;
1273
1274         rte_eth_macaddr_get(internals->port_id, &mode4->mac_addr);
1275         if (mode4->slowrx_cb)
1276                 return rte_eal_alarm_set(us, &bond_mode_8023ad_ext_periodic_cb,
1277                                          bond_dev);
1278
1279         return rte_eal_alarm_set(us, &bond_mode_8023ad_periodic_cb, bond_dev);
1280 }
1281
1282 void
1283 bond_mode_8023ad_stop(struct rte_eth_dev *bond_dev)
1284 {
1285         struct bond_dev_private *internals = bond_dev->data->dev_private;
1286         struct mode8023ad_private *mode4 = &internals->mode4;
1287
1288         if (mode4->slowrx_cb) {
1289                 rte_eal_alarm_cancel(&bond_mode_8023ad_ext_periodic_cb,
1290                                      bond_dev);
1291                 return;
1292         }
1293         rte_eal_alarm_cancel(&bond_mode_8023ad_periodic_cb, bond_dev);
1294 }
1295
1296 void
1297 bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
1298                                   uint16_t slave_id, struct rte_mbuf *pkt)
1299 {
1300         struct mode8023ad_private *mode4 = &internals->mode4;
1301         struct port *port = &bond_mode_8023ad_ports[slave_id];
1302         struct marker_header *m_hdr;
1303         uint64_t marker_timer, old_marker_timer;
1304         int retval;
1305         uint8_t wrn, subtype;
1306         /* If packet is a marker, we send response now by reusing given packet
1307          * and update only source MAC, destination MAC is multicast so don't
1308          * update it. Other frames will be handled later by state machines */
1309         subtype = rte_pktmbuf_mtod(pkt,
1310                         struct slow_protocol_frame *)->slow_protocol.subtype;
1311
1312         if (subtype == SLOW_SUBTYPE_MARKER) {
1313                 m_hdr = rte_pktmbuf_mtod(pkt, struct marker_header *);
1314
1315                 if (likely(m_hdr->marker.tlv_type_marker != MARKER_TLV_TYPE_INFO)) {
1316                         wrn = WRN_UNKNOWN_MARKER_TYPE;
1317                         goto free_out;
1318                 }
1319
1320                 /* Setup marker timer. Do it in loop in case concurrent access. */
1321                 do {
1322                         old_marker_timer = port->rx_marker_timer;
1323                         if (!timer_is_expired(&old_marker_timer)) {
1324                                 wrn = WRN_RX_MARKER_TO_FAST;
1325                                 goto free_out;
1326                         }
1327
1328                         timer_set(&marker_timer, mode4->rx_marker_timeout);
1329                         retval = rte_atomic64_cmpset(&port->rx_marker_timer,
1330                                 old_marker_timer, marker_timer);
1331                 } while (unlikely(retval == 0));
1332
1333                 m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP;
1334                 rte_eth_macaddr_get(slave_id, &m_hdr->eth_hdr.s_addr);
1335
1336                 if (internals->mode4.dedicated_queues.enabled == 0) {
1337                         int retval = rte_ring_enqueue(port->tx_ring, pkt);
1338                         if (retval != 0) {
1339                                 /* reset timer */
1340                                 port->rx_marker_timer = 0;
1341                                 wrn = WRN_TX_QUEUE_FULL;
1342                                 goto free_out;
1343                         }
1344                 } else {
1345                         /* Send packet directly to the slow queue */
1346                         uint16_t tx_count = rte_eth_tx_burst(slave_id,
1347                                         internals->mode4.dedicated_queues.tx_qid,
1348                                         &pkt, 1);
1349                         if (tx_count != 1) {
1350                                 /* reset timer */
1351                                 port->rx_marker_timer = 0;
1352                                 wrn = WRN_TX_QUEUE_FULL;
1353                                 goto free_out;
1354                         }
1355                 }
1356         } else if (likely(subtype == SLOW_SUBTYPE_LACP)) {
1357                 if (internals->mode4.dedicated_queues.enabled == 0) {
1358                         int retval = rte_ring_enqueue(port->rx_ring, pkt);
1359                         if (retval != 0) {
1360                                 /* If RX fing full free lacpdu message and drop packet */
1361                                 wrn = WRN_RX_QUEUE_FULL;
1362                                 goto free_out;
1363                         }
1364                 } else
1365                         rx_machine_update(internals, slave_id, pkt);
1366         } else {
1367                 wrn = WRN_UNKNOWN_SLOW_TYPE;
1368                 goto free_out;
1369         }
1370
1371         return;
1372
1373 free_out:
1374         set_warning_flags(port, wrn);
1375         rte_pktmbuf_free(pkt);
1376 }
1377
1378 int
1379 rte_eth_bond_8023ad_conf_get(uint16_t port_id,
1380                 struct rte_eth_bond_8023ad_conf *conf)
1381 {
1382         struct rte_eth_dev *bond_dev;
1383
1384         if (valid_bonded_port_id(port_id) != 0)
1385                 return -EINVAL;
1386
1387         if (conf == NULL)
1388                 return -EINVAL;
1389
1390         bond_dev = &rte_eth_devices[port_id];
1391         bond_mode_8023ad_conf_get(bond_dev, conf);
1392         return 0;
1393 }
1394
1395 int
1396 rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id,
1397                 enum rte_bond_8023ad_agg_selection agg_selection)
1398 {
1399         struct rte_eth_dev *bond_dev;
1400         struct bond_dev_private *internals;
1401         struct mode8023ad_private *mode4;
1402
1403         if (valid_bonded_port_id(port_id) != 0)
1404                 return -EINVAL;
1405
1406         bond_dev = &rte_eth_devices[port_id];
1407         internals = bond_dev->data->dev_private;
1408
1409         if (internals->mode != 4)
1410                 return -EINVAL;
1411
1412         mode4 = &internals->mode4;
1413         if (agg_selection == AGG_COUNT || agg_selection == AGG_BANDWIDTH
1414                         || agg_selection == AGG_STABLE)
1415                 mode4->agg_selection = agg_selection;
1416         return 0;
1417 }
1418
1419 int rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id)
1420 {
1421         struct rte_eth_dev *bond_dev;
1422         struct bond_dev_private *internals;
1423         struct mode8023ad_private *mode4;
1424
1425         if (valid_bonded_port_id(port_id) != 0)
1426                 return -EINVAL;
1427
1428         bond_dev = &rte_eth_devices[port_id];
1429         internals = bond_dev->data->dev_private;
1430
1431         if (internals->mode != 4)
1432                 return -EINVAL;
1433         mode4 = &internals->mode4;
1434
1435         return mode4->agg_selection;
1436 }
1437
1438
1439
1440 static int
1441 bond_8023ad_setup_validate(uint16_t port_id,
1442                 struct rte_eth_bond_8023ad_conf *conf)
1443 {
1444         if (valid_bonded_port_id(port_id) != 0)
1445                 return -EINVAL;
1446
1447         if (conf != NULL) {
1448                 /* Basic sanity check */
1449                 if (conf->slow_periodic_ms == 0 ||
1450                                 conf->fast_periodic_ms >= conf->slow_periodic_ms ||
1451                                 conf->long_timeout_ms == 0 ||
1452                                 conf->short_timeout_ms >= conf->long_timeout_ms ||
1453                                 conf->aggregate_wait_timeout_ms == 0 ||
1454                                 conf->tx_period_ms == 0 ||
1455                                 conf->rx_marker_period_ms == 0 ||
1456                                 conf->update_timeout_ms == 0) {
1457                         RTE_BOND_LOG(ERR, "given mode 4 configuration is invalid");
1458                         return -EINVAL;
1459                 }
1460         }
1461
1462         return 0;
1463 }
1464
1465
1466 int
1467 rte_eth_bond_8023ad_setup(uint16_t port_id,
1468                 struct rte_eth_bond_8023ad_conf *conf)
1469 {
1470         struct rte_eth_dev *bond_dev;
1471         int err;
1472
1473         err = bond_8023ad_setup_validate(port_id, conf);
1474         if (err != 0)
1475                 return err;
1476
1477         bond_dev = &rte_eth_devices[port_id];
1478         bond_mode_8023ad_setup(bond_dev, conf);
1479
1480         return 0;
1481 }
1482
1483
1484
1485
1486
1487 int
1488 rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id,
1489                 struct rte_eth_bond_8023ad_slave_info *info)
1490 {
1491         struct rte_eth_dev *bond_dev;
1492         struct bond_dev_private *internals;
1493         struct port *port;
1494
1495         if (info == NULL || valid_bonded_port_id(port_id) != 0 ||
1496                         rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1497                 return -EINVAL;
1498
1499         bond_dev = &rte_eth_devices[port_id];
1500
1501         internals = bond_dev->data->dev_private;
1502         if (find_slave_by_id(internals->active_slaves,
1503                         internals->active_slave_count, slave_id) ==
1504                                 internals->active_slave_count)
1505                 return -EINVAL;
1506
1507         port = &bond_mode_8023ad_ports[slave_id];
1508         info->selected = port->selected;
1509
1510         info->actor_state = port->actor_state;
1511         rte_memcpy(&info->actor, &port->actor, sizeof(port->actor));
1512
1513         info->partner_state = port->partner_state;
1514         rte_memcpy(&info->partner, &port->partner, sizeof(port->partner));
1515
1516         info->agg_port_id = port->aggregator_port_id;
1517         return 0;
1518 }
1519
1520 static int
1521 bond_8023ad_ext_validate(uint16_t port_id, uint16_t slave_id)
1522 {
1523         struct rte_eth_dev *bond_dev;
1524         struct bond_dev_private *internals;
1525         struct mode8023ad_private *mode4;
1526
1527         if (rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1528                 return -EINVAL;
1529
1530         bond_dev = &rte_eth_devices[port_id];
1531
1532         if (!bond_dev->data->dev_started)
1533                 return -EINVAL;
1534
1535         internals = bond_dev->data->dev_private;
1536         if (find_slave_by_id(internals->active_slaves,
1537                         internals->active_slave_count, slave_id) ==
1538                                 internals->active_slave_count)
1539                 return -EINVAL;
1540
1541         mode4 = &internals->mode4;
1542         if (mode4->slowrx_cb == NULL)
1543                 return -EINVAL;
1544
1545         return 0;
1546 }
1547
1548 int
1549 rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id,
1550                                 int enabled)
1551 {
1552         struct port *port;
1553         int res;
1554
1555         res = bond_8023ad_ext_validate(port_id, slave_id);
1556         if (res != 0)
1557                 return res;
1558
1559         port = &bond_mode_8023ad_ports[slave_id];
1560
1561         if (enabled)
1562                 ACTOR_STATE_SET(port, COLLECTING);
1563         else
1564                 ACTOR_STATE_CLR(port, COLLECTING);
1565
1566         return 0;
1567 }
1568
1569 int
1570 rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id,
1571                                 int enabled)
1572 {
1573         struct port *port;
1574         int res;
1575
1576         res = bond_8023ad_ext_validate(port_id, slave_id);
1577         if (res != 0)
1578                 return res;
1579
1580         port = &bond_mode_8023ad_ports[slave_id];
1581
1582         if (enabled)
1583                 ACTOR_STATE_SET(port, DISTRIBUTING);
1584         else
1585                 ACTOR_STATE_CLR(port, DISTRIBUTING);
1586
1587         return 0;
1588 }
1589
1590 int
1591 rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id)
1592 {
1593         struct port *port;
1594         int err;
1595
1596         err = bond_8023ad_ext_validate(port_id, slave_id);
1597         if (err != 0)
1598                 return err;
1599
1600         port = &bond_mode_8023ad_ports[slave_id];
1601         return ACTOR_STATE(port, DISTRIBUTING);
1602 }
1603
1604 int
1605 rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id)
1606 {
1607         struct port *port;
1608         int err;
1609
1610         err = bond_8023ad_ext_validate(port_id, slave_id);
1611         if (err != 0)
1612                 return err;
1613
1614         port = &bond_mode_8023ad_ports[slave_id];
1615         return ACTOR_STATE(port, COLLECTING);
1616 }
1617
1618 int
1619 rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id,
1620                 struct rte_mbuf *lacp_pkt)
1621 {
1622         struct port *port;
1623         int res;
1624
1625         res = bond_8023ad_ext_validate(port_id, slave_id);
1626         if (res != 0)
1627                 return res;
1628
1629         port = &bond_mode_8023ad_ports[slave_id];
1630
1631         if (rte_pktmbuf_pkt_len(lacp_pkt) < sizeof(struct lacpdu_header))
1632                 return -EINVAL;
1633
1634         struct lacpdu_header *lacp;
1635
1636         /* only enqueue LACPDUs */
1637         lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
1638         if (lacp->lacpdu.subtype != SLOW_SUBTYPE_LACP)
1639                 return -EINVAL;
1640
1641         MODE4_DEBUG("sending LACP frame\n");
1642
1643         return rte_ring_enqueue(port->tx_ring, lacp_pkt);
1644 }
1645
1646 static void
1647 bond_mode_8023ad_ext_periodic_cb(void *arg)
1648 {
1649         struct rte_eth_dev *bond_dev = arg;
1650         struct bond_dev_private *internals = bond_dev->data->dev_private;
1651         struct mode8023ad_private *mode4 = &internals->mode4;
1652         struct port *port;
1653         void *pkt = NULL;
1654         uint16_t i, slave_id;
1655
1656         for (i = 0; i < internals->active_slave_count; i++) {
1657                 slave_id = internals->active_slaves[i];
1658                 port = &bond_mode_8023ad_ports[slave_id];
1659
1660                 if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) {
1661                         struct rte_mbuf *lacp_pkt = pkt;
1662                         struct lacpdu_header *lacp;
1663
1664                         lacp = rte_pktmbuf_mtod(lacp_pkt,
1665                                                 struct lacpdu_header *);
1666                         RTE_VERIFY(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
1667
1668                         /* This is LACP frame so pass it to rx callback.
1669                          * Callback is responsible for freeing mbuf.
1670                          */
1671                         mode4->slowrx_cb(slave_id, lacp_pkt);
1672                 }
1673         }
1674
1675         rte_eal_alarm_set(internals->mode4.update_timeout_us,
1676                         bond_mode_8023ad_ext_periodic_cb, arg);
1677 }
1678
1679 int
1680 rte_eth_bond_8023ad_dedicated_queues_enable(uint16_t port)
1681 {
1682         int retval = 0;
1683         struct rte_eth_dev *dev;
1684         struct bond_dev_private *internals;
1685
1686         if (valid_bonded_port_id(port) != 0)
1687                 return -EINVAL;
1688
1689         dev = &rte_eth_devices[port];
1690         internals = dev->data->dev_private;
1691
1692         if (bond_8023ad_slow_pkt_hw_filter_supported(port) != 0)
1693                 return -1;
1694
1695         /* Device must be stopped to set up slow queue */
1696         if (dev->data->dev_started)
1697                 return -1;
1698
1699         internals->mode4.dedicated_queues.enabled = 1;
1700
1701         bond_ethdev_mode_set(dev, internals->mode);
1702         return retval;
1703 }
1704
1705 int
1706 rte_eth_bond_8023ad_dedicated_queues_disable(uint16_t port)
1707 {
1708         int retval = 0;
1709         struct rte_eth_dev *dev;
1710         struct bond_dev_private *internals;
1711
1712         if (valid_bonded_port_id(port) != 0)
1713                 return -EINVAL;
1714
1715         dev = &rte_eth_devices[port];
1716         internals = dev->data->dev_private;
1717
1718         /* Device must be stopped to set up slow queue */
1719         if (dev->data->dev_started)
1720                 return -1;
1721
1722         internals->mode4.dedicated_queues.enabled = 0;
1723
1724         bond_ethdev_mode_set(dev, internals->mode);
1725
1726         return retval;
1727 }