drivers/net/bonding/rte_eth_bond_8023ad.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <stddef.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_alarm.h>
  10 #include <rte_malloc.h>
  11 #include <rte_errno.h>
  12 #include <rte_cycles.h>
  13 #include <rte_compat.h>
  14
  15 #include "eth_bond_private.h"
  16
  17 static void bond_mode_8023ad_ext_periodic_cb(void *arg);
  18 #ifdef RTE_LIBRTE_BOND_DEBUG_8023AD
  19
  20 #define MODE4_DEBUG(fmt, ...)                           \
  21         rte_log(RTE_LOG_DEBUG, bond_logtype,            \
  22                 "%6u [Port %u: %s] " fmt,               \
  23                 bond_dbg_get_time_diff_ms(), slave_id,  \
  24                 __func__, ##__VA_ARGS__)
  25
  26 static uint64_t start_time;
  27
  28 static unsigned
  29 bond_dbg_get_time_diff_ms(void)
  30 {
  31         uint64_t now;
  32
  33         now = rte_rdtsc();
  34         if (start_time == 0)
  35                 start_time = now;
  36
  37         return ((now - start_time) * 1000) / rte_get_tsc_hz();
  38 }
  39
  40 static void
  41 bond_print_lacp(struct lacpdu *l)
  42 {
  43         char a_address[18];
  44         char p_address[18];
  45         char a_state[256] = { 0 };
  46         char p_state[256] = { 0 };
  47
  48         static const char * const state_labels[] = {
  49                 "ACT", "TIMEOUT", "AGG", "SYNC", "COL", "DIST", "DEF", "EXP"
  50         };
  51
  52         int a_len = 0;
  53         int p_len = 0;
  54         uint8_t i;
  55         uint8_t *addr;
  56
  57         addr = l->actor.port_params.system.addr_bytes;
  58         snprintf(a_address, sizeof(a_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  59                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  60
  61         addr = l->partner.port_params.system.addr_bytes;
  62         snprintf(p_address, sizeof(p_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  63                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  64
  65         for (i = 0; i < 8; i++) {
  66                 if ((l->actor.state >> i) & 1) {
  67                         a_len += snprintf(&a_state[a_len], RTE_DIM(a_state) - a_len, "%s ",
  68                                 state_labels[i]);
  69                 }
  70
  71                 if ((l->partner.state >> i) & 1) {
  72                         p_len += snprintf(&p_state[p_len], RTE_DIM(p_state) - p_len, "%s ",
  73                                 state_labels[i]);
  74                 }
  75         }
  76
  77         if (a_len && a_state[a_len-1] == ' ')
  78                 a_state[a_len-1] = '\0';
  79
  80         if (p_len && p_state[p_len-1] == ' ')
  81                 p_state[p_len-1] = '\0';
  82
  83         RTE_BOND_LOG(DEBUG,
  84                      "LACP: {\n"
  85                      "  subtype= %02X\n"
  86                      "  ver_num=%02X\n"
  87                      "  actor={ tlv=%02X, len=%02X\n"
  88                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  89                      "       state={ %s }\n"
  90                      "  }\n"
  91                      "  partner={ tlv=%02X, len=%02X\n"
  92                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  93                      "       state={ %s }\n"
  94                      "  }\n"
  95                      "  collector={info=%02X, length=%02X, max_delay=%04X\n, "
  96                      "type_term=%02X, terminator_length = %02X }",
  97                      l->subtype,
  98                      l->version_number,
  99                      l->actor.tlv_type_info,
 100                      l->actor.info_length,
 101                      l->actor.port_params.system_priority,
 102                      a_address,
 103                      l->actor.port_params.key,
 104                      l->actor.port_params.port_priority,
 105                      l->actor.port_params.port_number,
 106                      a_state,
 107                      l->partner.tlv_type_info,
 108                      l->partner.info_length,
 109                      l->partner.port_params.system_priority,
 110                      p_address,
 111                      l->partner.port_params.key,
 112                      l->partner.port_params.port_priority,
 113                      l->partner.port_params.port_number,
 114                      p_state,
 115                      l->tlv_type_collector_info,
 116                      l->collector_info_length,
 117                      l->collector_max_delay,
 118                      l->tlv_type_terminator,
 119                      l->terminator_length);
 120
 121 }
 122
 123 #define BOND_PRINT_LACP(lacpdu) bond_print_lacp(lacpdu)
 124 #else
 125 #define BOND_PRINT_LACP(lacpdu) do { } while (0)
 126 #define MODE4_DEBUG(fmt, ...) do { } while (0)
 127 #endif
 128
 129 static const struct rte_ether_addr lacp_mac_addr = {
 130         .addr_bytes = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 }
 131 };
 132
 133 struct port bond_mode_8023ad_ports[RTE_MAX_ETHPORTS];
 134
 135 static void
 136 timer_cancel(uint64_t *timer)
 137 {
 138         *timer = 0;
 139 }
 140
 141 static void
 142 timer_set(uint64_t *timer, uint64_t timeout)
 143 {
 144         *timer = rte_rdtsc() + timeout;
 145 }
 146
 147 /* Forces given timer to be in expired state. */
 148 static void
 149 timer_force_expired(uint64_t *timer)
 150 {
 151         *timer = rte_rdtsc();
 152 }
 153
 154 static bool
 155 timer_is_stopped(uint64_t *timer)
 156 {
 157         return *timer == 0;
 158 }
 159
 160 static bool
 161 timer_is_expired(uint64_t *timer)
 162 {
 163         return *timer < rte_rdtsc();
 164 }
 165
 166 /* Timer is in running state if it is not stopped nor expired */
 167 static bool
 168 timer_is_running(uint64_t *timer)
 169 {
 170         return !timer_is_stopped(timer) && !timer_is_expired(timer);
 171 }
 172
 173 static void
 174 set_warning_flags(struct port *port, uint16_t flags)
 175 {
 176         int retval;
 177         uint16_t old;
 178         uint16_t new_flag = 0;
 179
 180         do {
 181                 old = port->warnings_to_show;
 182                 new_flag = old | flags;
 183                 retval = rte_atomic16_cmpset(&port->warnings_to_show, old, new_flag);
 184         } while (unlikely(retval == 0));
 185 }
 186
 187 static void
 188 show_warnings(uint16_t slave_id)
 189 {
 190         struct port *port = &bond_mode_8023ad_ports[slave_id];
 191         uint8_t warnings;
 192
 193         do {
 194                 warnings = port->warnings_to_show;
 195         } while (rte_atomic16_cmpset(&port->warnings_to_show, warnings, 0) == 0);
 196
 197         if (!warnings)
 198                 return;
 199
 200         if (!timer_is_expired(&port->warning_timer))
 201                 return;
 202
 203
 204         timer_set(&port->warning_timer, BOND_8023AD_WARNINGS_PERIOD_MS *
 205                         rte_get_tsc_hz() / 1000);
 206
 207         if (warnings & WRN_RX_QUEUE_FULL) {
 208                 RTE_BOND_LOG(DEBUG,
 209                              "Slave %u: failed to enqueue LACP packet into RX ring.\n"
 210                              "Receive and transmit functions must be invoked on bonded"
 211                              "interface at least 10 times per second or LACP will notwork correctly",
 212                              slave_id);
 213         }
 214
 215         if (warnings & WRN_TX_QUEUE_FULL) {
 216                 RTE_BOND_LOG(DEBUG,
 217                              "Slave %u: failed to enqueue LACP packet into TX ring.\n"
 218                              "Receive and transmit functions must be invoked on bonded"
 219                              "interface at least 10 times per second or LACP will not work correctly",
 220                              slave_id);
 221         }
 222
 223         if (warnings & WRN_RX_MARKER_TO_FAST)
 224                 RTE_BOND_LOG(INFO, "Slave %u: marker to early - ignoring.",
 225                              slave_id);
 226
 227         if (warnings & WRN_UNKNOWN_SLOW_TYPE) {
 228                 RTE_BOND_LOG(INFO,
 229                         "Slave %u: ignoring unknown slow protocol frame type",
 230                              slave_id);
 231         }
 232
 233         if (warnings & WRN_UNKNOWN_MARKER_TYPE)
 234                 RTE_BOND_LOG(INFO, "Slave %u: ignoring unknown marker type",
 235                              slave_id);
 236
 237         if (warnings & WRN_NOT_LACP_CAPABLE)
 238                 MODE4_DEBUG("Port %u is not LACP capable!\n", slave_id);
 239 }
 240
 241 static void
 242 record_default(struct port *port)
 243 {
 244         /* Record default parameters for partner. Partner admin parameters
 245          * are not implemented so set them to arbitrary default (last known) and
 246          * mark actor that parner is in defaulted state. */
 247         port->partner_state = STATE_LACP_ACTIVE;
 248         ACTOR_STATE_SET(port, DEFAULTED);
 249 }
 250
 251 /** Function handles rx state machine.
 252  *
 253  * This function implements Receive State Machine from point 5.4.12 in
 254  * 802.1AX documentation. It should be called periodically.
 255  *
 256  * @param lacpdu                LACPDU received.
 257  * @param port                  Port on which LACPDU was received.
 258  */
 259 static void
 260 rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 261                 struct lacpdu *lacp)
 262 {
 263         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 264         uint64_t timeout;
 265
 266         if (SM_FLAG(port, BEGIN)) {
 267                 /* Initialize stuff */
 268                 MODE4_DEBUG("-> INITIALIZE\n");
 269                 SM_FLAG_CLR(port, MOVED);
 270                 port->selected = UNSELECTED;
 271
 272                 record_default(port);
 273
 274                 ACTOR_STATE_CLR(port, EXPIRED);
 275                 timer_cancel(&port->current_while_timer);
 276
 277                 /* DISABLED: On initialization partner is out of sync */
 278                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 279
 280                 /* LACP DISABLED stuff if LACP not enabled on this port */
 281                 if (!SM_FLAG(port, LACP_ENABLED))
 282                         PARTNER_STATE_CLR(port, AGGREGATION);
 283                 else
 284                         PARTNER_STATE_SET(port, AGGREGATION);
 285         }
 286
 287         if (!SM_FLAG(port, LACP_ENABLED)) {
 288                 /* Update parameters only if state changed */
 289                 if (!timer_is_stopped(&port->current_while_timer)) {
 290                         port->selected = UNSELECTED;
 291                         record_default(port);
 292                         PARTNER_STATE_CLR(port, AGGREGATION);
 293                         ACTOR_STATE_CLR(port, EXPIRED);
 294                         timer_cancel(&port->current_while_timer);
 295                 }
 296                 return;
 297         }
 298
 299         if (lacp) {
 300                 MODE4_DEBUG("LACP -> CURRENT\n");
 301                 BOND_PRINT_LACP(lacp);
 302                 /* Update selected flag. If partner parameters are defaulted assume they
 303                  * are match. If not defaulted  compare LACP actor with ports parner
 304                  * params. */
 305                 if (!ACTOR_STATE(port, DEFAULTED) &&
 306                         (ACTOR_STATE(port, AGGREGATION) != PARTNER_STATE(port, AGGREGATION)
 307                         || memcmp(&port->partner, &lacp->actor.port_params,
 308                                 sizeof(port->partner)) != 0)) {
 309                         MODE4_DEBUG("selected <- UNSELECTED\n");
 310                         port->selected = UNSELECTED;
 311                 }
 312
 313                 /* Record this PDU actor params as partner params */
 314                 memcpy(&port->partner, &lacp->actor.port_params,
 315                         sizeof(struct port_params));
 316                 port->partner_state = lacp->actor.state;
 317
 318                 /* Partner parameters are not defaulted any more */
 319                 ACTOR_STATE_CLR(port, DEFAULTED);
 320
 321                 /* If LACP partner params match this port actor params */
 322                 agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 323                 bool match = port->actor.system_priority ==
 324                         lacp->partner.port_params.system_priority &&
 325                         rte_is_same_ether_addr(&agg->actor.system,
 326                         &lacp->partner.port_params.system) &&
 327                         port->actor.port_priority ==
 328                         lacp->partner.port_params.port_priority &&
 329                         port->actor.port_number ==
 330                         lacp->partner.port_params.port_number;
 331
 332                 /* Update NTT if partners information are outdated (xored and masked
 333                  * bits are set)*/
 334                 uint8_t state_mask = STATE_LACP_ACTIVE | STATE_LACP_SHORT_TIMEOUT |
 335                         STATE_SYNCHRONIZATION | STATE_AGGREGATION;
 336
 337                 if (((port->actor_state ^ lacp->partner.state) & state_mask) ||
 338                                 match == false) {
 339                         SM_FLAG_SET(port, NTT);
 340                 }
 341
 342                 /* If LACP partner params match this port actor params */
 343                 if (match == true && ACTOR_STATE(port, AGGREGATION) ==
 344                                 PARTNER_STATE(port,     AGGREGATION))
 345                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 346                 else if (!PARTNER_STATE(port, AGGREGATION) && ACTOR_STATE(port,
 347                                 AGGREGATION))
 348                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 349                 else
 350                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 351
 352                 if (ACTOR_STATE(port, LACP_SHORT_TIMEOUT))
 353                         timeout = internals->mode4.short_timeout;
 354                 else
 355                         timeout = internals->mode4.long_timeout;
 356
 357                 timer_set(&port->current_while_timer, timeout);
 358                 ACTOR_STATE_CLR(port, EXPIRED);
 359                 SM_FLAG_CLR(port, EXPIRED);
 360                 return; /* No state change */
 361         }
 362
 363         /* If CURRENT state timer is not running (stopped or expired)
 364          * transit to EXPIRED state from DISABLED or CURRENT */
 365         if (!timer_is_running(&port->current_while_timer)) {
 366                 if (SM_FLAG(port, EXPIRED)) {
 367                         port->selected = UNSELECTED;
 368                         memcpy(&port->partner, &port->partner_admin,
 369                                 sizeof(struct port_params));
 370                         record_default(port);
 371                         ACTOR_STATE_CLR(port, EXPIRED);
 372                         timer_cancel(&port->current_while_timer);
 373                 } else {
 374                         SM_FLAG_SET(port, EXPIRED);
 375                         ACTOR_STATE_SET(port, EXPIRED);
 376                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 377                         PARTNER_STATE_SET(port, LACP_SHORT_TIMEOUT);
 378                         timer_set(&port->current_while_timer,
 379                                 internals->mode4.short_timeout);
 380                 }
 381         }
 382 }
 383
 384 /**
 385  * Function handles periodic tx state machine.
 386  *
 387  * Function implements Periodic Transmission state machine from point 5.4.13
 388  * in 802.1AX documentation. It should be called periodically.
 389  *
 390  * @param port                  Port to handle state machine.
 391  */
 392 static void
 393 periodic_machine(struct bond_dev_private *internals, uint16_t slave_id)
 394 {
 395         struct port *port = &bond_mode_8023ad_ports[slave_id];
 396         /* Calculate if either site is LACP enabled */
 397         uint64_t timeout;
 398         uint8_t active = ACTOR_STATE(port, LACP_ACTIVE) ||
 399                 PARTNER_STATE(port, LACP_ACTIVE);
 400
 401         uint8_t is_partner_fast, was_partner_fast;
 402         /* No periodic is on BEGIN, LACP DISABLE or when both sides are pasive */
 403         if (SM_FLAG(port, BEGIN) || !SM_FLAG(port, LACP_ENABLED) || !active) {
 404                 timer_cancel(&port->periodic_timer);
 405                 timer_force_expired(&port->tx_machine_timer);
 406                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 407
 408                 MODE4_DEBUG("-> NO_PERIODIC ( %s%s%s)\n",
 409                         SM_FLAG(port, BEGIN) ? "begind " : "",
 410                         SM_FLAG(port, LACP_ENABLED) ? "" : "LACP disabled ",
 411                         active ? "LACP active " : "LACP pasive ");
 412                 return;
 413         }
 414
 415         is_partner_fast = PARTNER_STATE(port, LACP_SHORT_TIMEOUT);
 416         was_partner_fast = SM_FLAG(port, PARTNER_SHORT_TIMEOUT);
 417
 418         /* If periodic timer is not started, transit from NO PERIODIC to FAST/SLOW.
 419          * Other case: check if timer expire or partners settings changed. */
 420         if (!timer_is_stopped(&port->periodic_timer)) {
 421                 if (timer_is_expired(&port->periodic_timer)) {
 422                         SM_FLAG_SET(port, NTT);
 423                 } else if (is_partner_fast != was_partner_fast) {
 424                         /* Partners timeout  was slow and now it is fast -> send LACP.
 425                          * In other case (was fast and now it is slow) just switch
 426                          * timeout to slow without forcing send of LACP (because standard
 427                          * say so)*/
 428                         if (is_partner_fast)
 429                                 SM_FLAG_SET(port, NTT);
 430                 } else
 431                         return; /* Nothing changed */
 432         }
 433
 434         /* Handle state transition to FAST/SLOW LACP timeout */
 435         if (is_partner_fast) {
 436                 timeout = internals->mode4.fast_periodic_timeout;
 437                 SM_FLAG_SET(port, PARTNER_SHORT_TIMEOUT);
 438         } else {
 439                 timeout = internals->mode4.slow_periodic_timeout;
 440                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 441         }
 442
 443         timer_set(&port->periodic_timer, timeout);
 444 }
 445
 446 /**
 447  * Function handles mux state machine.
 448  *
 449  * Function implements Mux Machine from point 5.4.15 in 802.1AX documentation.
 450  * It should be called periodically.
 451  *
 452  * @param port                  Port to handle state machine.
 453  */
 454 static void
 455 mux_machine(struct bond_dev_private *internals, uint16_t slave_id)
 456 {
 457         struct port *port = &bond_mode_8023ad_ports[slave_id];
 458
 459         /* Save current state for later use */
 460         const uint8_t state_mask = STATE_SYNCHRONIZATION | STATE_DISTRIBUTING |
 461                 STATE_COLLECTING;
 462
 463         /* Enter DETACHED state on BEGIN condition or from any other state if
 464          * port was unselected */
 465         if (SM_FLAG(port, BEGIN) ||
 466                         port->selected == UNSELECTED || (port->selected == STANDBY &&
 467                                 (port->actor_state & state_mask) != 0)) {
 468                 /* detach mux from aggregator */
 469                 port->actor_state &= ~state_mask;
 470                 /* Set ntt to true if BEGIN condition or transition from any other state
 471                  * which is indicated that wait_while_timer was started */
 472                 if (SM_FLAG(port, BEGIN) ||
 473                                 !timer_is_stopped(&port->wait_while_timer)) {
 474                         SM_FLAG_SET(port, NTT);
 475                         MODE4_DEBUG("-> DETACHED\n");
 476                 }
 477                 timer_cancel(&port->wait_while_timer);
 478         }
 479
 480         if (timer_is_stopped(&port->wait_while_timer)) {
 481                 if (port->selected == SELECTED || port->selected == STANDBY) {
 482                         timer_set(&port->wait_while_timer,
 483                                 internals->mode4.aggregate_wait_timeout);
 484
 485                         MODE4_DEBUG("DETACHED -> WAITING\n");
 486                 }
 487                 /* Waiting state entered */
 488                 return;
 489         }
 490
 491         /* Transit next state if port is ready */
 492         if (!timer_is_expired(&port->wait_while_timer))
 493                 return;
 494
 495         if ((ACTOR_STATE(port, DISTRIBUTING) || ACTOR_STATE(port, COLLECTING)) &&
 496                 !PARTNER_STATE(port, SYNCHRONIZATION)) {
 497                 /* If in COLLECTING or DISTRIBUTING state and partner becomes out of
 498                  * sync transit to ATACHED state.  */
 499                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 500                 ACTOR_STATE_CLR(port, COLLECTING);
 501                 /* Clear actor sync to activate transit ATACHED in condition bellow */
 502                 ACTOR_STATE_CLR(port, SYNCHRONIZATION);
 503                 MODE4_DEBUG("Out of sync -> ATTACHED\n");
 504         }
 505
 506         if (!ACTOR_STATE(port, SYNCHRONIZATION)) {
 507                 /* attach mux to aggregator */
 508                 RTE_ASSERT((port->actor_state & (STATE_COLLECTING |
 509                         STATE_DISTRIBUTING)) == 0);
 510
 511                 ACTOR_STATE_SET(port, SYNCHRONIZATION);
 512                 SM_FLAG_SET(port, NTT);
 513                 MODE4_DEBUG("ATTACHED Entered\n");
 514         } else if (!ACTOR_STATE(port, COLLECTING)) {
 515                 /* Start collecting if in sync */
 516                 if (PARTNER_STATE(port, SYNCHRONIZATION)) {
 517                         MODE4_DEBUG("ATTACHED -> COLLECTING\n");
 518                         ACTOR_STATE_SET(port, COLLECTING);
 519                         SM_FLAG_SET(port, NTT);
 520                 }
 521         } else if (ACTOR_STATE(port, COLLECTING)) {
 522                 /* Check if partner is in COLLECTING state. If so this port can
 523                  * distribute frames to it */
 524                 if (!ACTOR_STATE(port, DISTRIBUTING)) {
 525                         if (PARTNER_STATE(port, COLLECTING)) {
 526                                 /* Enable  DISTRIBUTING if partner is collecting */
 527                                 ACTOR_STATE_SET(port, DISTRIBUTING);
 528                                 SM_FLAG_SET(port, NTT);
 529                                 MODE4_DEBUG("COLLECTING -> DISTRIBUTING\n");
 530                                 RTE_BOND_LOG(INFO,
 531                                         "Bond %u: slave id %u distributing started.",
 532                                         internals->port_id, slave_id);
 533                         }
 534                 } else {
 535                         if (!PARTNER_STATE(port, COLLECTING)) {
 536                                 /* Disable DISTRIBUTING (enter COLLECTING state) if partner
 537                                  * is not collecting */
 538                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 539                                 SM_FLAG_SET(port, NTT);
 540                                 MODE4_DEBUG("DISTRIBUTING -> COLLECTING\n");
 541                                 RTE_BOND_LOG(INFO,
 542                                         "Bond %u: slave id %u distributing stopped.",
 543                                         internals->port_id, slave_id);
 544                         }
 545                 }
 546         }
 547 }
 548
 549 /**
 550  * Function handles transmit state machine.
 551  *
 552  * Function implements Transmit Machine from point 5.4.16 in 802.1AX
 553  * documentation.
 554  *
 555  * @param port
 556  */
 557 static void
 558 tx_machine(struct bond_dev_private *internals, uint16_t slave_id)
 559 {
 560         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 561
 562         struct rte_mbuf *lacp_pkt = NULL;
 563         struct lacpdu_header *hdr;
 564         struct lacpdu *lacpdu;
 565
 566         /* If periodic timer is not running periodic machine is in NO PERIODIC and
 567          * according to 802.3ax standard tx machine should not transmit any frames
 568          * and set ntt to false. */
 569         if (timer_is_stopped(&port->periodic_timer))
 570                 SM_FLAG_CLR(port, NTT);
 571
 572         if (!SM_FLAG(port, NTT))
 573                 return;
 574
 575         if (!timer_is_expired(&port->tx_machine_timer))
 576                 return;
 577
 578         lacp_pkt = rte_pktmbuf_alloc(port->mbuf_pool);
 579         if (lacp_pkt == NULL) {
 580                 RTE_BOND_LOG(ERR, "Failed to allocate LACP packet from pool");
 581                 return;
 582         }
 583
 584         lacp_pkt->data_len = sizeof(*hdr);
 585         lacp_pkt->pkt_len = sizeof(*hdr);
 586
 587         hdr = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 588
 589         /* Source and destination MAC */
 590         rte_ether_addr_copy(&lacp_mac_addr, &hdr->eth_hdr.d_addr);
 591         rte_eth_macaddr_get(slave_id, &hdr->eth_hdr.s_addr);
 592         hdr->eth_hdr.ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_SLOW);
 593
 594         lacpdu = &hdr->lacpdu;
 595         memset(lacpdu, 0, sizeof(*lacpdu));
 596
 597         /* Initialize LACP part */
 598         lacpdu->subtype = SLOW_SUBTYPE_LACP;
 599         lacpdu->version_number = 1;
 600
 601         /* ACTOR */
 602         lacpdu->actor.tlv_type_info = TLV_TYPE_ACTOR_INFORMATION;
 603         lacpdu->actor.info_length = sizeof(struct lacpdu_actor_partner_params);
 604         memcpy(&hdr->lacpdu.actor.port_params, &port->actor,
 605                         sizeof(port->actor));
 606         agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 607         rte_ether_addr_copy(&agg->actor.system,
 608                         &hdr->lacpdu.actor.port_params.system);
 609         lacpdu->actor.state = port->actor_state;
 610
 611         /* PARTNER */
 612         lacpdu->partner.tlv_type_info = TLV_TYPE_PARTNER_INFORMATION;
 613         lacpdu->partner.info_length = sizeof(struct lacpdu_actor_partner_params);
 614         memcpy(&lacpdu->partner.port_params, &port->partner,
 615                         sizeof(struct port_params));
 616         lacpdu->partner.state = port->partner_state;
 617
 618         /* Other fields */
 619         lacpdu->tlv_type_collector_info = TLV_TYPE_COLLECTOR_INFORMATION;
 620         lacpdu->collector_info_length = 0x10;
 621         lacpdu->collector_max_delay = 0;
 622
 623         lacpdu->tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION;
 624         lacpdu->terminator_length = 0;
 625
 626         MODE4_DEBUG("Sending LACP frame\n");
 627         BOND_PRINT_LACP(lacpdu);
 628
 629         if (internals->mode4.dedicated_queues.enabled == 0) {
 630                 int retval = rte_ring_enqueue(port->tx_ring, lacp_pkt);
 631                 if (retval != 0) {
 632                         /* If TX ring full, drop packet and free message.
 633                            Retransmission will happen in next function call. */
 634                         rte_pktmbuf_free(lacp_pkt);
 635                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 636                         return;
 637                 }
 638         } else {
 639                 uint16_t pkts_sent = rte_eth_tx_burst(slave_id,
 640                                 internals->mode4.dedicated_queues.tx_qid,
 641                                 &lacp_pkt, 1);
 642                 if (pkts_sent != 1) {
 643                         rte_pktmbuf_free(lacp_pkt);
 644                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 645                         return;
 646                 }
 647         }
 648
 649
 650         timer_set(&port->tx_machine_timer, internals->mode4.tx_period_timeout);
 651         SM_FLAG_CLR(port, NTT);
 652 }
 653
 654 static uint16_t
 655 max_index(uint64_t *a, int n)
 656 {
 657         if (n <= 0)
 658                 return -1;
 659
 660         int i, max_i = 0;
 661         uint64_t max = a[0];
 662
 663         for (i = 1; i < n; ++i) {
 664                 if (a[i] > max) {
 665                         max = a[i];
 666                         max_i = i;
 667                 }
 668         }
 669
 670         return max_i;
 671 }
 672
 673 /**
 674  * Function assigns port to aggregator.
 675  *
 676  * @param bond_dev_private      Pointer to bond_dev_private structure.
 677  * @param port_pos                      Port to assign.
 678  */
 679 static void
 680 selection_logic(struct bond_dev_private *internals, uint16_t slave_id)
 681 {
 682         struct port *agg, *port;
 683         uint16_t slaves_count, new_agg_id, i, j = 0;
 684         uint16_t *slaves;
 685         uint64_t agg_bandwidth[RTE_MAX_ETHPORTS] = {0};
 686         uint64_t agg_count[RTE_MAX_ETHPORTS] = {0};
 687         uint16_t default_slave = 0;
 688         struct rte_eth_link link_info;
 689         uint16_t agg_new_idx = 0;
 690         int ret;
 691
 692         slaves = internals->active_slaves;
 693         slaves_count = internals->active_slave_count;
 694         port = &bond_mode_8023ad_ports[slave_id];
 695
 696         /* Search for aggregator suitable for this port */
 697         for (i = 0; i < slaves_count; ++i) {
 698                 agg = &bond_mode_8023ad_ports[slaves[i]];
 699                 /* Skip ports that are not aggreagators */
 700                 if (agg->aggregator_port_id != slaves[i])
 701                         continue;
 702
 703                 ret = rte_eth_link_get_nowait(slaves[i], &link_info);
 704                 if (ret < 0) {
 705                         RTE_BOND_LOG(ERR,
 706                                 "Slave (port %u) link get failed: %s\n",
 707                                 slaves[i], rte_strerror(-ret));
 708                         continue;
 709                 }
 710                 agg_count[i] += 1;
 711                 agg_bandwidth[i] += link_info.link_speed;
 712
 713                 /* Actors system ID is not checked since all slave device have the same
 714                  * ID (MAC address). */
 715                 if ((agg->actor.key == port->actor.key &&
 716                         agg->partner.system_priority == port->partner.system_priority &&
 717                         rte_is_same_ether_addr(&agg->partner.system,
 718                                         &port->partner.system) == 1
 719                         && (agg->partner.key == port->partner.key)) &&
 720                         rte_is_zero_ether_addr(&port->partner.system) != 1 &&
 721                         (agg->actor.key &
 722                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) != 0) {
 723
 724                         if (j == 0)
 725                                 default_slave = i;
 726                         j++;
 727                 }
 728         }
 729
 730         switch (internals->mode4.agg_selection) {
 731         case AGG_COUNT:
 732                 agg_new_idx = max_index(agg_count, slaves_count);
 733                 new_agg_id = slaves[agg_new_idx];
 734                 break;
 735         case AGG_BANDWIDTH:
 736                 agg_new_idx = max_index(agg_bandwidth, slaves_count);
 737                 new_agg_id = slaves[agg_new_idx];
 738                 break;
 739         case AGG_STABLE:
 740                 if (default_slave == slaves_count)
 741                         new_agg_id = slaves[slave_id];
 742                 else
 743                         new_agg_id = slaves[default_slave];
 744                 break;
 745         default:
 746                 if (default_slave == slaves_count)
 747                         new_agg_id = slaves[slave_id];
 748                 else
 749                         new_agg_id = slaves[default_slave];
 750                 break;
 751         }
 752
 753         if (new_agg_id != port->aggregator_port_id) {
 754                 port->aggregator_port_id = new_agg_id;
 755
 756                 MODE4_DEBUG("-> SELECTED: ID=%3u\n"
 757                         "\t%s aggregator ID=%3u\n",
 758                         port->aggregator_port_id,
 759                         port->aggregator_port_id == slave_id ?
 760                                 "aggregator not found, using default" : "aggregator found",
 761                         port->aggregator_port_id);
 762         }
 763
 764         port->selected = SELECTED;
 765 }
 766
 767 /* Function maps DPDK speed to bonding speed stored in key field */
 768 static uint16_t
 769 link_speed_key(uint16_t speed) {
 770         uint16_t key_speed;
 771
 772         switch (speed) {
 773         case ETH_SPEED_NUM_NONE:
 774                 key_speed = 0x00;
 775                 break;
 776         case ETH_SPEED_NUM_10M:
 777                 key_speed = BOND_LINK_SPEED_KEY_10M;
 778                 break;
 779         case ETH_SPEED_NUM_100M:
 780                 key_speed = BOND_LINK_SPEED_KEY_100M;
 781                 break;
 782         case ETH_SPEED_NUM_1G:
 783                 key_speed = BOND_LINK_SPEED_KEY_1000M;
 784                 break;
 785         case ETH_SPEED_NUM_10G:
 786                 key_speed = BOND_LINK_SPEED_KEY_10G;
 787                 break;
 788         case ETH_SPEED_NUM_20G:
 789                 key_speed = BOND_LINK_SPEED_KEY_20G;
 790                 break;
 791         case ETH_SPEED_NUM_40G:
 792                 key_speed = BOND_LINK_SPEED_KEY_40G;
 793                 break;
 794         default:
 795                 /* Unknown speed*/
 796                 key_speed = 0xFFFF;
 797         }
 798
 799         return key_speed;
 800 }
 801
 802 static void
 803 rx_machine_update(struct bond_dev_private *internals, uint16_t slave_id,
 804                 struct rte_mbuf *lacp_pkt) {
 805         struct lacpdu_header *lacp;
 806         struct lacpdu_actor_partner_params *partner;
 807         struct port *port, *agg;
 808
 809         if (lacp_pkt != NULL) {
 810                 lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 811                 RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
 812
 813                 partner = &lacp->lacpdu.partner;
 814                 port = &bond_mode_8023ad_ports[slave_id];
 815                 agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 816
 817                 if (rte_is_zero_ether_addr(&partner->port_params.system) ||
 818                         rte_is_same_ether_addr(&partner->port_params.system,
 819                                 &agg->actor.system)) {
 820                         /* This LACP frame is sending to the bonding port
 821                          * so pass it to rx_machine.
 822                          */
 823                         rx_machine(internals, slave_id, &lacp->lacpdu);
 824                 } else {
 825                         char preferred_system_name[RTE_ETHER_ADDR_FMT_SIZE];
 826                         char self_system_name[RTE_ETHER_ADDR_FMT_SIZE];
 827
 828                         rte_ether_format_addr(preferred_system_name,
 829                                 RTE_ETHER_ADDR_FMT_SIZE, &partner->port_params.system);
 830                         rte_ether_format_addr(self_system_name,
 831                                 RTE_ETHER_ADDR_FMT_SIZE, &agg->actor.system);
 832                         MODE4_DEBUG("preferred partner system %s "
 833                                 "is not equal with self system: %s\n",
 834                                 preferred_system_name, self_system_name);
 835                 }
 836                 rte_pktmbuf_free(lacp_pkt);
 837         } else
 838                 rx_machine(internals, slave_id, NULL);
 839 }
 840
 841 static void
 842 bond_mode_8023ad_periodic_cb(void *arg)
 843 {
 844         struct rte_eth_dev *bond_dev = arg;
 845         struct bond_dev_private *internals = bond_dev->data->dev_private;
 846         struct port *port;
 847         struct rte_eth_link link_info;
 848         struct rte_ether_addr slave_addr;
 849         struct rte_mbuf *lacp_pkt = NULL;
 850         uint16_t slave_id;
 851         uint16_t i;
 852
 853
 854         /* Update link status on each port */
 855         for (i = 0; i < internals->active_slave_count; i++) {
 856                 uint16_t key;
 857                 int ret;
 858
 859                 slave_id = internals->active_slaves[i];
 860                 ret = rte_eth_link_get_nowait(slave_id, &link_info);
 861                 if (ret < 0) {
 862                         RTE_BOND_LOG(ERR,
 863                                 "Slave (port %u) link get failed: %s\n",
 864                                 slave_id, rte_strerror(-ret));
 865                 }
 866
 867                 if (ret >= 0 && link_info.link_status != 0) {
 868                         key = link_speed_key(link_info.link_speed) << 1;
 869                         if (link_info.link_duplex == ETH_LINK_FULL_DUPLEX)
 870                                 key |= BOND_LINK_FULL_DUPLEX_KEY;
 871                 } else {
 872                         key = 0;
 873                 }
 874
 875                 rte_eth_macaddr_get(slave_id, &slave_addr);
 876                 port = &bond_mode_8023ad_ports[slave_id];
 877
 878                 key = rte_cpu_to_be_16(key);
 879                 if (key != port->actor.key) {
 880                         if (!(key & rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)))
 881                                 set_warning_flags(port, WRN_NOT_LACP_CAPABLE);
 882
 883                         port->actor.key = key;
 884                         SM_FLAG_SET(port, NTT);
 885                 }
 886
 887                 if (!rte_is_same_ether_addr(&port->actor.system, &slave_addr)) {
 888                         rte_ether_addr_copy(&slave_addr, &port->actor.system);
 889                         if (port->aggregator_port_id == slave_id)
 890                                 SM_FLAG_SET(port, NTT);
 891                 }
 892         }
 893
 894         for (i = 0; i < internals->active_slave_count; i++) {
 895                 slave_id = internals->active_slaves[i];
 896                 port = &bond_mode_8023ad_ports[slave_id];
 897
 898                 if ((port->actor.key &
 899                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) == 0) {
 900
 901                         SM_FLAG_SET(port, BEGIN);
 902
 903                         /* LACP is disabled on half duples or link is down */
 904                         if (SM_FLAG(port, LACP_ENABLED)) {
 905                                 /* If port was enabled set it to BEGIN state */
 906                                 SM_FLAG_CLR(port, LACP_ENABLED);
 907                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 908                                 ACTOR_STATE_CLR(port, COLLECTING);
 909                         }
 910
 911                         /* Skip this port processing */
 912                         continue;
 913                 }
 914
 915                 SM_FLAG_SET(port, LACP_ENABLED);
 916
 917                 if (internals->mode4.dedicated_queues.enabled == 0) {
 918                         /* Find LACP packet to this port. Do not check subtype,
 919                          * it is done in function that queued packet
 920                          */
 921                         int retval = rte_ring_dequeue(port->rx_ring,
 922                                         (void **)&lacp_pkt);
 923
 924                         if (retval != 0)
 925                                 lacp_pkt = NULL;
 926
 927                         rx_machine_update(internals, slave_id, lacp_pkt);
 928                 } else {
 929                         uint16_t rx_count = rte_eth_rx_burst(slave_id,
 930                                         internals->mode4.dedicated_queues.rx_qid,
 931                                         &lacp_pkt, 1);
 932
 933                         if (rx_count == 1)
 934                                 bond_mode_8023ad_handle_slow_pkt(internals,
 935                                                 slave_id, lacp_pkt);
 936                         else
 937                                 rx_machine_update(internals, slave_id, NULL);
 938                 }
 939
 940                 periodic_machine(internals, slave_id);
 941                 mux_machine(internals, slave_id);
 942                 tx_machine(internals, slave_id);
 943                 selection_logic(internals, slave_id);
 944
 945                 SM_FLAG_CLR(port, BEGIN);
 946                 show_warnings(slave_id);
 947         }
 948
 949         rte_eal_alarm_set(internals->mode4.update_timeout_us,
 950                         bond_mode_8023ad_periodic_cb, arg);
 951 }
 952
 953 static int
 954 bond_mode_8023ad_register_lacp_mac(uint16_t slave_id)
 955 {
 956         int ret;
 957
 958         ret = rte_eth_allmulticast_enable(slave_id);
 959         if (ret != 0) {
 960                 RTE_BOND_LOG(ERR,
 961                         "failed to enable allmulti mode for port %u: %s",
 962                         slave_id, rte_strerror(-ret));
 963         }
 964         if (rte_eth_allmulticast_get(slave_id)) {
 965                 RTE_BOND_LOG(DEBUG, "forced allmulti for port %u",
 966                              slave_id);
 967                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 968                                 BOND_8023AD_FORCED_ALLMULTI;
 969                 return 0;
 970         }
 971
 972         ret = rte_eth_promiscuous_enable(slave_id);
 973         if (ret != 0) {
 974                 RTE_BOND_LOG(ERR,
 975                         "failed to enable promiscuous mode for port %u: %s",
 976                         slave_id, rte_strerror(-ret));
 977         }
 978         if (rte_eth_promiscuous_get(slave_id)) {
 979                 RTE_BOND_LOG(DEBUG, "forced promiscuous for port %u",
 980                              slave_id);
 981                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 982                                 BOND_8023AD_FORCED_PROMISC;
 983                 return 0;
 984         }
 985
 986         return -1;
 987 }
 988
 989 static void
 990 bond_mode_8023ad_unregister_lacp_mac(uint16_t slave_id)
 991 {
 992         int ret;
 993
 994         switch (bond_mode_8023ad_ports[slave_id].forced_rx_flags) {
 995         case BOND_8023AD_FORCED_ALLMULTI:
 996                 RTE_BOND_LOG(DEBUG, "unset allmulti for port %u", slave_id);
 997                 ret = rte_eth_allmulticast_disable(slave_id);
 998                 if (ret != 0)
 999                         RTE_BOND_LOG(ERR,
1000                                 "failed to disable allmulti mode for port %u: %s",
1001                                 slave_id, rte_strerror(-ret));
1002                 break;
1003
1004         case BOND_8023AD_FORCED_PROMISC:
1005                 RTE_BOND_LOG(DEBUG, "unset promisc for port %u", slave_id);
1006                 ret = rte_eth_promiscuous_disable(slave_id);
1007                 if (ret != 0)
1008                         RTE_BOND_LOG(ERR,
1009                                 "failed to disable promiscuous mode for port %u: %s",
1010                                 slave_id, rte_strerror(-ret));
1011                 break;
1012
1013         default:
1014                 break;
1015         }
1016 }
1017
1018 void
1019 bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev,
1020                                 uint16_t slave_id)
1021 {
1022         struct bond_dev_private *internals = bond_dev->data->dev_private;
1023
1024         struct port *port = &bond_mode_8023ad_ports[slave_id];
1025         struct port_params initial = {
1026                         .system = { { 0 } },
1027                         .system_priority = rte_cpu_to_be_16(0xFFFF),
1028                         .key = rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY),
1029                         .port_priority = rte_cpu_to_be_16(0x00FF),
1030                         .port_number = 0,
1031         };
1032
1033         char mem_name[RTE_ETH_NAME_MAX_LEN];
1034         int socket_id;
1035         unsigned element_size;
1036         uint32_t total_tx_desc;
1037         struct bond_tx_queue *bd_tx_q;
1038         uint16_t q_id;
1039
1040         /* Given slave mus not be in active list */
1041         RTE_ASSERT(find_slave_by_id(internals->active_slaves,
1042         internals->active_slave_count, slave_id) == internals->active_slave_count);
1043         RTE_SET_USED(internals); /* used only for assert when enabled */
1044
1045         memcpy(&port->actor, &initial, sizeof(struct port_params));
1046         /* Standard requires that port ID must be grater than 0.
1047          * Add 1 do get corresponding port_number */
1048         port->actor.port_number = rte_cpu_to_be_16(slave_id + 1);
1049
1050         memcpy(&port->partner, &initial, sizeof(struct port_params));
1051         memcpy(&port->partner_admin, &initial, sizeof(struct port_params));
1052
1053         /* default states */
1054         port->actor_state = STATE_AGGREGATION | STATE_LACP_ACTIVE | STATE_DEFAULTED;
1055         port->partner_state = STATE_LACP_ACTIVE | STATE_AGGREGATION;
1056         port->sm_flags = SM_FLAGS_BEGIN;
1057
1058         /* use this port as agregator */
1059         port->aggregator_port_id = slave_id;
1060
1061         if (bond_mode_8023ad_register_lacp_mac(slave_id) < 0) {
1062                 RTE_BOND_LOG(WARNING, "slave %u is most likely broken and won't receive LACP packets",
1063                              slave_id);
1064         }
1065
1066         timer_cancel(&port->warning_timer);
1067
1068         if (port->mbuf_pool != NULL)
1069                 return;
1070
1071         RTE_ASSERT(port->rx_ring == NULL);
1072         RTE_ASSERT(port->tx_ring == NULL);
1073
1074         socket_id = rte_eth_dev_socket_id(slave_id);
1075         if (socket_id == -1)
1076                 socket_id = rte_socket_id();
1077
1078         element_size = sizeof(struct slow_protocol_frame) +
1079                                 RTE_PKTMBUF_HEADROOM;
1080
1081         /* The size of the mempool should be at least:
1082          * the sum of the TX descriptors + BOND_MODE_8023AX_SLAVE_TX_PKTS */
1083         total_tx_desc = BOND_MODE_8023AX_SLAVE_TX_PKTS;
1084         for (q_id = 0; q_id < bond_dev->data->nb_tx_queues; q_id++) {
1085                 bd_tx_q = (struct bond_tx_queue*)bond_dev->data->tx_queues[q_id];
1086                 total_tx_desc += bd_tx_q->nb_tx_desc;
1087         }
1088
1089         snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_pool", slave_id);
1090         port->mbuf_pool = rte_pktmbuf_pool_create(mem_name, total_tx_desc,
1091                 RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ?
1092                         32 : RTE_MEMPOOL_CACHE_MAX_SIZE,
1093                 0, element_size, socket_id);
1094
1095         /* Any memory allocation failure in initialization is critical because
1096          * resources can't be free, so reinitialization is impossible. */
1097         if (port->mbuf_pool == NULL) {
1098                 rte_panic("Slave %u: Failed to create memory pool '%s': %s\n",
1099                         slave_id, mem_name, rte_strerror(rte_errno));
1100         }
1101
1102         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_rx", slave_id);
1103         port->rx_ring = rte_ring_create(mem_name,
1104                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_RX_PKTS), socket_id, 0);
1105
1106         if (port->rx_ring == NULL) {
1107                 rte_panic("Slave %u: Failed to create rx ring '%s': %s\n", slave_id,
1108                         mem_name, rte_strerror(rte_errno));
1109         }
1110
1111         /* TX ring is at least one pkt longer to make room for marker packet. */
1112         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_tx", slave_id);
1113         port->tx_ring = rte_ring_create(mem_name,
1114                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_TX_PKTS + 1), socket_id, 0);
1115
1116         if (port->tx_ring == NULL) {
1117                 rte_panic("Slave %u: Failed to create tx ring '%s': %s\n", slave_id,
1118                         mem_name, rte_strerror(rte_errno));
1119         }
1120 }
1121
1122 int
1123 bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev __rte_unused,
1124                 uint16_t slave_id)
1125 {
1126         void *pkt = NULL;
1127         struct port *port = NULL;
1128         uint8_t old_partner_state;
1129
1130         port = &bond_mode_8023ad_ports[slave_id];
1131
1132         ACTOR_STATE_CLR(port, AGGREGATION);
1133         port->selected = UNSELECTED;
1134
1135         old_partner_state = port->partner_state;
1136         record_default(port);
1137
1138         bond_mode_8023ad_unregister_lacp_mac(slave_id);
1139
1140         /* If partner timeout state changes then disable timer */
1141         if (!((old_partner_state ^ port->partner_state) &
1142                         STATE_LACP_SHORT_TIMEOUT))
1143                 timer_cancel(&port->current_while_timer);
1144
1145         PARTNER_STATE_CLR(port, AGGREGATION);
1146         ACTOR_STATE_CLR(port, EXPIRED);
1147
1148         /* flush rx/tx rings */
1149         while (rte_ring_dequeue(port->rx_ring, &pkt) == 0)
1150                 rte_pktmbuf_free((struct rte_mbuf *)pkt);
1151
1152         while (rte_ring_dequeue(port->tx_ring, &pkt) == 0)
1153                         rte_pktmbuf_free((struct rte_mbuf *)pkt);
1154         return 0;
1155 }
1156
1157 void
1158 bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev)
1159 {
1160         struct bond_dev_private *internals = bond_dev->data->dev_private;
1161         struct rte_ether_addr slave_addr;
1162         struct port *slave, *agg_slave;
1163         uint16_t slave_id, i, j;
1164
1165         bond_mode_8023ad_stop(bond_dev);
1166
1167         for (i = 0; i < internals->active_slave_count; i++) {
1168                 slave_id = internals->active_slaves[i];
1169                 slave = &bond_mode_8023ad_ports[slave_id];
1170                 rte_eth_macaddr_get(slave_id, &slave_addr);
1171
1172                 if (rte_is_same_ether_addr(&slave_addr, &slave->actor.system))
1173                         continue;
1174
1175                 rte_ether_addr_copy(&slave_addr, &slave->actor.system);
1176                 /* Do nothing if this port is not an aggregator. In other case
1177                  * Set NTT flag on every port that use this aggregator. */
1178                 if (slave->aggregator_port_id != slave_id)
1179                         continue;
1180
1181                 for (j = 0; j < internals->active_slave_count; j++) {
1182                         agg_slave = &bond_mode_8023ad_ports[internals->active_slaves[j]];
1183                         if (agg_slave->aggregator_port_id == slave_id)
1184                                 SM_FLAG_SET(agg_slave, NTT);
1185                 }
1186         }
1187
1188         if (bond_dev->data->dev_started)
1189                 bond_mode_8023ad_start(bond_dev);
1190 }
1191
1192 static void
1193 bond_mode_8023ad_conf_get(struct rte_eth_dev *dev,
1194                 struct rte_eth_bond_8023ad_conf *conf)
1195 {
1196         struct bond_dev_private *internals = dev->data->dev_private;
1197         struct mode8023ad_private *mode4 = &internals->mode4;
1198         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1199
1200         conf->fast_periodic_ms = mode4->fast_periodic_timeout / ms_ticks;
1201         conf->slow_periodic_ms = mode4->slow_periodic_timeout / ms_ticks;
1202         conf->short_timeout_ms = mode4->short_timeout / ms_ticks;
1203         conf->long_timeout_ms = mode4->long_timeout / ms_ticks;
1204         conf->aggregate_wait_timeout_ms = mode4->aggregate_wait_timeout / ms_ticks;
1205         conf->tx_period_ms = mode4->tx_period_timeout / ms_ticks;
1206         conf->update_timeout_ms = mode4->update_timeout_us / 1000;
1207         conf->rx_marker_period_ms = mode4->rx_marker_timeout / ms_ticks;
1208         conf->slowrx_cb = mode4->slowrx_cb;
1209         conf->agg_selection = mode4->agg_selection;
1210 }
1211
1212 static void
1213 bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf)
1214 {
1215         conf->fast_periodic_ms = BOND_8023AD_FAST_PERIODIC_MS;
1216         conf->slow_periodic_ms = BOND_8023AD_SLOW_PERIODIC_MS;
1217         conf->short_timeout_ms = BOND_8023AD_SHORT_TIMEOUT_MS;
1218         conf->long_timeout_ms = BOND_8023AD_LONG_TIMEOUT_MS;
1219         conf->aggregate_wait_timeout_ms = BOND_8023AD_AGGREGATE_WAIT_TIMEOUT_MS;
1220         conf->tx_period_ms = BOND_8023AD_TX_MACHINE_PERIOD_MS;
1221         conf->rx_marker_period_ms = BOND_8023AD_RX_MARKER_PERIOD_MS;
1222         conf->update_timeout_ms = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS;
1223         conf->slowrx_cb = NULL;
1224         conf->agg_selection = AGG_STABLE;
1225 }
1226
1227 static void
1228 bond_mode_8023ad_conf_assign(struct mode8023ad_private *mode4,
1229                 struct rte_eth_bond_8023ad_conf *conf)
1230 {
1231         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1232
1233         mode4->fast_periodic_timeout = conf->fast_periodic_ms * ms_ticks;
1234         mode4->slow_periodic_timeout = conf->slow_periodic_ms * ms_ticks;
1235         mode4->short_timeout = conf->short_timeout_ms * ms_ticks;
1236         mode4->long_timeout = conf->long_timeout_ms * ms_ticks;
1237         mode4->aggregate_wait_timeout = conf->aggregate_wait_timeout_ms * ms_ticks;
1238         mode4->tx_period_timeout = conf->tx_period_ms * ms_ticks;
1239         mode4->rx_marker_timeout = conf->rx_marker_period_ms * ms_ticks;
1240         mode4->update_timeout_us = conf->update_timeout_ms * 1000;
1241
1242         mode4->dedicated_queues.enabled = 0;
1243         mode4->dedicated_queues.rx_qid = UINT16_MAX;
1244         mode4->dedicated_queues.tx_qid = UINT16_MAX;
1245 }
1246
1247 void
1248 bond_mode_8023ad_setup(struct rte_eth_dev *dev,
1249                 struct rte_eth_bond_8023ad_conf *conf)
1250 {
1251         struct rte_eth_bond_8023ad_conf def_conf;
1252         struct bond_dev_private *internals = dev->data->dev_private;
1253         struct mode8023ad_private *mode4 = &internals->mode4;
1254
1255         if (conf == NULL) {
1256                 conf = &def_conf;
1257                 bond_mode_8023ad_conf_get_default(conf);
1258         }
1259
1260         bond_mode_8023ad_stop(dev);
1261         bond_mode_8023ad_conf_assign(mode4, conf);
1262         mode4->slowrx_cb = conf->slowrx_cb;
1263         mode4->agg_selection = AGG_STABLE;
1264
1265         if (dev->data->dev_started)
1266                 bond_mode_8023ad_start(dev);
1267 }
1268
1269 int
1270 bond_mode_8023ad_enable(struct rte_eth_dev *bond_dev)
1271 {
1272         struct bond_dev_private *internals = bond_dev->data->dev_private;
1273         uint16_t i;
1274
1275         for (i = 0; i < internals->active_slave_count; i++)
1276                 bond_mode_8023ad_activate_slave(bond_dev,
1277                                 internals->active_slaves[i]);
1278
1279         return 0;
1280 }
1281
1282 int
1283 bond_mode_8023ad_start(struct rte_eth_dev *bond_dev)
1284 {
1285         struct bond_dev_private *internals = bond_dev->data->dev_private;
1286         struct mode8023ad_private *mode4 = &internals->mode4;
1287         static const uint64_t us = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS * 1000;
1288
1289         rte_eth_macaddr_get(internals->port_id, &mode4->mac_addr);
1290         if (mode4->slowrx_cb)
1291                 return rte_eal_alarm_set(us, &bond_mode_8023ad_ext_periodic_cb,
1292                                          bond_dev);
1293
1294         return rte_eal_alarm_set(us, &bond_mode_8023ad_periodic_cb, bond_dev);
1295 }
1296
1297 void
1298 bond_mode_8023ad_stop(struct rte_eth_dev *bond_dev)
1299 {
1300         struct bond_dev_private *internals = bond_dev->data->dev_private;
1301         struct mode8023ad_private *mode4 = &internals->mode4;
1302
1303         if (mode4->slowrx_cb) {
1304                 rte_eal_alarm_cancel(&bond_mode_8023ad_ext_periodic_cb,
1305                                      bond_dev);
1306                 return;
1307         }
1308         rte_eal_alarm_cancel(&bond_mode_8023ad_periodic_cb, bond_dev);
1309 }
1310
1311 void
1312 bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
1313                                   uint16_t slave_id, struct rte_mbuf *pkt)
1314 {
1315         struct mode8023ad_private *mode4 = &internals->mode4;
1316         struct port *port = &bond_mode_8023ad_ports[slave_id];
1317         struct marker_header *m_hdr;
1318         uint64_t marker_timer, old_marker_timer;
1319         int retval;
1320         uint8_t wrn, subtype;
1321         /* If packet is a marker, we send response now by reusing given packet
1322          * and update only source MAC, destination MAC is multicast so don't
1323          * update it. Other frames will be handled later by state machines */
1324         subtype = rte_pktmbuf_mtod(pkt,
1325                         struct slow_protocol_frame *)->slow_protocol.subtype;
1326
1327         if (subtype == SLOW_SUBTYPE_MARKER) {
1328                 m_hdr = rte_pktmbuf_mtod(pkt, struct marker_header *);
1329
1330                 if (likely(m_hdr->marker.tlv_type_marker != MARKER_TLV_TYPE_INFO)) {
1331                         wrn = WRN_UNKNOWN_MARKER_TYPE;
1332                         goto free_out;
1333                 }
1334
1335                 /* Setup marker timer. Do it in loop in case concurrent access. */
1336                 do {
1337                         old_marker_timer = port->rx_marker_timer;
1338                         if (!timer_is_expired(&old_marker_timer)) {
1339                                 wrn = WRN_RX_MARKER_TO_FAST;
1340                                 goto free_out;
1341                         }
1342
1343                         timer_set(&marker_timer, mode4->rx_marker_timeout);
1344                         retval = rte_atomic64_cmpset(&port->rx_marker_timer,
1345                                 old_marker_timer, marker_timer);
1346                 } while (unlikely(retval == 0));
1347
1348                 m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP;
1349                 rte_eth_macaddr_get(slave_id, &m_hdr->eth_hdr.s_addr);
1350
1351                 if (internals->mode4.dedicated_queues.enabled == 0) {
1352                         if (rte_ring_enqueue(port->tx_ring, pkt) != 0) {
1353                                 /* reset timer */
1354                                 port->rx_marker_timer = 0;
1355                                 wrn = WRN_TX_QUEUE_FULL;
1356                                 goto free_out;
1357                         }
1358                 } else {
1359                         /* Send packet directly to the slow queue */
1360                         uint16_t tx_count = rte_eth_tx_burst(slave_id,
1361                                         internals->mode4.dedicated_queues.tx_qid,
1362                                         &pkt, 1);
1363                         if (tx_count != 1) {
1364                                 /* reset timer */
1365                                 port->rx_marker_timer = 0;
1366                                 wrn = WRN_TX_QUEUE_FULL;
1367                                 goto free_out;
1368                         }
1369                 }
1370         } else if (likely(subtype == SLOW_SUBTYPE_LACP)) {
1371                 if (internals->mode4.dedicated_queues.enabled == 0) {
1372                         if (rte_ring_enqueue(port->rx_ring, pkt) != 0) {
1373                                 /* If RX fing full free lacpdu message and drop packet */
1374                                 wrn = WRN_RX_QUEUE_FULL;
1375                                 goto free_out;
1376                         }
1377                 } else
1378                         rx_machine_update(internals, slave_id, pkt);
1379         } else {
1380                 wrn = WRN_UNKNOWN_SLOW_TYPE;
1381                 goto free_out;
1382         }
1383
1384         return;
1385
1386 free_out:
1387         set_warning_flags(port, wrn);
1388         rte_pktmbuf_free(pkt);
1389 }
1390
1391 int
1392 rte_eth_bond_8023ad_conf_get(uint16_t port_id,
1393                 struct rte_eth_bond_8023ad_conf *conf)
1394 {
1395         struct rte_eth_dev *bond_dev;
1396
1397         if (valid_bonded_port_id(port_id) != 0)
1398                 return -EINVAL;
1399
1400         if (conf == NULL)
1401                 return -EINVAL;
1402
1403         bond_dev = &rte_eth_devices[port_id];
1404         bond_mode_8023ad_conf_get(bond_dev, conf);
1405         return 0;
1406 }
1407
1408 int
1409 rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id,
1410                 enum rte_bond_8023ad_agg_selection agg_selection)
1411 {
1412         struct rte_eth_dev *bond_dev;
1413         struct bond_dev_private *internals;
1414         struct mode8023ad_private *mode4;
1415
1416         if (valid_bonded_port_id(port_id) != 0)
1417                 return -EINVAL;
1418
1419         bond_dev = &rte_eth_devices[port_id];
1420         internals = bond_dev->data->dev_private;
1421
1422         if (internals->mode != 4)
1423                 return -EINVAL;
1424
1425         mode4 = &internals->mode4;
1426         if (agg_selection == AGG_COUNT || agg_selection == AGG_BANDWIDTH
1427                         || agg_selection == AGG_STABLE)
1428                 mode4->agg_selection = agg_selection;
1429         return 0;
1430 }
1431
1432 int rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id)
1433 {
1434         struct rte_eth_dev *bond_dev;
1435         struct bond_dev_private *internals;
1436         struct mode8023ad_private *mode4;
1437
1438         if (valid_bonded_port_id(port_id) != 0)
1439                 return -EINVAL;
1440
1441         bond_dev = &rte_eth_devices[port_id];
1442         internals = bond_dev->data->dev_private;
1443
1444         if (internals->mode != 4)
1445                 return -EINVAL;
1446         mode4 = &internals->mode4;
1447
1448         return mode4->agg_selection;
1449 }
1450
1451
1452
1453 static int
1454 bond_8023ad_setup_validate(uint16_t port_id,
1455                 struct rte_eth_bond_8023ad_conf *conf)
1456 {
1457         if (valid_bonded_port_id(port_id) != 0)
1458                 return -EINVAL;
1459
1460         if (conf != NULL) {
1461                 /* Basic sanity check */
1462                 if (conf->slow_periodic_ms == 0 ||
1463                                 conf->fast_periodic_ms >= conf->slow_periodic_ms ||
1464                                 conf->long_timeout_ms == 0 ||
1465                                 conf->short_timeout_ms >= conf->long_timeout_ms ||
1466                                 conf->aggregate_wait_timeout_ms == 0 ||
1467                                 conf->tx_period_ms == 0 ||
1468                                 conf->rx_marker_period_ms == 0 ||
1469                                 conf->update_timeout_ms == 0) {
1470                         RTE_BOND_LOG(ERR, "given mode 4 configuration is invalid");
1471                         return -EINVAL;
1472                 }
1473         }
1474
1475         return 0;
1476 }
1477
1478
1479 int
1480 rte_eth_bond_8023ad_setup(uint16_t port_id,
1481                 struct rte_eth_bond_8023ad_conf *conf)
1482 {
1483         struct rte_eth_dev *bond_dev;
1484         int err;
1485
1486         err = bond_8023ad_setup_validate(port_id, conf);
1487         if (err != 0)
1488                 return err;
1489
1490         bond_dev = &rte_eth_devices[port_id];
1491         bond_mode_8023ad_setup(bond_dev, conf);
1492
1493         return 0;
1494 }
1495
1496
1497
1498
1499
1500 int
1501 rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id,
1502                 struct rte_eth_bond_8023ad_slave_info *info)
1503 {
1504         struct rte_eth_dev *bond_dev;
1505         struct bond_dev_private *internals;
1506         struct port *port;
1507
1508         if (info == NULL || valid_bonded_port_id(port_id) != 0 ||
1509                         rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1510                 return -EINVAL;
1511
1512         bond_dev = &rte_eth_devices[port_id];
1513
1514         internals = bond_dev->data->dev_private;
1515         if (find_slave_by_id(internals->active_slaves,
1516                         internals->active_slave_count, slave_id) ==
1517                                 internals->active_slave_count)
1518                 return -EINVAL;
1519
1520         port = &bond_mode_8023ad_ports[slave_id];
1521         info->selected = port->selected;
1522
1523         info->actor_state = port->actor_state;
1524         rte_memcpy(&info->actor, &port->actor, sizeof(port->actor));
1525
1526         info->partner_state = port->partner_state;
1527         rte_memcpy(&info->partner, &port->partner, sizeof(port->partner));
1528
1529         info->agg_port_id = port->aggregator_port_id;
1530         return 0;
1531 }
1532
1533 static int
1534 bond_8023ad_ext_validate(uint16_t port_id, uint16_t slave_id)
1535 {
1536         struct rte_eth_dev *bond_dev;
1537         struct bond_dev_private *internals;
1538         struct mode8023ad_private *mode4;
1539
1540         if (rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1541                 return -EINVAL;
1542
1543         bond_dev = &rte_eth_devices[port_id];
1544
1545         if (!bond_dev->data->dev_started)
1546                 return -EINVAL;
1547
1548         internals = bond_dev->data->dev_private;
1549         if (find_slave_by_id(internals->active_slaves,
1550                         internals->active_slave_count, slave_id) ==
1551                                 internals->active_slave_count)
1552                 return -EINVAL;
1553
1554         mode4 = &internals->mode4;
1555         if (mode4->slowrx_cb == NULL)
1556                 return -EINVAL;
1557
1558         return 0;
1559 }
1560
1561 int
1562 rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id,
1563                                 int enabled)
1564 {
1565         struct port *port;
1566         int res;
1567
1568         res = bond_8023ad_ext_validate(port_id, slave_id);
1569         if (res != 0)
1570                 return res;
1571
1572         port = &bond_mode_8023ad_ports[slave_id];
1573
1574         if (enabled)
1575                 ACTOR_STATE_SET(port, COLLECTING);
1576         else
1577                 ACTOR_STATE_CLR(port, COLLECTING);
1578
1579         return 0;
1580 }
1581
1582 int
1583 rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id,
1584                                 int enabled)
1585 {
1586         struct port *port;
1587         int res;
1588
1589         res = bond_8023ad_ext_validate(port_id, slave_id);
1590         if (res != 0)
1591                 return res;
1592
1593         port = &bond_mode_8023ad_ports[slave_id];
1594
1595         if (enabled)
1596                 ACTOR_STATE_SET(port, DISTRIBUTING);
1597         else
1598                 ACTOR_STATE_CLR(port, DISTRIBUTING);
1599
1600         return 0;
1601 }
1602
1603 int
1604 rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id)
1605 {
1606         struct port *port;
1607         int err;
1608
1609         err = bond_8023ad_ext_validate(port_id, slave_id);
1610         if (err != 0)
1611                 return err;
1612
1613         port = &bond_mode_8023ad_ports[slave_id];
1614         return ACTOR_STATE(port, DISTRIBUTING);
1615 }
1616
1617 int
1618 rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id)
1619 {
1620         struct port *port;
1621         int err;
1622
1623         err = bond_8023ad_ext_validate(port_id, slave_id);
1624         if (err != 0)
1625                 return err;
1626
1627         port = &bond_mode_8023ad_ports[slave_id];
1628         return ACTOR_STATE(port, COLLECTING);
1629 }
1630
1631 int
1632 rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id,
1633                 struct rte_mbuf *lacp_pkt)
1634 {
1635         struct port *port;
1636         int res;
1637
1638         res = bond_8023ad_ext_validate(port_id, slave_id);
1639         if (res != 0)
1640                 return res;
1641
1642         port = &bond_mode_8023ad_ports[slave_id];
1643
1644         if (rte_pktmbuf_pkt_len(lacp_pkt) < sizeof(struct lacpdu_header))
1645                 return -EINVAL;
1646
1647         struct lacpdu_header *lacp;
1648
1649         /* only enqueue LACPDUs */
1650         lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
1651         if (lacp->lacpdu.subtype != SLOW_SUBTYPE_LACP)
1652                 return -EINVAL;
1653
1654         MODE4_DEBUG("sending LACP frame\n");
1655
1656         return rte_ring_enqueue(port->tx_ring, lacp_pkt);
1657 }
1658
1659 static void
1660 bond_mode_8023ad_ext_periodic_cb(void *arg)
1661 {
1662         struct rte_eth_dev *bond_dev = arg;
1663         struct bond_dev_private *internals = bond_dev->data->dev_private;
1664         struct mode8023ad_private *mode4 = &internals->mode4;
1665         struct port *port;
1666         void *pkt = NULL;
1667         uint16_t i, slave_id;
1668
1669         for (i = 0; i < internals->active_slave_count; i++) {
1670                 slave_id = internals->active_slaves[i];
1671                 port = &bond_mode_8023ad_ports[slave_id];
1672
1673                 if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) {
1674                         struct rte_mbuf *lacp_pkt = pkt;
1675                         struct lacpdu_header *lacp;
1676
1677                         lacp = rte_pktmbuf_mtod(lacp_pkt,
1678                                                 struct lacpdu_header *);
1679                         RTE_VERIFY(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
1680
1681                         /* This is LACP frame so pass it to rx callback.
1682                          * Callback is responsible for freeing mbuf.
1683                          */
1684                         mode4->slowrx_cb(slave_id, lacp_pkt);
1685                 }
1686         }
1687
1688         rte_eal_alarm_set(internals->mode4.update_timeout_us,
1689                         bond_mode_8023ad_ext_periodic_cb, arg);
1690 }
1691
1692 int
1693 rte_eth_bond_8023ad_dedicated_queues_enable(uint16_t port)
1694 {
1695         int retval = 0;
1696         struct rte_eth_dev *dev;
1697         struct bond_dev_private *internals;
1698
1699         if (valid_bonded_port_id(port) != 0)
1700                 return -EINVAL;
1701
1702         dev = &rte_eth_devices[port];
1703         internals = dev->data->dev_private;
1704
1705         if (bond_8023ad_slow_pkt_hw_filter_supported(port) != 0)
1706                 return -1;
1707
1708         /* Device must be stopped to set up slow queue */
1709         if (dev->data->dev_started)
1710                 return -1;
1711
1712         internals->mode4.dedicated_queues.enabled = 1;
1713
1714         bond_ethdev_mode_set(dev, internals->mode);
1715         return retval;
1716 }
1717
1718 int
1719 rte_eth_bond_8023ad_dedicated_queues_disable(uint16_t port)
1720 {
1721         int retval = 0;
1722         struct rte_eth_dev *dev;
1723         struct bond_dev_private *internals;
1724
1725         if (valid_bonded_port_id(port) != 0)
1726                 return -EINVAL;
1727
1728         dev = &rte_eth_devices[port];
1729         internals = dev->data->dev_private;
1730
1731         /* Device must be stopped to set up slow queue */
1732         if (dev->data->dev_started)
1733                 return -1;
1734
1735         internals->mode4.dedicated_queues.enabled = 0;
1736
1737         bond_ethdev_mode_set(dev, internals->mode);
1738
1739         return retval;
1740 }