drivers/net/bonding/rte_eth_bond_8023ad.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <stddef.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_alarm.h>
  10 #include <rte_malloc.h>
  11 #include <rte_errno.h>
  12 #include <rte_cycles.h>
  13 #include <rte_compat.h>
  14
  15 #include "eth_bond_private.h"
  16
  17 static void bond_mode_8023ad_ext_periodic_cb(void *arg);
  18 #ifdef RTE_LIBRTE_BOND_DEBUG_8023AD
  19
  20 #define MODE4_DEBUG(fmt, ...)                           \
  21         rte_log(RTE_LOG_DEBUG, bond_logtype,            \
  22                 "%6u [Port %u: %s] " fmt,               \
  23                 bond_dbg_get_time_diff_ms(), slave_id,  \
  24                 __func__, ##__VA_ARGS__)
  25
  26 static uint64_t start_time;
  27
  28 static unsigned
  29 bond_dbg_get_time_diff_ms(void)
  30 {
  31         uint64_t now;
  32
  33         now = rte_rdtsc();
  34         if (start_time == 0)
  35                 start_time = now;
  36
  37         return ((now - start_time) * 1000) / rte_get_tsc_hz();
  38 }
  39
  40 static void
  41 bond_print_lacp(struct lacpdu *l)
  42 {
  43         char a_address[18];
  44         char p_address[18];
  45         char a_state[256] = { 0 };
  46         char p_state[256] = { 0 };
  47
  48         static const char * const state_labels[] = {
  49                 "ACT", "TIMEOUT", "AGG", "SYNC", "COL", "DIST", "DEF", "EXP"
  50         };
  51
  52         int a_len = 0;
  53         int p_len = 0;
  54         uint8_t i;
  55         uint8_t *addr;
  56
  57         addr = l->actor.port_params.system.addr_bytes;
  58         snprintf(a_address, sizeof(a_address), RTE_ETHER_ADDR_PRT_FMT,
  59                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  60
  61         addr = l->partner.port_params.system.addr_bytes;
  62         snprintf(p_address, sizeof(p_address), RTE_ETHER_ADDR_PRT_FMT,
  63                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  64
  65         for (i = 0; i < 8; i++) {
  66                 if ((l->actor.state >> i) & 1) {
  67                         a_len += snprintf(&a_state[a_len], RTE_DIM(a_state) - a_len, "%s ",
  68                                 state_labels[i]);
  69                 }
  70
  71                 if ((l->partner.state >> i) & 1) {
  72                         p_len += snprintf(&p_state[p_len], RTE_DIM(p_state) - p_len, "%s ",
  73                                 state_labels[i]);
  74                 }
  75         }
  76
  77         if (a_len && a_state[a_len-1] == ' ')
  78                 a_state[a_len-1] = '\0';
  79
  80         if (p_len && p_state[p_len-1] == ' ')
  81                 p_state[p_len-1] = '\0';
  82
  83         RTE_BOND_LOG(DEBUG,
  84                      "LACP: {\n"
  85                      "  subtype= %02X\n"
  86                      "  ver_num=%02X\n"
  87                      "  actor={ tlv=%02X, len=%02X\n"
  88                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  89                      "       state={ %s }\n"
  90                      "  }\n"
  91                      "  partner={ tlv=%02X, len=%02X\n"
  92                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  93                      "       state={ %s }\n"
  94                      "  }\n"
  95                      "  collector={info=%02X, length=%02X, max_delay=%04X\n, "
  96                      "type_term=%02X, terminator_length = %02X }",
  97                      l->subtype,
  98                      l->version_number,
  99                      l->actor.tlv_type_info,
 100                      l->actor.info_length,
 101                      l->actor.port_params.system_priority,
 102                      a_address,
 103                      l->actor.port_params.key,
 104                      l->actor.port_params.port_priority,
 105                      l->actor.port_params.port_number,
 106                      a_state,
 107                      l->partner.tlv_type_info,
 108                      l->partner.info_length,
 109                      l->partner.port_params.system_priority,
 110                      p_address,
 111                      l->partner.port_params.key,
 112                      l->partner.port_params.port_priority,
 113                      l->partner.port_params.port_number,
 114                      p_state,
 115                      l->tlv_type_collector_info,
 116                      l->collector_info_length,
 117                      l->collector_max_delay,
 118                      l->tlv_type_terminator,
 119                      l->terminator_length);
 120
 121 }
 122
 123 #define BOND_PRINT_LACP(lacpdu) bond_print_lacp(lacpdu)
 124 #else
 125 #define BOND_PRINT_LACP(lacpdu) do { } while (0)
 126 #define MODE4_DEBUG(fmt, ...) do { } while (0)
 127 #endif
 128
 129 static const struct rte_ether_addr lacp_mac_addr = {
 130         .addr_bytes = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 }
 131 };
 132
 133 struct port bond_mode_8023ad_ports[RTE_MAX_ETHPORTS];
 134
 135 static void
 136 timer_cancel(uint64_t *timer)
 137 {
 138         *timer = 0;
 139 }
 140
 141 static void
 142 timer_set(uint64_t *timer, uint64_t timeout)
 143 {
 144         *timer = rte_rdtsc() + timeout;
 145 }
 146
 147 /* Forces given timer to be in expired state. */
 148 static void
 149 timer_force_expired(uint64_t *timer)
 150 {
 151         *timer = rte_rdtsc();
 152 }
 153
 154 static bool
 155 timer_is_stopped(uint64_t *timer)
 156 {
 157         return *timer == 0;
 158 }
 159
 160 static bool
 161 timer_is_expired(uint64_t *timer)
 162 {
 163         return *timer < rte_rdtsc();
 164 }
 165
 166 /* Timer is in running state if it is not stopped nor expired */
 167 static bool
 168 timer_is_running(uint64_t *timer)
 169 {
 170         return !timer_is_stopped(timer) && !timer_is_expired(timer);
 171 }
 172
 173 static void
 174 set_warning_flags(struct port *port, uint16_t flags)
 175 {
 176         int retval;
 177         uint16_t old;
 178         uint16_t new_flag = 0;
 179
 180         do {
 181                 old = port->warnings_to_show;
 182                 new_flag = old | flags;
 183                 retval = rte_atomic16_cmpset(&port->warnings_to_show, old, new_flag);
 184         } while (unlikely(retval == 0));
 185 }
 186
 187 static void
 188 show_warnings(uint16_t slave_id)
 189 {
 190         struct port *port = &bond_mode_8023ad_ports[slave_id];
 191         uint8_t warnings;
 192
 193         do {
 194                 warnings = port->warnings_to_show;
 195         } while (rte_atomic16_cmpset(&port->warnings_to_show, warnings, 0) == 0);
 196
 197         if (!warnings)
 198                 return;
 199
 200         if (!timer_is_expired(&port->warning_timer))
 201                 return;
 202
 203
 204         timer_set(&port->warning_timer, BOND_8023AD_WARNINGS_PERIOD_MS *
 205                         rte_get_tsc_hz() / 1000);
 206
 207         if (warnings & WRN_RX_QUEUE_FULL) {
 208                 RTE_BOND_LOG(DEBUG,
 209                              "Slave %u: failed to enqueue LACP packet into RX ring.\n"
 210                              "Receive and transmit functions must be invoked on bonded"
 211                              "interface at least 10 times per second or LACP will notwork correctly",
 212                              slave_id);
 213         }
 214
 215         if (warnings & WRN_TX_QUEUE_FULL) {
 216                 RTE_BOND_LOG(DEBUG,
 217                              "Slave %u: failed to enqueue LACP packet into TX ring.\n"
 218                              "Receive and transmit functions must be invoked on bonded"
 219                              "interface at least 10 times per second or LACP will not work correctly",
 220                              slave_id);
 221         }
 222
 223         if (warnings & WRN_RX_MARKER_TO_FAST)
 224                 RTE_BOND_LOG(INFO, "Slave %u: marker to early - ignoring.",
 225                              slave_id);
 226
 227         if (warnings & WRN_UNKNOWN_SLOW_TYPE) {
 228                 RTE_BOND_LOG(INFO,
 229                         "Slave %u: ignoring unknown slow protocol frame type",
 230                              slave_id);
 231         }
 232
 233         if (warnings & WRN_UNKNOWN_MARKER_TYPE)
 234                 RTE_BOND_LOG(INFO, "Slave %u: ignoring unknown marker type",
 235                              slave_id);
 236
 237         if (warnings & WRN_NOT_LACP_CAPABLE)
 238                 MODE4_DEBUG("Port %u is not LACP capable!\n", slave_id);
 239 }
 240
 241 static void
 242 record_default(struct port *port)
 243 {
 244         /* Record default parameters for partner. Partner admin parameters
 245          * are not implemented so set them to arbitrary default (last known) and
 246          * mark actor that parner is in defaulted state. */
 247         port->partner_state = STATE_LACP_ACTIVE;
 248         ACTOR_STATE_SET(port, DEFAULTED);
 249 }
 250
 251 /** Function handles rx state machine.
 252  *
 253  * This function implements Receive State Machine from point 5.4.12 in
 254  * 802.1AX documentation. It should be called periodically.
 255  *
 256  * @param lacpdu                LACPDU received.
 257  * @param port                  Port on which LACPDU was received.
 258  */
 259 static void
 260 rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 261                 struct lacpdu *lacp)
 262 {
 263         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 264         uint64_t timeout;
 265
 266         if (SM_FLAG(port, BEGIN)) {
 267                 /* Initialize stuff */
 268                 MODE4_DEBUG("-> INITIALIZE\n");
 269                 SM_FLAG_CLR(port, MOVED);
 270                 port->selected = UNSELECTED;
 271
 272                 record_default(port);
 273
 274                 ACTOR_STATE_CLR(port, EXPIRED);
 275                 timer_cancel(&port->current_while_timer);
 276
 277                 /* DISABLED: On initialization partner is out of sync */
 278                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 279
 280                 /* LACP DISABLED stuff if LACP not enabled on this port */
 281                 if (!SM_FLAG(port, LACP_ENABLED))
 282                         PARTNER_STATE_CLR(port, AGGREGATION);
 283                 else
 284                         PARTNER_STATE_SET(port, AGGREGATION);
 285         }
 286
 287         if (!SM_FLAG(port, LACP_ENABLED)) {
 288                 /* Update parameters only if state changed */
 289                 if (!timer_is_stopped(&port->current_while_timer)) {
 290                         port->selected = UNSELECTED;
 291                         record_default(port);
 292                         PARTNER_STATE_CLR(port, AGGREGATION);
 293                         ACTOR_STATE_CLR(port, EXPIRED);
 294                         timer_cancel(&port->current_while_timer);
 295                 }
 296                 return;
 297         }
 298
 299         if (lacp) {
 300                 MODE4_DEBUG("LACP -> CURRENT\n");
 301                 BOND_PRINT_LACP(lacp);
 302                 /* Update selected flag. If partner parameters are defaulted assume they
 303                  * are match. If not defaulted  compare LACP actor with ports parner
 304                  * params. */
 305                 if (!ACTOR_STATE(port, DEFAULTED) &&
 306                         (ACTOR_STATE(port, AGGREGATION) != PARTNER_STATE(port, AGGREGATION)
 307                         || memcmp(&port->partner, &lacp->actor.port_params,
 308                                 sizeof(port->partner)) != 0)) {
 309                         MODE4_DEBUG("selected <- UNSELECTED\n");
 310                         port->selected = UNSELECTED;
 311                 }
 312
 313                 /* Record this PDU actor params as partner params */
 314                 memcpy(&port->partner, &lacp->actor.port_params,
 315                         sizeof(struct port_params));
 316                 port->partner_state = lacp->actor.state;
 317
 318                 /* Partner parameters are not defaulted any more */
 319                 ACTOR_STATE_CLR(port, DEFAULTED);
 320
 321                 /* If LACP partner params match this port actor params */
 322                 agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 323                 bool match = port->actor.system_priority ==
 324                         lacp->partner.port_params.system_priority &&
 325                         rte_is_same_ether_addr(&agg->actor.system,
 326                         &lacp->partner.port_params.system) &&
 327                         port->actor.port_priority ==
 328                         lacp->partner.port_params.port_priority &&
 329                         port->actor.port_number ==
 330                         lacp->partner.port_params.port_number;
 331
 332                 /* Update NTT if partners information are outdated (xored and masked
 333                  * bits are set)*/
 334                 uint8_t state_mask = STATE_LACP_ACTIVE | STATE_LACP_SHORT_TIMEOUT |
 335                         STATE_SYNCHRONIZATION | STATE_AGGREGATION;
 336
 337                 if (((port->actor_state ^ lacp->partner.state) & state_mask) ||
 338                                 match == false) {
 339                         SM_FLAG_SET(port, NTT);
 340                 }
 341
 342                 /* If LACP partner params match this port actor params */
 343                 if (match == true && ACTOR_STATE(port, AGGREGATION) ==
 344                                 PARTNER_STATE(port,     AGGREGATION))
 345                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 346                 else if (!PARTNER_STATE(port, AGGREGATION) && ACTOR_STATE(port,
 347                                 AGGREGATION))
 348                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 349                 else
 350                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 351
 352                 if (ACTOR_STATE(port, LACP_SHORT_TIMEOUT))
 353                         timeout = internals->mode4.short_timeout;
 354                 else
 355                         timeout = internals->mode4.long_timeout;
 356
 357                 timer_set(&port->current_while_timer, timeout);
 358                 ACTOR_STATE_CLR(port, EXPIRED);
 359                 SM_FLAG_CLR(port, EXPIRED);
 360                 return; /* No state change */
 361         }
 362
 363         /* If CURRENT state timer is not running (stopped or expired)
 364          * transit to EXPIRED state from DISABLED or CURRENT */
 365         if (!timer_is_running(&port->current_while_timer)) {
 366                 if (SM_FLAG(port, EXPIRED)) {
 367                         port->selected = UNSELECTED;
 368                         memcpy(&port->partner, &port->partner_admin,
 369                                 sizeof(struct port_params));
 370                         record_default(port);
 371                         ACTOR_STATE_CLR(port, EXPIRED);
 372                         timer_cancel(&port->current_while_timer);
 373                 } else {
 374                         SM_FLAG_SET(port, EXPIRED);
 375                         ACTOR_STATE_SET(port, EXPIRED);
 376                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 377                         PARTNER_STATE_SET(port, LACP_SHORT_TIMEOUT);
 378                         timer_set(&port->current_while_timer,
 379                                 internals->mode4.short_timeout);
 380                 }
 381         }
 382 }
 383
 384 /**
 385  * Function handles periodic tx state machine.
 386  *
 387  * Function implements Periodic Transmission state machine from point 5.4.13
 388  * in 802.1AX documentation. It should be called periodically.
 389  *
 390  * @param port                  Port to handle state machine.
 391  */
 392 static void
 393 periodic_machine(struct bond_dev_private *internals, uint16_t slave_id)
 394 {
 395         struct port *port = &bond_mode_8023ad_ports[slave_id];
 396         /* Calculate if either site is LACP enabled */
 397         uint64_t timeout;
 398         uint8_t active = ACTOR_STATE(port, LACP_ACTIVE) ||
 399                 PARTNER_STATE(port, LACP_ACTIVE);
 400
 401         uint8_t is_partner_fast, was_partner_fast;
 402         /* No periodic is on BEGIN, LACP DISABLE or when both sides are pasive */
 403         if (SM_FLAG(port, BEGIN) || !SM_FLAG(port, LACP_ENABLED) || !active) {
 404                 timer_cancel(&port->periodic_timer);
 405                 timer_force_expired(&port->tx_machine_timer);
 406                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 407
 408                 MODE4_DEBUG("-> NO_PERIODIC ( %s%s%s)\n",
 409                         SM_FLAG(port, BEGIN) ? "begind " : "",
 410                         SM_FLAG(port, LACP_ENABLED) ? "" : "LACP disabled ",
 411                         active ? "LACP active " : "LACP pasive ");
 412                 return;
 413         }
 414
 415         is_partner_fast = PARTNER_STATE(port, LACP_SHORT_TIMEOUT);
 416         was_partner_fast = SM_FLAG(port, PARTNER_SHORT_TIMEOUT);
 417
 418         /* If periodic timer is not started, transit from NO PERIODIC to FAST/SLOW.
 419          * Other case: check if timer expire or partners settings changed. */
 420         if (!timer_is_stopped(&port->periodic_timer)) {
 421                 if (timer_is_expired(&port->periodic_timer)) {
 422                         SM_FLAG_SET(port, NTT);
 423                 } else if (is_partner_fast != was_partner_fast) {
 424                         /* Partners timeout  was slow and now it is fast -> send LACP.
 425                          * In other case (was fast and now it is slow) just switch
 426                          * timeout to slow without forcing send of LACP (because standard
 427                          * say so)*/
 428                         if (is_partner_fast)
 429                                 SM_FLAG_SET(port, NTT);
 430                 } else
 431                         return; /* Nothing changed */
 432         }
 433
 434         /* Handle state transition to FAST/SLOW LACP timeout */
 435         if (is_partner_fast) {
 436                 timeout = internals->mode4.fast_periodic_timeout;
 437                 SM_FLAG_SET(port, PARTNER_SHORT_TIMEOUT);
 438         } else {
 439                 timeout = internals->mode4.slow_periodic_timeout;
 440                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 441         }
 442
 443         timer_set(&port->periodic_timer, timeout);
 444 }
 445
 446 /**
 447  * Function handles mux state machine.
 448  *
 449  * Function implements Mux Machine from point 5.4.15 in 802.1AX documentation.
 450  * It should be called periodically.
 451  *
 452  * @param port                  Port to handle state machine.
 453  */
 454 static void
 455 mux_machine(struct bond_dev_private *internals, uint16_t slave_id)
 456 {
 457         struct port *port = &bond_mode_8023ad_ports[slave_id];
 458
 459         /* Save current state for later use */
 460         const uint8_t state_mask = STATE_SYNCHRONIZATION | STATE_DISTRIBUTING |
 461                 STATE_COLLECTING;
 462
 463         /* Enter DETACHED state on BEGIN condition or from any other state if
 464          * port was unselected */
 465         if (SM_FLAG(port, BEGIN) ||
 466                         port->selected == UNSELECTED || (port->selected == STANDBY &&
 467                                 (port->actor_state & state_mask) != 0)) {
 468                 /* detach mux from aggregator */
 469                 port->actor_state &= ~state_mask;
 470                 /* Set ntt to true if BEGIN condition or transition from any other state
 471                  * which is indicated that wait_while_timer was started */
 472                 if (SM_FLAG(port, BEGIN) ||
 473                                 !timer_is_stopped(&port->wait_while_timer)) {
 474                         SM_FLAG_SET(port, NTT);
 475                         MODE4_DEBUG("-> DETACHED\n");
 476                 }
 477                 timer_cancel(&port->wait_while_timer);
 478         }
 479
 480         if (timer_is_stopped(&port->wait_while_timer)) {
 481                 if (port->selected == SELECTED || port->selected == STANDBY) {
 482                         timer_set(&port->wait_while_timer,
 483                                 internals->mode4.aggregate_wait_timeout);
 484
 485                         MODE4_DEBUG("DETACHED -> WAITING\n");
 486                 }
 487                 /* Waiting state entered */
 488                 return;
 489         }
 490
 491         /* Transit next state if port is ready */
 492         if (!timer_is_expired(&port->wait_while_timer))
 493                 return;
 494
 495         if ((ACTOR_STATE(port, DISTRIBUTING) || ACTOR_STATE(port, COLLECTING)) &&
 496                 !PARTNER_STATE(port, SYNCHRONIZATION)) {
 497                 /* If in COLLECTING or DISTRIBUTING state and partner becomes out of
 498                  * sync transit to ATACHED state.  */
 499                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 500                 ACTOR_STATE_CLR(port, COLLECTING);
 501                 /* Clear actor sync to activate transit ATACHED in condition bellow */
 502                 ACTOR_STATE_CLR(port, SYNCHRONIZATION);
 503                 MODE4_DEBUG("Out of sync -> ATTACHED\n");
 504         }
 505
 506         if (!ACTOR_STATE(port, SYNCHRONIZATION)) {
 507                 /* attach mux to aggregator */
 508                 RTE_ASSERT((port->actor_state & (STATE_COLLECTING |
 509                         STATE_DISTRIBUTING)) == 0);
 510
 511                 ACTOR_STATE_SET(port, SYNCHRONIZATION);
 512                 SM_FLAG_SET(port, NTT);
 513                 MODE4_DEBUG("ATTACHED Entered\n");
 514         } else if (!ACTOR_STATE(port, COLLECTING)) {
 515                 /* Start collecting if in sync */
 516                 if (PARTNER_STATE(port, SYNCHRONIZATION)) {
 517                         MODE4_DEBUG("ATTACHED -> COLLECTING\n");
 518                         ACTOR_STATE_SET(port, COLLECTING);
 519                         SM_FLAG_SET(port, NTT);
 520                 }
 521         } else if (ACTOR_STATE(port, COLLECTING)) {
 522                 /* Check if partner is in COLLECTING state. If so this port can
 523                  * distribute frames to it */
 524                 if (!ACTOR_STATE(port, DISTRIBUTING)) {
 525                         if (PARTNER_STATE(port, COLLECTING)) {
 526                                 /* Enable  DISTRIBUTING if partner is collecting */
 527                                 ACTOR_STATE_SET(port, DISTRIBUTING);
 528                                 SM_FLAG_SET(port, NTT);
 529                                 MODE4_DEBUG("COLLECTING -> DISTRIBUTING\n");
 530                                 RTE_BOND_LOG(INFO,
 531                                         "Bond %u: slave id %u distributing started.",
 532                                         internals->port_id, slave_id);
 533                         }
 534                 } else {
 535                         if (!PARTNER_STATE(port, COLLECTING)) {
 536                                 /* Disable DISTRIBUTING (enter COLLECTING state) if partner
 537                                  * is not collecting */
 538                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 539                                 SM_FLAG_SET(port, NTT);
 540                                 MODE4_DEBUG("DISTRIBUTING -> COLLECTING\n");
 541                                 RTE_BOND_LOG(INFO,
 542                                         "Bond %u: slave id %u distributing stopped.",
 543                                         internals->port_id, slave_id);
 544                         }
 545                 }
 546         }
 547 }
 548
 549 /**
 550  * Function handles transmit state machine.
 551  *
 552  * Function implements Transmit Machine from point 5.4.16 in 802.1AX
 553  * documentation.
 554  *
 555  * @param port
 556  */
 557 static void
 558 tx_machine(struct bond_dev_private *internals, uint16_t slave_id)
 559 {
 560         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 561
 562         struct rte_mbuf *lacp_pkt = NULL;
 563         struct lacpdu_header *hdr;
 564         struct lacpdu *lacpdu;
 565
 566         /* If periodic timer is not running periodic machine is in NO PERIODIC and
 567          * according to 802.3ax standard tx machine should not transmit any frames
 568          * and set ntt to false. */
 569         if (timer_is_stopped(&port->periodic_timer))
 570                 SM_FLAG_CLR(port, NTT);
 571
 572         if (!SM_FLAG(port, NTT))
 573                 return;
 574
 575         if (!timer_is_expired(&port->tx_machine_timer))
 576                 return;
 577
 578         lacp_pkt = rte_pktmbuf_alloc(port->mbuf_pool);
 579         if (lacp_pkt == NULL) {
 580                 RTE_BOND_LOG(ERR, "Failed to allocate LACP packet from pool");
 581                 return;
 582         }
 583
 584         lacp_pkt->data_len = sizeof(*hdr);
 585         lacp_pkt->pkt_len = sizeof(*hdr);
 586
 587         hdr = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 588
 589         /* Source and destination MAC */
 590         rte_ether_addr_copy(&lacp_mac_addr, &hdr->eth_hdr.dst_addr);
 591         rte_eth_macaddr_get(slave_id, &hdr->eth_hdr.src_addr);
 592         hdr->eth_hdr.ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_SLOW);
 593
 594         lacpdu = &hdr->lacpdu;
 595         memset(lacpdu, 0, sizeof(*lacpdu));
 596
 597         /* Initialize LACP part */
 598         lacpdu->subtype = SLOW_SUBTYPE_LACP;
 599         lacpdu->version_number = 1;
 600
 601         /* ACTOR */
 602         lacpdu->actor.tlv_type_info = TLV_TYPE_ACTOR_INFORMATION;
 603         lacpdu->actor.info_length = sizeof(struct lacpdu_actor_partner_params);
 604         memcpy(&hdr->lacpdu.actor.port_params, &port->actor,
 605                         sizeof(port->actor));
 606         agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 607         rte_ether_addr_copy(&agg->actor.system,
 608                         &hdr->lacpdu.actor.port_params.system);
 609         lacpdu->actor.state = port->actor_state;
 610
 611         /* PARTNER */
 612         lacpdu->partner.tlv_type_info = TLV_TYPE_PARTNER_INFORMATION;
 613         lacpdu->partner.info_length = sizeof(struct lacpdu_actor_partner_params);
 614         memcpy(&lacpdu->partner.port_params, &port->partner,
 615                         sizeof(struct port_params));
 616         lacpdu->partner.state = port->partner_state;
 617
 618         /* Other fields */
 619         lacpdu->tlv_type_collector_info = TLV_TYPE_COLLECTOR_INFORMATION;
 620         lacpdu->collector_info_length = 0x10;
 621         lacpdu->collector_max_delay = 0;
 622
 623         lacpdu->tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION;
 624         lacpdu->terminator_length = 0;
 625
 626         MODE4_DEBUG("Sending LACP frame\n");
 627         BOND_PRINT_LACP(lacpdu);
 628
 629         if (internals->mode4.dedicated_queues.enabled == 0) {
 630                 int retval = rte_ring_enqueue(port->tx_ring, lacp_pkt);
 631                 if (retval != 0) {
 632                         /* If TX ring full, drop packet and free message.
 633                            Retransmission will happen in next function call. */
 634                         rte_pktmbuf_free(lacp_pkt);
 635                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 636                         return;
 637                 }
 638         } else {
 639                 uint16_t pkts_sent = rte_eth_tx_burst(slave_id,
 640                                 internals->mode4.dedicated_queues.tx_qid,
 641                                 &lacp_pkt, 1);
 642                 if (pkts_sent != 1) {
 643                         rte_pktmbuf_free(lacp_pkt);
 644                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 645                         return;
 646                 }
 647         }
 648
 649
 650         timer_set(&port->tx_machine_timer, internals->mode4.tx_period_timeout);
 651         SM_FLAG_CLR(port, NTT);
 652 }
 653
 654 static uint16_t
 655 max_index(uint64_t *a, int n)
 656 {
 657         if (n <= 0)
 658                 return -1;
 659
 660         int i, max_i = 0;
 661         uint64_t max = a[0];
 662
 663         for (i = 1; i < n; ++i) {
 664                 if (a[i] > max) {
 665                         max = a[i];
 666                         max_i = i;
 667                 }
 668         }
 669
 670         return max_i;
 671 }
 672
 673 /**
 674  * Function assigns port to aggregator.
 675  *
 676  * @param bond_dev_private      Pointer to bond_dev_private structure.
 677  * @param port_pos                      Port to assign.
 678  */
 679 static void
 680 selection_logic(struct bond_dev_private *internals, uint16_t slave_id)
 681 {
 682         struct port *agg, *port;
 683         uint16_t slaves_count, new_agg_id, i, j = 0;
 684         uint16_t *slaves;
 685         uint64_t agg_bandwidth[RTE_MAX_ETHPORTS] = {0};
 686         uint64_t agg_count[RTE_MAX_ETHPORTS] = {0};
 687         uint16_t default_slave = 0;
 688         struct rte_eth_link link_info;
 689         uint16_t agg_new_idx = 0;
 690         int ret;
 691
 692         slaves = internals->active_slaves;
 693         slaves_count = internals->active_slave_count;
 694         port = &bond_mode_8023ad_ports[slave_id];
 695
 696         /* Search for aggregator suitable for this port */
 697         for (i = 0; i < slaves_count; ++i) {
 698                 agg = &bond_mode_8023ad_ports[slaves[i]];
 699                 /* Skip ports that are not aggreagators */
 700                 if (agg->aggregator_port_id != slaves[i])
 701                         continue;
 702
 703                 ret = rte_eth_link_get_nowait(slaves[i], &link_info);
 704                 if (ret < 0) {
 705                         RTE_BOND_LOG(ERR,
 706                                 "Slave (port %u) link get failed: %s\n",
 707                                 slaves[i], rte_strerror(-ret));
 708                         continue;
 709                 }
 710                 agg_count[i] += 1;
 711                 agg_bandwidth[i] += link_info.link_speed;
 712
 713                 /* Actors system ID is not checked since all slave device have the same
 714                  * ID (MAC address). */
 715                 if ((agg->actor.key == port->actor.key &&
 716                         agg->partner.system_priority == port->partner.system_priority &&
 717                         rte_is_same_ether_addr(&agg->partner.system,
 718                                         &port->partner.system) == 1
 719                         && (agg->partner.key == port->partner.key)) &&
 720                         rte_is_zero_ether_addr(&port->partner.system) != 1 &&
 721                         (agg->actor.key &
 722                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) != 0) {
 723
 724                         if (j == 0)
 725                                 default_slave = i;
 726                         j++;
 727                 }
 728         }
 729
 730         switch (internals->mode4.agg_selection) {
 731         case AGG_COUNT:
 732                 agg_new_idx = max_index(agg_count, slaves_count);
 733                 new_agg_id = slaves[agg_new_idx];
 734                 break;
 735         case AGG_BANDWIDTH:
 736                 agg_new_idx = max_index(agg_bandwidth, slaves_count);
 737                 new_agg_id = slaves[agg_new_idx];
 738                 break;
 739         case AGG_STABLE:
 740                 if (default_slave == slaves_count)
 741                         new_agg_id = slaves[slave_id];
 742                 else
 743                         new_agg_id = slaves[default_slave];
 744                 break;
 745         default:
 746                 if (default_slave == slaves_count)
 747                         new_agg_id = slaves[slave_id];
 748                 else
 749                         new_agg_id = slaves[default_slave];
 750                 break;
 751         }
 752
 753         if (new_agg_id != port->aggregator_port_id) {
 754                 port->aggregator_port_id = new_agg_id;
 755
 756                 MODE4_DEBUG("-> SELECTED: ID=%3u\n"
 757                         "\t%s aggregator ID=%3u\n",
 758                         port->aggregator_port_id,
 759                         port->aggregator_port_id == slave_id ?
 760                                 "aggregator not found, using default" : "aggregator found",
 761                         port->aggregator_port_id);
 762         }
 763
 764         port->selected = SELECTED;
 765 }
 766
 767 /* Function maps DPDK speed to bonding speed stored in key field */
 768 static uint16_t
 769 link_speed_key(uint16_t speed) {
 770         uint16_t key_speed;
 771
 772         switch (speed) {
 773         case RTE_ETH_SPEED_NUM_NONE:
 774                 key_speed = 0x00;
 775                 break;
 776         case RTE_ETH_SPEED_NUM_10M:
 777                 key_speed = BOND_LINK_SPEED_KEY_10M;
 778                 break;
 779         case RTE_ETH_SPEED_NUM_100M:
 780                 key_speed = BOND_LINK_SPEED_KEY_100M;
 781                 break;
 782         case RTE_ETH_SPEED_NUM_1G:
 783                 key_speed = BOND_LINK_SPEED_KEY_1000M;
 784                 break;
 785         case RTE_ETH_SPEED_NUM_10G:
 786                 key_speed = BOND_LINK_SPEED_KEY_10G;
 787                 break;
 788         case RTE_ETH_SPEED_NUM_20G:
 789                 key_speed = BOND_LINK_SPEED_KEY_20G;
 790                 break;
 791         case RTE_ETH_SPEED_NUM_40G:
 792                 key_speed = BOND_LINK_SPEED_KEY_40G;
 793                 break;
 794         default:
 795                 /* Unknown speed*/
 796                 key_speed = 0xFFFF;
 797         }
 798
 799         return key_speed;
 800 }
 801
 802 static void
 803 rx_machine_update(struct bond_dev_private *internals, uint16_t slave_id,
 804                 struct rte_mbuf *lacp_pkt) {
 805         struct lacpdu_header *lacp;
 806         struct lacpdu_actor_partner_params *partner;
 807         struct port *port, *agg;
 808
 809         if (lacp_pkt != NULL) {
 810                 lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 811                 RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
 812
 813                 partner = &lacp->lacpdu.partner;
 814                 port = &bond_mode_8023ad_ports[slave_id];
 815                 agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 816
 817                 if (rte_is_zero_ether_addr(&partner->port_params.system) ||
 818                         rte_is_same_ether_addr(&partner->port_params.system,
 819                                 &agg->actor.system)) {
 820                         /* This LACP frame is sending to the bonding port
 821                          * so pass it to rx_machine.
 822                          */
 823                         rx_machine(internals, slave_id, &lacp->lacpdu);
 824                 } else {
 825                         char preferred_system_name[RTE_ETHER_ADDR_FMT_SIZE];
 826                         char self_system_name[RTE_ETHER_ADDR_FMT_SIZE];
 827
 828                         rte_ether_format_addr(preferred_system_name,
 829                                 RTE_ETHER_ADDR_FMT_SIZE, &partner->port_params.system);
 830                         rte_ether_format_addr(self_system_name,
 831                                 RTE_ETHER_ADDR_FMT_SIZE, &agg->actor.system);
 832                         MODE4_DEBUG("preferred partner system %s "
 833                                 "is not equal with self system: %s\n",
 834                                 preferred_system_name, self_system_name);
 835                 }
 836                 rte_pktmbuf_free(lacp_pkt);
 837         } else
 838                 rx_machine(internals, slave_id, NULL);
 839 }
 840
 841 static void
 842 bond_mode_8023ad_dedicated_rxq_process(struct bond_dev_private *internals,
 843                         uint16_t slave_id)
 844 {
 845 #define DEDICATED_QUEUE_BURST_SIZE 32
 846         struct rte_mbuf *lacp_pkt[DEDICATED_QUEUE_BURST_SIZE];
 847         uint16_t rx_count = rte_eth_rx_burst(slave_id,
 848                                 internals->mode4.dedicated_queues.rx_qid,
 849                                 lacp_pkt, DEDICATED_QUEUE_BURST_SIZE);
 850
 851         if (rx_count) {
 852                 uint16_t i;
 853
 854                 for (i = 0; i < rx_count; i++)
 855                         bond_mode_8023ad_handle_slow_pkt(internals, slave_id,
 856                                         lacp_pkt[i]);
 857         } else {
 858                 rx_machine_update(internals, slave_id, NULL);
 859         }
 860 }
 861
 862 static void
 863 bond_mode_8023ad_periodic_cb(void *arg)
 864 {
 865         struct rte_eth_dev *bond_dev = arg;
 866         struct bond_dev_private *internals = bond_dev->data->dev_private;
 867         struct port *port;
 868         struct rte_eth_link link_info;
 869         struct rte_ether_addr slave_addr;
 870         struct rte_mbuf *lacp_pkt = NULL;
 871         uint16_t slave_id;
 872         uint16_t i;
 873
 874
 875         /* Update link status on each port */
 876         for (i = 0; i < internals->active_slave_count; i++) {
 877                 uint16_t key;
 878                 int ret;
 879
 880                 slave_id = internals->active_slaves[i];
 881                 ret = rte_eth_link_get_nowait(slave_id, &link_info);
 882                 if (ret < 0) {
 883                         RTE_BOND_LOG(ERR,
 884                                 "Slave (port %u) link get failed: %s\n",
 885                                 slave_id, rte_strerror(-ret));
 886                 }
 887
 888                 if (ret >= 0 && link_info.link_status != 0) {
 889                         key = link_speed_key(link_info.link_speed) << 1;
 890                         if (link_info.link_duplex == RTE_ETH_LINK_FULL_DUPLEX)
 891                                 key |= BOND_LINK_FULL_DUPLEX_KEY;
 892                 } else {
 893                         key = 0;
 894                 }
 895
 896                 rte_eth_macaddr_get(slave_id, &slave_addr);
 897                 port = &bond_mode_8023ad_ports[slave_id];
 898
 899                 key = rte_cpu_to_be_16(key);
 900                 if (key != port->actor.key) {
 901                         if (!(key & rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)))
 902                                 set_warning_flags(port, WRN_NOT_LACP_CAPABLE);
 903
 904                         port->actor.key = key;
 905                         SM_FLAG_SET(port, NTT);
 906                 }
 907
 908                 if (!rte_is_same_ether_addr(&port->actor.system, &slave_addr)) {
 909                         rte_ether_addr_copy(&slave_addr, &port->actor.system);
 910                         if (port->aggregator_port_id == slave_id)
 911                                 SM_FLAG_SET(port, NTT);
 912                 }
 913         }
 914
 915         for (i = 0; i < internals->active_slave_count; i++) {
 916                 slave_id = internals->active_slaves[i];
 917                 port = &bond_mode_8023ad_ports[slave_id];
 918
 919                 if ((port->actor.key &
 920                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) == 0) {
 921
 922                         SM_FLAG_SET(port, BEGIN);
 923
 924                         /* LACP is disabled on half duples or link is down */
 925                         if (SM_FLAG(port, LACP_ENABLED)) {
 926                                 /* If port was enabled set it to BEGIN state */
 927                                 SM_FLAG_CLR(port, LACP_ENABLED);
 928                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 929                                 ACTOR_STATE_CLR(port, COLLECTING);
 930                         }
 931
 932                         /* Skip this port processing */
 933                         continue;
 934                 }
 935
 936                 SM_FLAG_SET(port, LACP_ENABLED);
 937
 938                 if (internals->mode4.dedicated_queues.enabled == 0) {
 939                         /* Find LACP packet to this port. Do not check subtype,
 940                          * it is done in function that queued packet
 941                          */
 942                         int retval = rte_ring_dequeue(port->rx_ring,
 943                                         (void **)&lacp_pkt);
 944
 945                         if (retval != 0)
 946                                 lacp_pkt = NULL;
 947
 948                         rx_machine_update(internals, slave_id, lacp_pkt);
 949                 } else {
 950                         bond_mode_8023ad_dedicated_rxq_process(internals,
 951                                         slave_id);
 952                 }
 953
 954                 periodic_machine(internals, slave_id);
 955                 mux_machine(internals, slave_id);
 956                 tx_machine(internals, slave_id);
 957                 selection_logic(internals, slave_id);
 958
 959                 SM_FLAG_CLR(port, BEGIN);
 960                 show_warnings(slave_id);
 961         }
 962
 963         rte_eal_alarm_set(internals->mode4.update_timeout_us,
 964                         bond_mode_8023ad_periodic_cb, arg);
 965 }
 966
 967 static int
 968 bond_mode_8023ad_register_lacp_mac(uint16_t slave_id)
 969 {
 970         int ret;
 971
 972         ret = rte_eth_allmulticast_enable(slave_id);
 973         if (ret != 0) {
 974                 RTE_BOND_LOG(ERR,
 975                         "failed to enable allmulti mode for port %u: %s",
 976                         slave_id, rte_strerror(-ret));
 977         }
 978         if (rte_eth_allmulticast_get(slave_id)) {
 979                 RTE_BOND_LOG(DEBUG, "forced allmulti for port %u",
 980                              slave_id);
 981                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 982                                 BOND_8023AD_FORCED_ALLMULTI;
 983                 return 0;
 984         }
 985
 986         ret = rte_eth_promiscuous_enable(slave_id);
 987         if (ret != 0) {
 988                 RTE_BOND_LOG(ERR,
 989                         "failed to enable promiscuous mode for port %u: %s",
 990                         slave_id, rte_strerror(-ret));
 991         }
 992         if (rte_eth_promiscuous_get(slave_id)) {
 993                 RTE_BOND_LOG(DEBUG, "forced promiscuous for port %u",
 994                              slave_id);
 995                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 996                                 BOND_8023AD_FORCED_PROMISC;
 997                 return 0;
 998         }
 999
1000         return -1;
1001 }
1002
1003 static void
1004 bond_mode_8023ad_unregister_lacp_mac(uint16_t slave_id)
1005 {
1006         int ret;
1007
1008         switch (bond_mode_8023ad_ports[slave_id].forced_rx_flags) {
1009         case BOND_8023AD_FORCED_ALLMULTI:
1010                 RTE_BOND_LOG(DEBUG, "unset allmulti for port %u", slave_id);
1011                 ret = rte_eth_allmulticast_disable(slave_id);
1012                 if (ret != 0)
1013                         RTE_BOND_LOG(ERR,
1014                                 "failed to disable allmulti mode for port %u: %s",
1015                                 slave_id, rte_strerror(-ret));
1016                 break;
1017
1018         case BOND_8023AD_FORCED_PROMISC:
1019                 RTE_BOND_LOG(DEBUG, "unset promisc for port %u", slave_id);
1020                 ret = rte_eth_promiscuous_disable(slave_id);
1021                 if (ret != 0)
1022                         RTE_BOND_LOG(ERR,
1023                                 "failed to disable promiscuous mode for port %u: %s",
1024                                 slave_id, rte_strerror(-ret));
1025                 break;
1026
1027         default:
1028                 break;
1029         }
1030 }
1031
1032 void
1033 bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev,
1034                                 uint16_t slave_id)
1035 {
1036         struct bond_dev_private *internals = bond_dev->data->dev_private;
1037
1038         struct port *port = &bond_mode_8023ad_ports[slave_id];
1039         struct port_params initial = {
1040                         .system = { { 0 } },
1041                         .system_priority = rte_cpu_to_be_16(0xFFFF),
1042                         .key = rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY),
1043                         .port_priority = rte_cpu_to_be_16(0x00FF),
1044                         .port_number = 0,
1045         };
1046
1047         char mem_name[RTE_ETH_NAME_MAX_LEN];
1048         int socket_id;
1049         unsigned element_size;
1050         uint32_t total_tx_desc;
1051         struct bond_tx_queue *bd_tx_q;
1052         uint16_t q_id;
1053
1054         /* Given slave mus not be in active list */
1055         RTE_ASSERT(find_slave_by_id(internals->active_slaves,
1056         internals->active_slave_count, slave_id) == internals->active_slave_count);
1057         RTE_SET_USED(internals); /* used only for assert when enabled */
1058
1059         memcpy(&port->actor, &initial, sizeof(struct port_params));
1060         /* Standard requires that port ID must be grater than 0.
1061          * Add 1 do get corresponding port_number */
1062         port->actor.port_number = rte_cpu_to_be_16(slave_id + 1);
1063
1064         memcpy(&port->partner, &initial, sizeof(struct port_params));
1065         memcpy(&port->partner_admin, &initial, sizeof(struct port_params));
1066
1067         /* default states */
1068         port->actor_state = STATE_AGGREGATION | STATE_LACP_ACTIVE | STATE_DEFAULTED;
1069         port->partner_state = STATE_LACP_ACTIVE | STATE_AGGREGATION;
1070         port->sm_flags = SM_FLAGS_BEGIN;
1071
1072         /* use this port as agregator */
1073         port->aggregator_port_id = slave_id;
1074
1075         if (bond_mode_8023ad_register_lacp_mac(slave_id) < 0) {
1076                 RTE_BOND_LOG(WARNING, "slave %u is most likely broken and won't receive LACP packets",
1077                              slave_id);
1078         }
1079
1080         timer_cancel(&port->warning_timer);
1081
1082         if (port->mbuf_pool != NULL)
1083                 return;
1084
1085         RTE_ASSERT(port->rx_ring == NULL);
1086         RTE_ASSERT(port->tx_ring == NULL);
1087
1088         socket_id = rte_eth_dev_socket_id(slave_id);
1089         if (socket_id == -1)
1090                 socket_id = rte_socket_id();
1091
1092         element_size = sizeof(struct slow_protocol_frame) +
1093                                 RTE_PKTMBUF_HEADROOM;
1094
1095         /* The size of the mempool should be at least:
1096          * the sum of the TX descriptors + BOND_MODE_8023AX_SLAVE_TX_PKTS */
1097         total_tx_desc = BOND_MODE_8023AX_SLAVE_TX_PKTS;
1098         for (q_id = 0; q_id < bond_dev->data->nb_tx_queues; q_id++) {
1099                 bd_tx_q = (struct bond_tx_queue*)bond_dev->data->tx_queues[q_id];
1100                 total_tx_desc += bd_tx_q->nb_tx_desc;
1101         }
1102
1103         snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_pool", slave_id);
1104         port->mbuf_pool = rte_pktmbuf_pool_create(mem_name, total_tx_desc,
1105                 RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ?
1106                         32 : RTE_MEMPOOL_CACHE_MAX_SIZE,
1107                 0, element_size, socket_id);
1108
1109         /* Any memory allocation failure in initialization is critical because
1110          * resources can't be free, so reinitialization is impossible. */
1111         if (port->mbuf_pool == NULL) {
1112                 rte_panic("Slave %u: Failed to create memory pool '%s': %s\n",
1113                         slave_id, mem_name, rte_strerror(rte_errno));
1114         }
1115
1116         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_rx", slave_id);
1117         port->rx_ring = rte_ring_create(mem_name,
1118                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_RX_PKTS), socket_id, 0);
1119
1120         if (port->rx_ring == NULL) {
1121                 rte_panic("Slave %u: Failed to create rx ring '%s': %s\n", slave_id,
1122                         mem_name, rte_strerror(rte_errno));
1123         }
1124
1125         /* TX ring is at least one pkt longer to make room for marker packet. */
1126         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_tx", slave_id);
1127         port->tx_ring = rte_ring_create(mem_name,
1128                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_TX_PKTS + 1), socket_id, 0);
1129
1130         if (port->tx_ring == NULL) {
1131                 rte_panic("Slave %u: Failed to create tx ring '%s': %s\n", slave_id,
1132                         mem_name, rte_strerror(rte_errno));
1133         }
1134 }
1135
1136 int
1137 bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev __rte_unused,
1138                 uint16_t slave_id)
1139 {
1140         void *pkt = NULL;
1141         struct port *port = NULL;
1142         uint8_t old_partner_state;
1143
1144         port = &bond_mode_8023ad_ports[slave_id];
1145
1146         ACTOR_STATE_CLR(port, AGGREGATION);
1147         port->selected = UNSELECTED;
1148
1149         old_partner_state = port->partner_state;
1150         record_default(port);
1151
1152         bond_mode_8023ad_unregister_lacp_mac(slave_id);
1153
1154         /* If partner timeout state changes then disable timer */
1155         if (!((old_partner_state ^ port->partner_state) &
1156                         STATE_LACP_SHORT_TIMEOUT))
1157                 timer_cancel(&port->current_while_timer);
1158
1159         PARTNER_STATE_CLR(port, AGGREGATION);
1160         ACTOR_STATE_CLR(port, EXPIRED);
1161
1162         /* flush rx/tx rings */
1163         while (rte_ring_dequeue(port->rx_ring, &pkt) == 0)
1164                 rte_pktmbuf_free((struct rte_mbuf *)pkt);
1165
1166         while (rte_ring_dequeue(port->tx_ring, &pkt) == 0)
1167                         rte_pktmbuf_free((struct rte_mbuf *)pkt);
1168         return 0;
1169 }
1170
1171 void
1172 bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev)
1173 {
1174         struct bond_dev_private *internals = bond_dev->data->dev_private;
1175         struct rte_ether_addr slave_addr;
1176         struct port *slave, *agg_slave;
1177         uint16_t slave_id, i, j;
1178
1179         bond_mode_8023ad_stop(bond_dev);
1180
1181         for (i = 0; i < internals->active_slave_count; i++) {
1182                 slave_id = internals->active_slaves[i];
1183                 slave = &bond_mode_8023ad_ports[slave_id];
1184                 rte_eth_macaddr_get(slave_id, &slave_addr);
1185
1186                 if (rte_is_same_ether_addr(&slave_addr, &slave->actor.system))
1187                         continue;
1188
1189                 rte_ether_addr_copy(&slave_addr, &slave->actor.system);
1190                 /* Do nothing if this port is not an aggregator. In other case
1191                  * Set NTT flag on every port that use this aggregator. */
1192                 if (slave->aggregator_port_id != slave_id)
1193                         continue;
1194
1195                 for (j = 0; j < internals->active_slave_count; j++) {
1196                         agg_slave = &bond_mode_8023ad_ports[internals->active_slaves[j]];
1197                         if (agg_slave->aggregator_port_id == slave_id)
1198                                 SM_FLAG_SET(agg_slave, NTT);
1199                 }
1200         }
1201
1202         if (bond_dev->data->dev_started)
1203                 bond_mode_8023ad_start(bond_dev);
1204 }
1205
1206 static void
1207 bond_mode_8023ad_conf_get(struct rte_eth_dev *dev,
1208                 struct rte_eth_bond_8023ad_conf *conf)
1209 {
1210         struct bond_dev_private *internals = dev->data->dev_private;
1211         struct mode8023ad_private *mode4 = &internals->mode4;
1212         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1213
1214         conf->fast_periodic_ms = mode4->fast_periodic_timeout / ms_ticks;
1215         conf->slow_periodic_ms = mode4->slow_periodic_timeout / ms_ticks;
1216         conf->short_timeout_ms = mode4->short_timeout / ms_ticks;
1217         conf->long_timeout_ms = mode4->long_timeout / ms_ticks;
1218         conf->aggregate_wait_timeout_ms = mode4->aggregate_wait_timeout / ms_ticks;
1219         conf->tx_period_ms = mode4->tx_period_timeout / ms_ticks;
1220         conf->update_timeout_ms = mode4->update_timeout_us / 1000;
1221         conf->rx_marker_period_ms = mode4->rx_marker_timeout / ms_ticks;
1222         conf->slowrx_cb = mode4->slowrx_cb;
1223         conf->agg_selection = mode4->agg_selection;
1224 }
1225
1226 static void
1227 bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf)
1228 {
1229         conf->fast_periodic_ms = BOND_8023AD_FAST_PERIODIC_MS;
1230         conf->slow_periodic_ms = BOND_8023AD_SLOW_PERIODIC_MS;
1231         conf->short_timeout_ms = BOND_8023AD_SHORT_TIMEOUT_MS;
1232         conf->long_timeout_ms = BOND_8023AD_LONG_TIMEOUT_MS;
1233         conf->aggregate_wait_timeout_ms = BOND_8023AD_AGGREGATE_WAIT_TIMEOUT_MS;
1234         conf->tx_period_ms = BOND_8023AD_TX_MACHINE_PERIOD_MS;
1235         conf->rx_marker_period_ms = BOND_8023AD_RX_MARKER_PERIOD_MS;
1236         conf->update_timeout_ms = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS;
1237         conf->slowrx_cb = NULL;
1238         conf->agg_selection = AGG_STABLE;
1239 }
1240
1241 static void
1242 bond_mode_8023ad_conf_assign(struct mode8023ad_private *mode4,
1243                 struct rte_eth_bond_8023ad_conf *conf)
1244 {
1245         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1246
1247         mode4->fast_periodic_timeout = conf->fast_periodic_ms * ms_ticks;
1248         mode4->slow_periodic_timeout = conf->slow_periodic_ms * ms_ticks;
1249         mode4->short_timeout = conf->short_timeout_ms * ms_ticks;
1250         mode4->long_timeout = conf->long_timeout_ms * ms_ticks;
1251         mode4->aggregate_wait_timeout = conf->aggregate_wait_timeout_ms * ms_ticks;
1252         mode4->tx_period_timeout = conf->tx_period_ms * ms_ticks;
1253         mode4->rx_marker_timeout = conf->rx_marker_period_ms * ms_ticks;
1254         mode4->update_timeout_us = conf->update_timeout_ms * 1000;
1255
1256         mode4->dedicated_queues.enabled = 0;
1257         mode4->dedicated_queues.rx_qid = UINT16_MAX;
1258         mode4->dedicated_queues.tx_qid = UINT16_MAX;
1259 }
1260
1261 void
1262 bond_mode_8023ad_setup(struct rte_eth_dev *dev,
1263                 struct rte_eth_bond_8023ad_conf *conf)
1264 {
1265         struct rte_eth_bond_8023ad_conf def_conf;
1266         struct bond_dev_private *internals = dev->data->dev_private;
1267         struct mode8023ad_private *mode4 = &internals->mode4;
1268
1269         if (conf == NULL) {
1270                 conf = &def_conf;
1271                 bond_mode_8023ad_conf_get_default(conf);
1272         }
1273
1274         bond_mode_8023ad_stop(dev);
1275         bond_mode_8023ad_conf_assign(mode4, conf);
1276         mode4->slowrx_cb = conf->slowrx_cb;
1277         mode4->agg_selection = AGG_STABLE;
1278
1279         if (dev->data->dev_started)
1280                 bond_mode_8023ad_start(dev);
1281 }
1282
1283 int
1284 bond_mode_8023ad_enable(struct rte_eth_dev *bond_dev)
1285 {
1286         struct bond_dev_private *internals = bond_dev->data->dev_private;
1287         uint16_t i;
1288
1289         for (i = 0; i < internals->active_slave_count; i++)
1290                 bond_mode_8023ad_activate_slave(bond_dev,
1291                                 internals->active_slaves[i]);
1292
1293         return 0;
1294 }
1295
1296 int
1297 bond_mode_8023ad_start(struct rte_eth_dev *bond_dev)
1298 {
1299         struct bond_dev_private *internals = bond_dev->data->dev_private;
1300         struct mode8023ad_private *mode4 = &internals->mode4;
1301         static const uint64_t us = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS * 1000;
1302
1303         rte_eth_macaddr_get(internals->port_id, &mode4->mac_addr);
1304         if (mode4->slowrx_cb)
1305                 return rte_eal_alarm_set(us, &bond_mode_8023ad_ext_periodic_cb,
1306                                          bond_dev);
1307
1308         return rte_eal_alarm_set(us, &bond_mode_8023ad_periodic_cb, bond_dev);
1309 }
1310
1311 void
1312 bond_mode_8023ad_stop(struct rte_eth_dev *bond_dev)
1313 {
1314         struct bond_dev_private *internals = bond_dev->data->dev_private;
1315         struct mode8023ad_private *mode4 = &internals->mode4;
1316
1317         if (mode4->slowrx_cb) {
1318                 rte_eal_alarm_cancel(&bond_mode_8023ad_ext_periodic_cb,
1319                                      bond_dev);
1320                 return;
1321         }
1322         rte_eal_alarm_cancel(&bond_mode_8023ad_periodic_cb, bond_dev);
1323 }
1324
1325 void
1326 bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
1327                                   uint16_t slave_id, struct rte_mbuf *pkt)
1328 {
1329         struct mode8023ad_private *mode4 = &internals->mode4;
1330         struct port *port = &bond_mode_8023ad_ports[slave_id];
1331         struct marker_header *m_hdr;
1332         uint64_t marker_timer, old_marker_timer;
1333         int retval;
1334         uint8_t wrn, subtype;
1335         /* If packet is a marker, we send response now by reusing given packet
1336          * and update only source MAC, destination MAC is multicast so don't
1337          * update it. Other frames will be handled later by state machines */
1338         subtype = rte_pktmbuf_mtod(pkt,
1339                         struct slow_protocol_frame *)->slow_protocol.subtype;
1340
1341         if (subtype == SLOW_SUBTYPE_MARKER) {
1342                 m_hdr = rte_pktmbuf_mtod(pkt, struct marker_header *);
1343
1344                 if (likely(m_hdr->marker.tlv_type_marker != MARKER_TLV_TYPE_INFO)) {
1345                         wrn = WRN_UNKNOWN_MARKER_TYPE;
1346                         goto free_out;
1347                 }
1348
1349                 /* Setup marker timer. Do it in loop in case concurrent access. */
1350                 do {
1351                         old_marker_timer = port->rx_marker_timer;
1352                         if (!timer_is_expired(&old_marker_timer)) {
1353                                 wrn = WRN_RX_MARKER_TO_FAST;
1354                                 goto free_out;
1355                         }
1356
1357                         timer_set(&marker_timer, mode4->rx_marker_timeout);
1358                         retval = rte_atomic64_cmpset(&port->rx_marker_timer,
1359                                 old_marker_timer, marker_timer);
1360                 } while (unlikely(retval == 0));
1361
1362                 m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP;
1363                 rte_eth_macaddr_get(slave_id, &m_hdr->eth_hdr.src_addr);
1364
1365                 if (internals->mode4.dedicated_queues.enabled == 0) {
1366                         if (rte_ring_enqueue(port->tx_ring, pkt) != 0) {
1367                                 /* reset timer */
1368                                 port->rx_marker_timer = 0;
1369                                 wrn = WRN_TX_QUEUE_FULL;
1370                                 goto free_out;
1371                         }
1372                 } else {
1373                         /* Send packet directly to the slow queue */
1374                         uint16_t tx_count = rte_eth_tx_burst(slave_id,
1375                                         internals->mode4.dedicated_queues.tx_qid,
1376                                         &pkt, 1);
1377                         if (tx_count != 1) {
1378                                 /* reset timer */
1379                                 port->rx_marker_timer = 0;
1380                                 wrn = WRN_TX_QUEUE_FULL;
1381                                 goto free_out;
1382                         }
1383                 }
1384         } else if (likely(subtype == SLOW_SUBTYPE_LACP)) {
1385                 if (internals->mode4.dedicated_queues.enabled == 0) {
1386                         if (rte_ring_enqueue(port->rx_ring, pkt) != 0) {
1387                                 /* If RX fing full free lacpdu message and drop packet */
1388                                 wrn = WRN_RX_QUEUE_FULL;
1389                                 goto free_out;
1390                         }
1391                 } else
1392                         rx_machine_update(internals, slave_id, pkt);
1393         } else {
1394                 wrn = WRN_UNKNOWN_SLOW_TYPE;
1395                 goto free_out;
1396         }
1397
1398         return;
1399
1400 free_out:
1401         set_warning_flags(port, wrn);
1402         rte_pktmbuf_free(pkt);
1403 }
1404
1405 int
1406 rte_eth_bond_8023ad_conf_get(uint16_t port_id,
1407                 struct rte_eth_bond_8023ad_conf *conf)
1408 {
1409         struct rte_eth_dev *bond_dev;
1410
1411         if (valid_bonded_port_id(port_id) != 0)
1412                 return -EINVAL;
1413
1414         if (conf == NULL)
1415                 return -EINVAL;
1416
1417         bond_dev = &rte_eth_devices[port_id];
1418         bond_mode_8023ad_conf_get(bond_dev, conf);
1419         return 0;
1420 }
1421
1422 int
1423 rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id,
1424                 enum rte_bond_8023ad_agg_selection agg_selection)
1425 {
1426         struct rte_eth_dev *bond_dev;
1427         struct bond_dev_private *internals;
1428         struct mode8023ad_private *mode4;
1429
1430         if (valid_bonded_port_id(port_id) != 0)
1431                 return -EINVAL;
1432
1433         bond_dev = &rte_eth_devices[port_id];
1434         internals = bond_dev->data->dev_private;
1435
1436         if (internals->mode != 4)
1437                 return -EINVAL;
1438
1439         mode4 = &internals->mode4;
1440         if (agg_selection == AGG_COUNT || agg_selection == AGG_BANDWIDTH
1441                         || agg_selection == AGG_STABLE)
1442                 mode4->agg_selection = agg_selection;
1443         return 0;
1444 }
1445
1446 int rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id)
1447 {
1448         struct rte_eth_dev *bond_dev;
1449         struct bond_dev_private *internals;
1450         struct mode8023ad_private *mode4;
1451
1452         if (valid_bonded_port_id(port_id) != 0)
1453                 return -EINVAL;
1454
1455         bond_dev = &rte_eth_devices[port_id];
1456         internals = bond_dev->data->dev_private;
1457
1458         if (internals->mode != 4)
1459                 return -EINVAL;
1460         mode4 = &internals->mode4;
1461
1462         return mode4->agg_selection;
1463 }
1464
1465
1466
1467 static int
1468 bond_8023ad_setup_validate(uint16_t port_id,
1469                 struct rte_eth_bond_8023ad_conf *conf)
1470 {
1471         if (valid_bonded_port_id(port_id) != 0)
1472                 return -EINVAL;
1473
1474         if (conf != NULL) {
1475                 /* Basic sanity check */
1476                 if (conf->slow_periodic_ms == 0 ||
1477                                 conf->fast_periodic_ms >= conf->slow_periodic_ms ||
1478                                 conf->long_timeout_ms == 0 ||
1479                                 conf->short_timeout_ms >= conf->long_timeout_ms ||
1480                                 conf->aggregate_wait_timeout_ms == 0 ||
1481                                 conf->tx_period_ms == 0 ||
1482                                 conf->rx_marker_period_ms == 0 ||
1483                                 conf->update_timeout_ms == 0) {
1484                         RTE_BOND_LOG(ERR, "given mode 4 configuration is invalid");
1485                         return -EINVAL;
1486                 }
1487         }
1488
1489         return 0;
1490 }
1491
1492
1493 int
1494 rte_eth_bond_8023ad_setup(uint16_t port_id,
1495                 struct rte_eth_bond_8023ad_conf *conf)
1496 {
1497         struct rte_eth_dev *bond_dev;
1498         int err;
1499
1500         err = bond_8023ad_setup_validate(port_id, conf);
1501         if (err != 0)
1502                 return err;
1503
1504         bond_dev = &rte_eth_devices[port_id];
1505         bond_mode_8023ad_setup(bond_dev, conf);
1506
1507         return 0;
1508 }
1509
1510
1511
1512
1513
1514 int
1515 rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id,
1516                 struct rte_eth_bond_8023ad_slave_info *info)
1517 {
1518         struct rte_eth_dev *bond_dev;
1519         struct bond_dev_private *internals;
1520         struct port *port;
1521
1522         if (info == NULL || valid_bonded_port_id(port_id) != 0 ||
1523                         rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1524                 return -EINVAL;
1525
1526         bond_dev = &rte_eth_devices[port_id];
1527
1528         internals = bond_dev->data->dev_private;
1529         if (find_slave_by_id(internals->active_slaves,
1530                         internals->active_slave_count, slave_id) ==
1531                                 internals->active_slave_count)
1532                 return -EINVAL;
1533
1534         port = &bond_mode_8023ad_ports[slave_id];
1535         info->selected = port->selected;
1536
1537         info->actor_state = port->actor_state;
1538         rte_memcpy(&info->actor, &port->actor, sizeof(port->actor));
1539
1540         info->partner_state = port->partner_state;
1541         rte_memcpy(&info->partner, &port->partner, sizeof(port->partner));
1542
1543         info->agg_port_id = port->aggregator_port_id;
1544         return 0;
1545 }
1546
1547 static int
1548 bond_8023ad_ext_validate(uint16_t port_id, uint16_t slave_id)
1549 {
1550         struct rte_eth_dev *bond_dev;
1551         struct bond_dev_private *internals;
1552         struct mode8023ad_private *mode4;
1553
1554         if (rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1555                 return -EINVAL;
1556
1557         bond_dev = &rte_eth_devices[port_id];
1558
1559         if (!bond_dev->data->dev_started)
1560                 return -EINVAL;
1561
1562         internals = bond_dev->data->dev_private;
1563         if (find_slave_by_id(internals->active_slaves,
1564                         internals->active_slave_count, slave_id) ==
1565                                 internals->active_slave_count)
1566                 return -EINVAL;
1567
1568         mode4 = &internals->mode4;
1569         if (mode4->slowrx_cb == NULL)
1570                 return -EINVAL;
1571
1572         return 0;
1573 }
1574
1575 int
1576 rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id,
1577                                 int enabled)
1578 {
1579         struct port *port;
1580         int res;
1581
1582         res = bond_8023ad_ext_validate(port_id, slave_id);
1583         if (res != 0)
1584                 return res;
1585
1586         port = &bond_mode_8023ad_ports[slave_id];
1587
1588         if (enabled)
1589                 ACTOR_STATE_SET(port, COLLECTING);
1590         else
1591                 ACTOR_STATE_CLR(port, COLLECTING);
1592
1593         return 0;
1594 }
1595
1596 int
1597 rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id,
1598                                 int enabled)
1599 {
1600         struct port *port;
1601         int res;
1602
1603         res = bond_8023ad_ext_validate(port_id, slave_id);
1604         if (res != 0)
1605                 return res;
1606
1607         port = &bond_mode_8023ad_ports[slave_id];
1608
1609         if (enabled)
1610                 ACTOR_STATE_SET(port, DISTRIBUTING);
1611         else
1612                 ACTOR_STATE_CLR(port, DISTRIBUTING);
1613
1614         return 0;
1615 }
1616
1617 int
1618 rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id)
1619 {
1620         struct port *port;
1621         int err;
1622
1623         err = bond_8023ad_ext_validate(port_id, slave_id);
1624         if (err != 0)
1625                 return err;
1626
1627         port = &bond_mode_8023ad_ports[slave_id];
1628         return ACTOR_STATE(port, DISTRIBUTING);
1629 }
1630
1631 int
1632 rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id)
1633 {
1634         struct port *port;
1635         int err;
1636
1637         err = bond_8023ad_ext_validate(port_id, slave_id);
1638         if (err != 0)
1639                 return err;
1640
1641         port = &bond_mode_8023ad_ports[slave_id];
1642         return ACTOR_STATE(port, COLLECTING);
1643 }
1644
1645 int
1646 rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id,
1647                 struct rte_mbuf *lacp_pkt)
1648 {
1649         struct port *port;
1650         int res;
1651
1652         res = bond_8023ad_ext_validate(port_id, slave_id);
1653         if (res != 0)
1654                 return res;
1655
1656         port = &bond_mode_8023ad_ports[slave_id];
1657
1658         if (rte_pktmbuf_pkt_len(lacp_pkt) < sizeof(struct lacpdu_header))
1659                 return -EINVAL;
1660
1661         struct lacpdu_header *lacp;
1662
1663         /* only enqueue LACPDUs */
1664         lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
1665         if (lacp->lacpdu.subtype != SLOW_SUBTYPE_LACP)
1666                 return -EINVAL;
1667
1668         MODE4_DEBUG("sending LACP frame\n");
1669
1670         return rte_ring_enqueue(port->tx_ring, lacp_pkt);
1671 }
1672
1673 static void
1674 bond_mode_8023ad_ext_periodic_cb(void *arg)
1675 {
1676         struct rte_eth_dev *bond_dev = arg;
1677         struct bond_dev_private *internals = bond_dev->data->dev_private;
1678         struct mode8023ad_private *mode4 = &internals->mode4;
1679         struct port *port;
1680         void *pkt = NULL;
1681         uint16_t i, slave_id;
1682
1683         for (i = 0; i < internals->active_slave_count; i++) {
1684                 slave_id = internals->active_slaves[i];
1685                 port = &bond_mode_8023ad_ports[slave_id];
1686
1687                 if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) {
1688                         struct rte_mbuf *lacp_pkt = pkt;
1689                         struct lacpdu_header *lacp;
1690
1691                         lacp = rte_pktmbuf_mtod(lacp_pkt,
1692                                                 struct lacpdu_header *);
1693                         RTE_VERIFY(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
1694
1695                         /* This is LACP frame so pass it to rx callback.
1696                          * Callback is responsible for freeing mbuf.
1697                          */
1698                         mode4->slowrx_cb(slave_id, lacp_pkt);
1699                 }
1700         }
1701
1702         rte_eal_alarm_set(internals->mode4.update_timeout_us,
1703                         bond_mode_8023ad_ext_periodic_cb, arg);
1704 }
1705
1706 int
1707 rte_eth_bond_8023ad_dedicated_queues_enable(uint16_t port)
1708 {
1709         int retval = 0;
1710         struct rte_eth_dev *dev;
1711         struct bond_dev_private *internals;
1712
1713         if (valid_bonded_port_id(port) != 0)
1714                 return -EINVAL;
1715
1716         dev = &rte_eth_devices[port];
1717         internals = dev->data->dev_private;
1718
1719         if (bond_8023ad_slow_pkt_hw_filter_supported(port) != 0)
1720                 return -1;
1721
1722         /* Device must be stopped to set up slow queue */
1723         if (dev->data->dev_started)
1724                 return -1;
1725
1726         internals->mode4.dedicated_queues.enabled = 1;
1727
1728         bond_ethdev_mode_set(dev, internals->mode);
1729         return retval;
1730 }
1731
1732 int
1733 rte_eth_bond_8023ad_dedicated_queues_disable(uint16_t port)
1734 {
1735         int retval = 0;
1736         struct rte_eth_dev *dev;
1737         struct bond_dev_private *internals;
1738
1739         if (valid_bonded_port_id(port) != 0)
1740                 return -EINVAL;
1741
1742         dev = &rte_eth_devices[port];
1743         internals = dev->data->dev_private;
1744
1745         /* Device must be stopped to set up slow queue */
1746         if (dev->data->dev_started)
1747                 return -1;
1748
1749         internals->mode4.dedicated_queues.enabled = 0;
1750
1751         bond_ethdev_mode_set(dev, internals->mode);
1752
1753         return retval;
1754 }