drivers/net/bonding/rte_eth_bond_8023ad.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <stddef.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_alarm.h>
  10 #include <rte_malloc.h>
  11 #include <rte_errno.h>
  12 #include <rte_cycles.h>
  13 #include <rte_compat.h>
  14
  15 #include "eth_bond_private.h"
  16
  17 static void bond_mode_8023ad_ext_periodic_cb(void *arg);
  18 #ifdef RTE_LIBRTE_BOND_DEBUG_8023AD
  19
  20 #define MODE4_DEBUG(fmt, ...)                           \
  21         rte_log(RTE_LOG_DEBUG, bond_logtype,            \
  22                 "%6u [Port %u: %s] " fmt,               \
  23                 bond_dbg_get_time_diff_ms(), slave_id,  \
  24                 __func__, ##__VA_ARGS__)
  25
  26 static uint64_t start_time;
  27
  28 static unsigned
  29 bond_dbg_get_time_diff_ms(void)
  30 {
  31         uint64_t now;
  32
  33         now = rte_rdtsc();
  34         if (start_time == 0)
  35                 start_time = now;
  36
  37         return ((now - start_time) * 1000) / rte_get_tsc_hz();
  38 }
  39
  40 static void
  41 bond_print_lacp(struct lacpdu *l)
  42 {
  43         char a_address[18];
  44         char p_address[18];
  45         char a_state[256] = { 0 };
  46         char p_state[256] = { 0 };
  47
  48         static const char * const state_labels[] = {
  49                 "ACT", "TIMEOUT", "AGG", "SYNC", "COL", "DIST", "DEF", "EXP"
  50         };
  51
  52         int a_len = 0;
  53         int p_len = 0;
  54         uint8_t i;
  55         uint8_t *addr;
  56
  57         addr = l->actor.port_params.system.addr_bytes;
  58         snprintf(a_address, sizeof(a_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  59                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  60
  61         addr = l->partner.port_params.system.addr_bytes;
  62         snprintf(p_address, sizeof(p_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  63                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  64
  65         for (i = 0; i < 8; i++) {
  66                 if ((l->actor.state >> i) & 1) {
  67                         a_len += snprintf(&a_state[a_len], RTE_DIM(a_state) - a_len, "%s ",
  68                                 state_labels[i]);
  69                 }
  70
  71                 if ((l->partner.state >> i) & 1) {
  72                         p_len += snprintf(&p_state[p_len], RTE_DIM(p_state) - p_len, "%s ",
  73                                 state_labels[i]);
  74                 }
  75         }
  76
  77         if (a_len && a_state[a_len-1] == ' ')
  78                 a_state[a_len-1] = '\0';
  79
  80         if (p_len && p_state[p_len-1] == ' ')
  81                 p_state[p_len-1] = '\0';
  82
  83         RTE_BOND_LOG(DEBUG,
  84                      "LACP: {\n"
  85                      "  subtype= %02X\n"
  86                      "  ver_num=%02X\n"
  87                      "  actor={ tlv=%02X, len=%02X\n"
  88                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  89                      "       state={ %s }\n"
  90                      "  }\n"
  91                      "  partner={ tlv=%02X, len=%02X\n"
  92                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  93                      "       state={ %s }\n"
  94                      "  }\n"
  95                      "  collector={info=%02X, length=%02X, max_delay=%04X\n, "
  96                      "type_term=%02X, terminator_length = %02X }",
  97                      l->subtype,
  98                      l->version_number,
  99                      l->actor.tlv_type_info,
 100                      l->actor.info_length,
 101                      l->actor.port_params.system_priority,
 102                      a_address,
 103                      l->actor.port_params.key,
 104                      l->actor.port_params.port_priority,
 105                      l->actor.port_params.port_number,
 106                      a_state,
 107                      l->partner.tlv_type_info,
 108                      l->partner.info_length,
 109                      l->partner.port_params.system_priority,
 110                      p_address,
 111                      l->partner.port_params.key,
 112                      l->partner.port_params.port_priority,
 113                      l->partner.port_params.port_number,
 114                      p_state,
 115                      l->tlv_type_collector_info,
 116                      l->collector_info_length,
 117                      l->collector_max_delay,
 118                      l->tlv_type_terminator,
 119                      l->terminator_length);
 120
 121 }
 122
 123 #define BOND_PRINT_LACP(lacpdu) bond_print_lacp(lacpdu)
 124 #else
 125 #define BOND_PRINT_LACP(lacpdu) do { } while (0)
 126 #define MODE4_DEBUG(fmt, ...) do { } while (0)
 127 #endif
 128
 129 static const struct rte_ether_addr lacp_mac_addr = {
 130         .addr_bytes = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 }
 131 };
 132
 133 struct port bond_mode_8023ad_ports[RTE_MAX_ETHPORTS];
 134
 135 static void
 136 timer_cancel(uint64_t *timer)
 137 {
 138         *timer = 0;
 139 }
 140
 141 static void
 142 timer_set(uint64_t *timer, uint64_t timeout)
 143 {
 144         *timer = rte_rdtsc() + timeout;
 145 }
 146
 147 /* Forces given timer to be in expired state. */
 148 static void
 149 timer_force_expired(uint64_t *timer)
 150 {
 151         *timer = rte_rdtsc();
 152 }
 153
 154 static bool
 155 timer_is_stopped(uint64_t *timer)
 156 {
 157         return *timer == 0;
 158 }
 159
 160 static bool
 161 timer_is_expired(uint64_t *timer)
 162 {
 163         return *timer < rte_rdtsc();
 164 }
 165
 166 /* Timer is in running state if it is not stopped nor expired */
 167 static bool
 168 timer_is_running(uint64_t *timer)
 169 {
 170         return !timer_is_stopped(timer) && !timer_is_expired(timer);
 171 }
 172
 173 static void
 174 set_warning_flags(struct port *port, uint16_t flags)
 175 {
 176         int retval;
 177         uint16_t old;
 178         uint16_t new_flag = 0;
 179
 180         do {
 181                 old = port->warnings_to_show;
 182                 new_flag = old | flags;
 183                 retval = rte_atomic16_cmpset(&port->warnings_to_show, old, new_flag);
 184         } while (unlikely(retval == 0));
 185 }
 186
 187 static void
 188 show_warnings(uint16_t slave_id)
 189 {
 190         struct port *port = &bond_mode_8023ad_ports[slave_id];
 191         uint8_t warnings;
 192
 193         do {
 194                 warnings = port->warnings_to_show;
 195         } while (rte_atomic16_cmpset(&port->warnings_to_show, warnings, 0) == 0);
 196
 197         if (!warnings)
 198                 return;
 199
 200         if (!timer_is_expired(&port->warning_timer))
 201                 return;
 202
 203
 204         timer_set(&port->warning_timer, BOND_8023AD_WARNINGS_PERIOD_MS *
 205                         rte_get_tsc_hz() / 1000);
 206
 207         if (warnings & WRN_RX_QUEUE_FULL) {
 208                 RTE_BOND_LOG(DEBUG,
 209                              "Slave %u: failed to enqueue LACP packet into RX ring.\n"
 210                              "Receive and transmit functions must be invoked on bonded"
 211                              "interface at least 10 times per second or LACP will notwork correctly",
 212                              slave_id);
 213         }
 214
 215         if (warnings & WRN_TX_QUEUE_FULL) {
 216                 RTE_BOND_LOG(DEBUG,
 217                              "Slave %u: failed to enqueue LACP packet into TX ring.\n"
 218                              "Receive and transmit functions must be invoked on bonded"
 219                              "interface at least 10 times per second or LACP will not work correctly",
 220                              slave_id);
 221         }
 222
 223         if (warnings & WRN_RX_MARKER_TO_FAST)
 224                 RTE_BOND_LOG(INFO, "Slave %u: marker to early - ignoring.",
 225                              slave_id);
 226
 227         if (warnings & WRN_UNKNOWN_SLOW_TYPE) {
 228                 RTE_BOND_LOG(INFO,
 229                         "Slave %u: ignoring unknown slow protocol frame type",
 230                              slave_id);
 231         }
 232
 233         if (warnings & WRN_UNKNOWN_MARKER_TYPE)
 234                 RTE_BOND_LOG(INFO, "Slave %u: ignoring unknown marker type",
 235                              slave_id);
 236
 237         if (warnings & WRN_NOT_LACP_CAPABLE)
 238                 MODE4_DEBUG("Port %u is not LACP capable!\n", slave_id);
 239 }
 240
 241 static void
 242 record_default(struct port *port)
 243 {
 244         /* Record default parameters for partner. Partner admin parameters
 245          * are not implemented so set them to arbitrary default (last known) and
 246          * mark actor that parner is in defaulted state. */
 247         port->partner_state = STATE_LACP_ACTIVE;
 248         ACTOR_STATE_SET(port, DEFAULTED);
 249 }
 250
 251 /** Function handles rx state machine.
 252  *
 253  * This function implements Receive State Machine from point 5.4.12 in
 254  * 802.1AX documentation. It should be called periodically.
 255  *
 256  * @param lacpdu                LACPDU received.
 257  * @param port                  Port on which LACPDU was received.
 258  */
 259 static void
 260 rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 261                 struct lacpdu *lacp)
 262 {
 263         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 264         uint64_t timeout;
 265
 266         if (SM_FLAG(port, BEGIN)) {
 267                 /* Initialize stuff */
 268                 MODE4_DEBUG("-> INITIALIZE\n");
 269                 SM_FLAG_CLR(port, MOVED);
 270                 port->selected = UNSELECTED;
 271
 272                 record_default(port);
 273
 274                 ACTOR_STATE_CLR(port, EXPIRED);
 275                 timer_cancel(&port->current_while_timer);
 276
 277                 /* DISABLED: On initialization partner is out of sync */
 278                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 279
 280                 /* LACP DISABLED stuff if LACP not enabled on this port */
 281                 if (!SM_FLAG(port, LACP_ENABLED))
 282                         PARTNER_STATE_CLR(port, AGGREGATION);
 283                 else
 284                         PARTNER_STATE_SET(port, AGGREGATION);
 285         }
 286
 287         if (!SM_FLAG(port, LACP_ENABLED)) {
 288                 /* Update parameters only if state changed */
 289                 if (!timer_is_stopped(&port->current_while_timer)) {
 290                         port->selected = UNSELECTED;
 291                         record_default(port);
 292                         PARTNER_STATE_CLR(port, AGGREGATION);
 293                         ACTOR_STATE_CLR(port, EXPIRED);
 294                         timer_cancel(&port->current_while_timer);
 295                 }
 296                 return;
 297         }
 298
 299         if (lacp) {
 300                 MODE4_DEBUG("LACP -> CURRENT\n");
 301                 BOND_PRINT_LACP(lacp);
 302                 /* Update selected flag. If partner parameters are defaulted assume they
 303                  * are match. If not defaulted  compare LACP actor with ports parner
 304                  * params. */
 305                 if (!ACTOR_STATE(port, DEFAULTED) &&
 306                         (ACTOR_STATE(port, AGGREGATION) != PARTNER_STATE(port, AGGREGATION)
 307                         || memcmp(&port->partner, &lacp->actor.port_params,
 308                                 sizeof(port->partner)) != 0)) {
 309                         MODE4_DEBUG("selected <- UNSELECTED\n");
 310                         port->selected = UNSELECTED;
 311                 }
 312
 313                 /* Record this PDU actor params as partner params */
 314                 memcpy(&port->partner, &lacp->actor.port_params,
 315                         sizeof(struct port_params));
 316                 port->partner_state = lacp->actor.state;
 317
 318                 /* Partner parameters are not defaulted any more */
 319                 ACTOR_STATE_CLR(port, DEFAULTED);
 320
 321                 /* If LACP partner params match this port actor params */
 322                 agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 323                 bool match = port->actor.system_priority ==
 324                         lacp->partner.port_params.system_priority &&
 325                         rte_is_same_ether_addr(&agg->actor.system,
 326                         &lacp->partner.port_params.system) &&
 327                         port->actor.port_priority ==
 328                         lacp->partner.port_params.port_priority &&
 329                         port->actor.port_number ==
 330                         lacp->partner.port_params.port_number;
 331
 332                 /* Update NTT if partners information are outdated (xored and masked
 333                  * bits are set)*/
 334                 uint8_t state_mask = STATE_LACP_ACTIVE | STATE_LACP_SHORT_TIMEOUT |
 335                         STATE_SYNCHRONIZATION | STATE_AGGREGATION;
 336
 337                 if (((port->actor_state ^ lacp->partner.state) & state_mask) ||
 338                                 match == false) {
 339                         SM_FLAG_SET(port, NTT);
 340                 }
 341
 342                 /* If LACP partner params match this port actor params */
 343                 if (match == true && ACTOR_STATE(port, AGGREGATION) ==
 344                                 PARTNER_STATE(port,     AGGREGATION))
 345                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 346                 else if (!PARTNER_STATE(port, AGGREGATION) && ACTOR_STATE(port,
 347                                 AGGREGATION))
 348                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 349                 else
 350                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 351
 352                 if (ACTOR_STATE(port, LACP_SHORT_TIMEOUT))
 353                         timeout = internals->mode4.short_timeout;
 354                 else
 355                         timeout = internals->mode4.long_timeout;
 356
 357                 timer_set(&port->current_while_timer, timeout);
 358                 ACTOR_STATE_CLR(port, EXPIRED);
 359                 return; /* No state change */
 360         }
 361
 362         /* If CURRENT state timer is not running (stopped or expired)
 363          * transit to EXPIRED state from DISABLED or CURRENT */
 364         if (!timer_is_running(&port->current_while_timer)) {
 365                 ACTOR_STATE_SET(port, EXPIRED);
 366                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 367                 PARTNER_STATE_SET(port, LACP_SHORT_TIMEOUT);
 368                 timer_set(&port->current_while_timer, internals->mode4.short_timeout);
 369         }
 370 }
 371
 372 /**
 373  * Function handles periodic tx state machine.
 374  *
 375  * Function implements Periodic Transmission state machine from point 5.4.13
 376  * in 802.1AX documentation. It should be called periodically.
 377  *
 378  * @param port                  Port to handle state machine.
 379  */
 380 static void
 381 periodic_machine(struct bond_dev_private *internals, uint16_t slave_id)
 382 {
 383         struct port *port = &bond_mode_8023ad_ports[slave_id];
 384         /* Calculate if either site is LACP enabled */
 385         uint64_t timeout;
 386         uint8_t active = ACTOR_STATE(port, LACP_ACTIVE) ||
 387                 PARTNER_STATE(port, LACP_ACTIVE);
 388
 389         uint8_t is_partner_fast, was_partner_fast;
 390         /* No periodic is on BEGIN, LACP DISABLE or when both sides are pasive */
 391         if (SM_FLAG(port, BEGIN) || !SM_FLAG(port, LACP_ENABLED) || !active) {
 392                 timer_cancel(&port->periodic_timer);
 393                 timer_force_expired(&port->tx_machine_timer);
 394                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 395
 396                 MODE4_DEBUG("-> NO_PERIODIC ( %s%s%s)\n",
 397                         SM_FLAG(port, BEGIN) ? "begind " : "",
 398                         SM_FLAG(port, LACP_ENABLED) ? "" : "LACP disabled ",
 399                         active ? "LACP active " : "LACP pasive ");
 400                 return;
 401         }
 402
 403         is_partner_fast = PARTNER_STATE(port, LACP_SHORT_TIMEOUT);
 404         was_partner_fast = SM_FLAG(port, PARTNER_SHORT_TIMEOUT);
 405
 406         /* If periodic timer is not started, transit from NO PERIODIC to FAST/SLOW.
 407          * Other case: check if timer expire or partners settings changed. */
 408         if (!timer_is_stopped(&port->periodic_timer)) {
 409                 if (timer_is_expired(&port->periodic_timer)) {
 410                         SM_FLAG_SET(port, NTT);
 411                 } else if (is_partner_fast != was_partner_fast) {
 412                         /* Partners timeout  was slow and now it is fast -> send LACP.
 413                          * In other case (was fast and now it is slow) just switch
 414                          * timeout to slow without forcing send of LACP (because standard
 415                          * say so)*/
 416                         if (is_partner_fast)
 417                                 SM_FLAG_SET(port, NTT);
 418                 } else
 419                         return; /* Nothing changed */
 420         }
 421
 422         /* Handle state transition to FAST/SLOW LACP timeout */
 423         if (is_partner_fast) {
 424                 timeout = internals->mode4.fast_periodic_timeout;
 425                 SM_FLAG_SET(port, PARTNER_SHORT_TIMEOUT);
 426         } else {
 427                 timeout = internals->mode4.slow_periodic_timeout;
 428                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 429         }
 430
 431         timer_set(&port->periodic_timer, timeout);
 432 }
 433
 434 /**
 435  * Function handles mux state machine.
 436  *
 437  * Function implements Mux Machine from point 5.4.15 in 802.1AX documentation.
 438  * It should be called periodically.
 439  *
 440  * @param port                  Port to handle state machine.
 441  */
 442 static void
 443 mux_machine(struct bond_dev_private *internals, uint16_t slave_id)
 444 {
 445         struct port *port = &bond_mode_8023ad_ports[slave_id];
 446
 447         /* Save current state for later use */
 448         const uint8_t state_mask = STATE_SYNCHRONIZATION | STATE_DISTRIBUTING |
 449                 STATE_COLLECTING;
 450
 451         /* Enter DETACHED state on BEGIN condition or from any other state if
 452          * port was unselected */
 453         if (SM_FLAG(port, BEGIN) ||
 454                         port->selected == UNSELECTED || (port->selected == STANDBY &&
 455                                 (port->actor_state & state_mask) != 0)) {
 456                 /* detach mux from aggregator */
 457                 port->actor_state &= ~state_mask;
 458                 /* Set ntt to true if BEGIN condition or transition from any other state
 459                  * which is indicated that wait_while_timer was started */
 460                 if (SM_FLAG(port, BEGIN) ||
 461                                 !timer_is_stopped(&port->wait_while_timer)) {
 462                         SM_FLAG_SET(port, NTT);
 463                         MODE4_DEBUG("-> DETACHED\n");
 464                 }
 465                 timer_cancel(&port->wait_while_timer);
 466         }
 467
 468         if (timer_is_stopped(&port->wait_while_timer)) {
 469                 if (port->selected == SELECTED || port->selected == STANDBY) {
 470                         timer_set(&port->wait_while_timer,
 471                                 internals->mode4.aggregate_wait_timeout);
 472
 473                         MODE4_DEBUG("DETACHED -> WAITING\n");
 474                 }
 475                 /* Waiting state entered */
 476                 return;
 477         }
 478
 479         /* Transit next state if port is ready */
 480         if (!timer_is_expired(&port->wait_while_timer))
 481                 return;
 482
 483         if ((ACTOR_STATE(port, DISTRIBUTING) || ACTOR_STATE(port, COLLECTING)) &&
 484                 !PARTNER_STATE(port, SYNCHRONIZATION)) {
 485                 /* If in COLLECTING or DISTRIBUTING state and partner becomes out of
 486                  * sync transit to ATACHED state.  */
 487                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 488                 ACTOR_STATE_CLR(port, COLLECTING);
 489                 /* Clear actor sync to activate transit ATACHED in condition bellow */
 490                 ACTOR_STATE_CLR(port, SYNCHRONIZATION);
 491                 MODE4_DEBUG("Out of sync -> ATTACHED\n");
 492         }
 493
 494         if (!ACTOR_STATE(port, SYNCHRONIZATION)) {
 495                 /* attach mux to aggregator */
 496                 RTE_ASSERT((port->actor_state & (STATE_COLLECTING |
 497                         STATE_DISTRIBUTING)) == 0);
 498
 499                 ACTOR_STATE_SET(port, SYNCHRONIZATION);
 500                 SM_FLAG_SET(port, NTT);
 501                 MODE4_DEBUG("ATTACHED Entered\n");
 502         } else if (!ACTOR_STATE(port, COLLECTING)) {
 503                 /* Start collecting if in sync */
 504                 if (PARTNER_STATE(port, SYNCHRONIZATION)) {
 505                         MODE4_DEBUG("ATTACHED -> COLLECTING\n");
 506                         ACTOR_STATE_SET(port, COLLECTING);
 507                         SM_FLAG_SET(port, NTT);
 508                 }
 509         } else if (ACTOR_STATE(port, COLLECTING)) {
 510                 /* Check if partner is in COLLECTING state. If so this port can
 511                  * distribute frames to it */
 512                 if (!ACTOR_STATE(port, DISTRIBUTING)) {
 513                         if (PARTNER_STATE(port, COLLECTING)) {
 514                                 /* Enable  DISTRIBUTING if partner is collecting */
 515                                 ACTOR_STATE_SET(port, DISTRIBUTING);
 516                                 SM_FLAG_SET(port, NTT);
 517                                 MODE4_DEBUG("COLLECTING -> DISTRIBUTING\n");
 518                                 RTE_BOND_LOG(INFO,
 519                                         "Bond %u: slave id %u distributing started.",
 520                                         internals->port_id, slave_id);
 521                         }
 522                 } else {
 523                         if (!PARTNER_STATE(port, COLLECTING)) {
 524                                 /* Disable DISTRIBUTING (enter COLLECTING state) if partner
 525                                  * is not collecting */
 526                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 527                                 SM_FLAG_SET(port, NTT);
 528                                 MODE4_DEBUG("DISTRIBUTING -> COLLECTING\n");
 529                                 RTE_BOND_LOG(INFO,
 530                                         "Bond %u: slave id %u distributing stopped.",
 531                                         internals->port_id, slave_id);
 532                         }
 533                 }
 534         }
 535 }
 536
 537 /**
 538  * Function handles transmit state machine.
 539  *
 540  * Function implements Transmit Machine from point 5.4.16 in 802.1AX
 541  * documentation.
 542  *
 543  * @param port
 544  */
 545 static void
 546 tx_machine(struct bond_dev_private *internals, uint16_t slave_id)
 547 {
 548         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 549
 550         struct rte_mbuf *lacp_pkt = NULL;
 551         struct lacpdu_header *hdr;
 552         struct lacpdu *lacpdu;
 553
 554         /* If periodic timer is not running periodic machine is in NO PERIODIC and
 555          * according to 802.3ax standard tx machine should not transmit any frames
 556          * and set ntt to false. */
 557         if (timer_is_stopped(&port->periodic_timer))
 558                 SM_FLAG_CLR(port, NTT);
 559
 560         if (!SM_FLAG(port, NTT))
 561                 return;
 562
 563         if (!timer_is_expired(&port->tx_machine_timer))
 564                 return;
 565
 566         lacp_pkt = rte_pktmbuf_alloc(port->mbuf_pool);
 567         if (lacp_pkt == NULL) {
 568                 RTE_BOND_LOG(ERR, "Failed to allocate LACP packet from pool");
 569                 return;
 570         }
 571
 572         lacp_pkt->data_len = sizeof(*hdr);
 573         lacp_pkt->pkt_len = sizeof(*hdr);
 574
 575         hdr = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 576
 577         /* Source and destination MAC */
 578         rte_ether_addr_copy(&lacp_mac_addr, &hdr->eth_hdr.d_addr);
 579         rte_eth_macaddr_get(slave_id, &hdr->eth_hdr.s_addr);
 580         hdr->eth_hdr.ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_SLOW);
 581
 582         lacpdu = &hdr->lacpdu;
 583         memset(lacpdu, 0, sizeof(*lacpdu));
 584
 585         /* Initialize LACP part */
 586         lacpdu->subtype = SLOW_SUBTYPE_LACP;
 587         lacpdu->version_number = 1;
 588
 589         /* ACTOR */
 590         lacpdu->actor.tlv_type_info = TLV_TYPE_ACTOR_INFORMATION;
 591         lacpdu->actor.info_length = sizeof(struct lacpdu_actor_partner_params);
 592         memcpy(&hdr->lacpdu.actor.port_params, &port->actor,
 593                         sizeof(port->actor));
 594         agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 595         rte_ether_addr_copy(&agg->actor.system,
 596                         &hdr->lacpdu.actor.port_params.system);
 597         lacpdu->actor.state = port->actor_state;
 598
 599         /* PARTNER */
 600         lacpdu->partner.tlv_type_info = TLV_TYPE_PARTNER_INFORMATION;
 601         lacpdu->partner.info_length = sizeof(struct lacpdu_actor_partner_params);
 602         memcpy(&lacpdu->partner.port_params, &port->partner,
 603                         sizeof(struct port_params));
 604         lacpdu->partner.state = port->partner_state;
 605
 606         /* Other fields */
 607         lacpdu->tlv_type_collector_info = TLV_TYPE_COLLECTOR_INFORMATION;
 608         lacpdu->collector_info_length = 0x10;
 609         lacpdu->collector_max_delay = 0;
 610
 611         lacpdu->tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION;
 612         lacpdu->terminator_length = 0;
 613
 614         MODE4_DEBUG("Sending LACP frame\n");
 615         BOND_PRINT_LACP(lacpdu);
 616
 617         if (internals->mode4.dedicated_queues.enabled == 0) {
 618                 int retval = rte_ring_enqueue(port->tx_ring, lacp_pkt);
 619                 if (retval != 0) {
 620                         /* If TX ring full, drop packet and free message.
 621                            Retransmission will happen in next function call. */
 622                         rte_pktmbuf_free(lacp_pkt);
 623                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 624                         return;
 625                 }
 626         } else {
 627                 uint16_t pkts_sent = rte_eth_tx_burst(slave_id,
 628                                 internals->mode4.dedicated_queues.tx_qid,
 629                                 &lacp_pkt, 1);
 630                 if (pkts_sent != 1) {
 631                         rte_pktmbuf_free(lacp_pkt);
 632                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 633                         return;
 634                 }
 635         }
 636
 637
 638         timer_set(&port->tx_machine_timer, internals->mode4.tx_period_timeout);
 639         SM_FLAG_CLR(port, NTT);
 640 }
 641
 642 static uint16_t
 643 max_index(uint64_t *a, int n)
 644 {
 645         if (n <= 0)
 646                 return -1;
 647
 648         int i, max_i = 0;
 649         uint64_t max = a[0];
 650
 651         for (i = 1; i < n; ++i) {
 652                 if (a[i] > max) {
 653                         max = a[i];
 654                         max_i = i;
 655                 }
 656         }
 657
 658         return max_i;
 659 }
 660
 661 /**
 662  * Function assigns port to aggregator.
 663  *
 664  * @param bond_dev_private      Pointer to bond_dev_private structure.
 665  * @param port_pos                      Port to assign.
 666  */
 667 static void
 668 selection_logic(struct bond_dev_private *internals, uint16_t slave_id)
 669 {
 670         struct port *agg, *port;
 671         uint16_t slaves_count, new_agg_id, i, j = 0;
 672         uint16_t *slaves;
 673         uint64_t agg_bandwidth[RTE_MAX_ETHPORTS] = {0};
 674         uint64_t agg_count[RTE_MAX_ETHPORTS] = {0};
 675         uint16_t default_slave = 0;
 676         struct rte_eth_link link_info;
 677         uint16_t agg_new_idx = 0;
 678         int ret;
 679
 680         slaves = internals->active_slaves;
 681         slaves_count = internals->active_slave_count;
 682         port = &bond_mode_8023ad_ports[slave_id];
 683
 684         /* Search for aggregator suitable for this port */
 685         for (i = 0; i < slaves_count; ++i) {
 686                 agg = &bond_mode_8023ad_ports[slaves[i]];
 687                 /* Skip ports that are not aggreagators */
 688                 if (agg->aggregator_port_id != slaves[i])
 689                         continue;
 690
 691                 ret = rte_eth_link_get_nowait(slaves[i], &link_info);
 692                 if (ret < 0) {
 693                         RTE_BOND_LOG(ERR,
 694                                 "Slave (port %u) link get failed: %s\n",
 695                                 slaves[i], rte_strerror(-ret));
 696                         continue;
 697                 }
 698                 agg_count[i] += 1;
 699                 agg_bandwidth[i] += link_info.link_speed;
 700
 701                 /* Actors system ID is not checked since all slave device have the same
 702                  * ID (MAC address). */
 703                 if ((agg->actor.key == port->actor.key &&
 704                         agg->partner.system_priority == port->partner.system_priority &&
 705                         rte_is_same_ether_addr(&agg->partner.system,
 706                                         &port->partner.system) == 1
 707                         && (agg->partner.key == port->partner.key)) &&
 708                         rte_is_zero_ether_addr(&port->partner.system) != 1 &&
 709                         (agg->actor.key &
 710                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) != 0) {
 711
 712                         if (j == 0)
 713                                 default_slave = i;
 714                         j++;
 715                 }
 716         }
 717
 718         switch (internals->mode4.agg_selection) {
 719         case AGG_COUNT:
 720                 agg_new_idx = max_index(agg_count, slaves_count);
 721                 new_agg_id = slaves[agg_new_idx];
 722                 break;
 723         case AGG_BANDWIDTH:
 724                 agg_new_idx = max_index(agg_bandwidth, slaves_count);
 725                 new_agg_id = slaves[agg_new_idx];
 726                 break;
 727         case AGG_STABLE:
 728                 if (default_slave == slaves_count)
 729                         new_agg_id = slaves[slave_id];
 730                 else
 731                         new_agg_id = slaves[default_slave];
 732                 break;
 733         default:
 734                 if (default_slave == slaves_count)
 735                         new_agg_id = slaves[slave_id];
 736                 else
 737                         new_agg_id = slaves[default_slave];
 738                 break;
 739         }
 740
 741         if (new_agg_id != port->aggregator_port_id) {
 742                 port->aggregator_port_id = new_agg_id;
 743
 744                 MODE4_DEBUG("-> SELECTED: ID=%3u\n"
 745                         "\t%s aggregator ID=%3u\n",
 746                         port->aggregator_port_id,
 747                         port->aggregator_port_id == slave_id ?
 748                                 "aggregator not found, using default" : "aggregator found",
 749                         port->aggregator_port_id);
 750         }
 751
 752         port->selected = SELECTED;
 753 }
 754
 755 /* Function maps DPDK speed to bonding speed stored in key field */
 756 static uint16_t
 757 link_speed_key(uint16_t speed) {
 758         uint16_t key_speed;
 759
 760         switch (speed) {
 761         case ETH_SPEED_NUM_NONE:
 762                 key_speed = 0x00;
 763                 break;
 764         case ETH_SPEED_NUM_10M:
 765                 key_speed = BOND_LINK_SPEED_KEY_10M;
 766                 break;
 767         case ETH_SPEED_NUM_100M:
 768                 key_speed = BOND_LINK_SPEED_KEY_100M;
 769                 break;
 770         case ETH_SPEED_NUM_1G:
 771                 key_speed = BOND_LINK_SPEED_KEY_1000M;
 772                 break;
 773         case ETH_SPEED_NUM_10G:
 774                 key_speed = BOND_LINK_SPEED_KEY_10G;
 775                 break;
 776         case ETH_SPEED_NUM_20G:
 777                 key_speed = BOND_LINK_SPEED_KEY_20G;
 778                 break;
 779         case ETH_SPEED_NUM_40G:
 780                 key_speed = BOND_LINK_SPEED_KEY_40G;
 781                 break;
 782         default:
 783                 /* Unknown speed*/
 784                 key_speed = 0xFFFF;
 785         }
 786
 787         return key_speed;
 788 }
 789
 790 static void
 791 rx_machine_update(struct bond_dev_private *internals, uint16_t slave_id,
 792                 struct rte_mbuf *lacp_pkt) {
 793         struct lacpdu_header *lacp;
 794         struct lacpdu_actor_partner_params *partner;
 795
 796         if (lacp_pkt != NULL) {
 797                 lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 798                 RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
 799
 800                 partner = &lacp->lacpdu.partner;
 801                 if (rte_is_zero_ether_addr(&partner->port_params.system) ||
 802                         rte_is_same_ether_addr(&partner->port_params.system,
 803                         &internals->mode4.mac_addr)) {
 804                         /* This LACP frame is sending to the bonding port
 805                          * so pass it to rx_machine.
 806                          */
 807                         rx_machine(internals, slave_id, &lacp->lacpdu);
 808                 }
 809                 rte_pktmbuf_free(lacp_pkt);
 810         } else
 811                 rx_machine(internals, slave_id, NULL);
 812 }
 813
 814 static void
 815 bond_mode_8023ad_periodic_cb(void *arg)
 816 {
 817         struct rte_eth_dev *bond_dev = arg;
 818         struct bond_dev_private *internals = bond_dev->data->dev_private;
 819         struct port *port;
 820         struct rte_eth_link link_info;
 821         struct rte_ether_addr slave_addr;
 822         struct rte_mbuf *lacp_pkt = NULL;
 823         uint16_t slave_id;
 824         uint16_t i;
 825
 826
 827         /* Update link status on each port */
 828         for (i = 0; i < internals->active_slave_count; i++) {
 829                 uint16_t key;
 830                 int ret;
 831
 832                 slave_id = internals->active_slaves[i];
 833                 ret = rte_eth_link_get_nowait(slave_id, &link_info);
 834                 if (ret < 0) {
 835                         RTE_BOND_LOG(ERR,
 836                                 "Slave (port %u) link get failed: %s\n",
 837                                 slave_id, rte_strerror(-ret));
 838                 }
 839
 840                 if (ret >= 0 && link_info.link_status != 0) {
 841                         key = link_speed_key(link_info.link_speed) << 1;
 842                         if (link_info.link_duplex == ETH_LINK_FULL_DUPLEX)
 843                                 key |= BOND_LINK_FULL_DUPLEX_KEY;
 844                 } else {
 845                         key = 0;
 846                 }
 847
 848                 rte_eth_macaddr_get(slave_id, &slave_addr);
 849                 port = &bond_mode_8023ad_ports[slave_id];
 850
 851                 key = rte_cpu_to_be_16(key);
 852                 if (key != port->actor.key) {
 853                         if (!(key & rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)))
 854                                 set_warning_flags(port, WRN_NOT_LACP_CAPABLE);
 855
 856                         port->actor.key = key;
 857                         SM_FLAG_SET(port, NTT);
 858                 }
 859
 860                 if (!rte_is_same_ether_addr(&port->actor.system, &slave_addr)) {
 861                         rte_ether_addr_copy(&slave_addr, &port->actor.system);
 862                         if (port->aggregator_port_id == slave_id)
 863                                 SM_FLAG_SET(port, NTT);
 864                 }
 865         }
 866
 867         for (i = 0; i < internals->active_slave_count; i++) {
 868                 slave_id = internals->active_slaves[i];
 869                 port = &bond_mode_8023ad_ports[slave_id];
 870
 871                 if ((port->actor.key &
 872                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) == 0) {
 873
 874                         SM_FLAG_SET(port, BEGIN);
 875
 876                         /* LACP is disabled on half duples or link is down */
 877                         if (SM_FLAG(port, LACP_ENABLED)) {
 878                                 /* If port was enabled set it to BEGIN state */
 879                                 SM_FLAG_CLR(port, LACP_ENABLED);
 880                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 881                                 ACTOR_STATE_CLR(port, COLLECTING);
 882                         }
 883
 884                         /* Skip this port processing */
 885                         continue;
 886                 }
 887
 888                 SM_FLAG_SET(port, LACP_ENABLED);
 889
 890                 if (internals->mode4.dedicated_queues.enabled == 0) {
 891                         /* Find LACP packet to this port. Do not check subtype,
 892                          * it is done in function that queued packet
 893                          */
 894                         int retval = rte_ring_dequeue(port->rx_ring,
 895                                         (void **)&lacp_pkt);
 896
 897                         if (retval != 0)
 898                                 lacp_pkt = NULL;
 899
 900                         rx_machine_update(internals, slave_id, lacp_pkt);
 901                 } else {
 902                         uint16_t rx_count = rte_eth_rx_burst(slave_id,
 903                                         internals->mode4.dedicated_queues.rx_qid,
 904                                         &lacp_pkt, 1);
 905
 906                         if (rx_count == 1)
 907                                 bond_mode_8023ad_handle_slow_pkt(internals,
 908                                                 slave_id, lacp_pkt);
 909                         else
 910                                 rx_machine_update(internals, slave_id, NULL);
 911                 }
 912
 913                 periodic_machine(internals, slave_id);
 914                 mux_machine(internals, slave_id);
 915                 tx_machine(internals, slave_id);
 916                 selection_logic(internals, slave_id);
 917
 918                 SM_FLAG_CLR(port, BEGIN);
 919                 show_warnings(slave_id);
 920         }
 921
 922         rte_eal_alarm_set(internals->mode4.update_timeout_us,
 923                         bond_mode_8023ad_periodic_cb, arg);
 924 }
 925
 926 static int
 927 bond_mode_8023ad_register_lacp_mac(uint16_t slave_id)
 928 {
 929         int ret;
 930
 931         ret = rte_eth_allmulticast_enable(slave_id);
 932         if (ret != 0) {
 933                 RTE_BOND_LOG(ERR,
 934                         "failed to enable allmulti mode for port %u: %s",
 935                         slave_id, rte_strerror(-ret));
 936         }
 937         if (rte_eth_allmulticast_get(slave_id)) {
 938                 RTE_BOND_LOG(DEBUG, "forced allmulti for port %u",
 939                              slave_id);
 940                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 941                                 BOND_8023AD_FORCED_ALLMULTI;
 942                 return 0;
 943         }
 944
 945         ret = rte_eth_promiscuous_enable(slave_id);
 946         if (ret != 0) {
 947                 RTE_BOND_LOG(ERR,
 948                         "failed to enable promiscuous mode for port %u: %s",
 949                         slave_id, rte_strerror(-ret));
 950         }
 951         if (rte_eth_promiscuous_get(slave_id)) {
 952                 RTE_BOND_LOG(DEBUG, "forced promiscuous for port %u",
 953                              slave_id);
 954                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 955                                 BOND_8023AD_FORCED_PROMISC;
 956                 return 0;
 957         }
 958
 959         return -1;
 960 }
 961
 962 static void
 963 bond_mode_8023ad_unregister_lacp_mac(uint16_t slave_id)
 964 {
 965         int ret;
 966
 967         switch (bond_mode_8023ad_ports[slave_id].forced_rx_flags) {
 968         case BOND_8023AD_FORCED_ALLMULTI:
 969                 RTE_BOND_LOG(DEBUG, "unset allmulti for port %u", slave_id);
 970                 ret = rte_eth_allmulticast_disable(slave_id);
 971                 if (ret != 0)
 972                         RTE_BOND_LOG(ERR,
 973                                 "failed to disable allmulti mode for port %u: %s",
 974                                 slave_id, rte_strerror(-ret));
 975                 break;
 976
 977         case BOND_8023AD_FORCED_PROMISC:
 978                 RTE_BOND_LOG(DEBUG, "unset promisc for port %u", slave_id);
 979                 ret = rte_eth_promiscuous_disable(slave_id);
 980                 if (ret != 0)
 981                         RTE_BOND_LOG(ERR,
 982                                 "failed to disable promiscuous mode for port %u: %s",
 983                                 slave_id, rte_strerror(-ret));
 984                 break;
 985
 986         default:
 987                 break;
 988         }
 989 }
 990
 991 void
 992 bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev,
 993                                 uint16_t slave_id)
 994 {
 995         struct bond_dev_private *internals = bond_dev->data->dev_private;
 996
 997         struct port *port = &bond_mode_8023ad_ports[slave_id];
 998         struct port_params initial = {
 999                         .system = { { 0 } },
1000                         .system_priority = rte_cpu_to_be_16(0xFFFF),
1001                         .key = rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY),
1002                         .port_priority = rte_cpu_to_be_16(0x00FF),
1003                         .port_number = 0,
1004         };
1005
1006         char mem_name[RTE_ETH_NAME_MAX_LEN];
1007         int socket_id;
1008         unsigned element_size;
1009         uint32_t total_tx_desc;
1010         struct bond_tx_queue *bd_tx_q;
1011         uint16_t q_id;
1012
1013         /* Given slave mus not be in active list */
1014         RTE_ASSERT(find_slave_by_id(internals->active_slaves,
1015         internals->active_slave_count, slave_id) == internals->active_slave_count);
1016         RTE_SET_USED(internals); /* used only for assert when enabled */
1017
1018         memcpy(&port->actor, &initial, sizeof(struct port_params));
1019         /* Standard requires that port ID must be grater than 0.
1020          * Add 1 do get corresponding port_number */
1021         port->actor.port_number = rte_cpu_to_be_16(slave_id + 1);
1022
1023         memcpy(&port->partner, &initial, sizeof(struct port_params));
1024
1025         /* default states */
1026         port->actor_state = STATE_AGGREGATION | STATE_LACP_ACTIVE | STATE_DEFAULTED;
1027         port->partner_state = STATE_LACP_ACTIVE | STATE_AGGREGATION;
1028         port->sm_flags = SM_FLAGS_BEGIN;
1029
1030         /* use this port as agregator */
1031         port->aggregator_port_id = slave_id;
1032
1033         if (bond_mode_8023ad_register_lacp_mac(slave_id) < 0) {
1034                 RTE_BOND_LOG(WARNING, "slave %u is most likely broken and won't receive LACP packets",
1035                              slave_id);
1036         }
1037
1038         timer_cancel(&port->warning_timer);
1039
1040         if (port->mbuf_pool != NULL)
1041                 return;
1042
1043         RTE_ASSERT(port->rx_ring == NULL);
1044         RTE_ASSERT(port->tx_ring == NULL);
1045
1046         socket_id = rte_eth_dev_socket_id(slave_id);
1047         if (socket_id == -1)
1048                 socket_id = rte_socket_id();
1049
1050         element_size = sizeof(struct slow_protocol_frame) +
1051                                 RTE_PKTMBUF_HEADROOM;
1052
1053         /* The size of the mempool should be at least:
1054          * the sum of the TX descriptors + BOND_MODE_8023AX_SLAVE_TX_PKTS */
1055         total_tx_desc = BOND_MODE_8023AX_SLAVE_TX_PKTS;
1056         for (q_id = 0; q_id < bond_dev->data->nb_tx_queues; q_id++) {
1057                 bd_tx_q = (struct bond_tx_queue*)bond_dev->data->tx_queues[q_id];
1058                 total_tx_desc += bd_tx_q->nb_tx_desc;
1059         }
1060
1061         snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_pool", slave_id);
1062         port->mbuf_pool = rte_pktmbuf_pool_create(mem_name, total_tx_desc,
1063                 RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ?
1064                         32 : RTE_MEMPOOL_CACHE_MAX_SIZE,
1065                 0, element_size, socket_id);
1066
1067         /* Any memory allocation failure in initialization is critical because
1068          * resources can't be free, so reinitialization is impossible. */
1069         if (port->mbuf_pool == NULL) {
1070                 rte_panic("Slave %u: Failed to create memory pool '%s': %s\n",
1071                         slave_id, mem_name, rte_strerror(rte_errno));
1072         }
1073
1074         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_rx", slave_id);
1075         port->rx_ring = rte_ring_create(mem_name,
1076                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_RX_PKTS), socket_id, 0);
1077
1078         if (port->rx_ring == NULL) {
1079                 rte_panic("Slave %u: Failed to create rx ring '%s': %s\n", slave_id,
1080                         mem_name, rte_strerror(rte_errno));
1081         }
1082
1083         /* TX ring is at least one pkt longer to make room for marker packet. */
1084         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_tx", slave_id);
1085         port->tx_ring = rte_ring_create(mem_name,
1086                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_TX_PKTS + 1), socket_id, 0);
1087
1088         if (port->tx_ring == NULL) {
1089                 rte_panic("Slave %u: Failed to create tx ring '%s': %s\n", slave_id,
1090                         mem_name, rte_strerror(rte_errno));
1091         }
1092 }
1093
1094 int
1095 bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev __rte_unused,
1096                 uint16_t slave_id)
1097 {
1098         void *pkt = NULL;
1099         struct port *port = NULL;
1100         uint8_t old_partner_state;
1101
1102         port = &bond_mode_8023ad_ports[slave_id];
1103
1104         ACTOR_STATE_CLR(port, AGGREGATION);
1105         port->selected = UNSELECTED;
1106
1107         old_partner_state = port->partner_state;
1108         record_default(port);
1109
1110         bond_mode_8023ad_unregister_lacp_mac(slave_id);
1111
1112         /* If partner timeout state changes then disable timer */
1113         if (!((old_partner_state ^ port->partner_state) &
1114                         STATE_LACP_SHORT_TIMEOUT))
1115                 timer_cancel(&port->current_while_timer);
1116
1117         PARTNER_STATE_CLR(port, AGGREGATION);
1118         ACTOR_STATE_CLR(port, EXPIRED);
1119
1120         /* flush rx/tx rings */
1121         while (rte_ring_dequeue(port->rx_ring, &pkt) == 0)
1122                 rte_pktmbuf_free((struct rte_mbuf *)pkt);
1123
1124         while (rte_ring_dequeue(port->tx_ring, &pkt) == 0)
1125                         rte_pktmbuf_free((struct rte_mbuf *)pkt);
1126         return 0;
1127 }
1128
1129 void
1130 bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev)
1131 {
1132         struct bond_dev_private *internals = bond_dev->data->dev_private;
1133         struct rte_ether_addr slave_addr;
1134         struct port *slave, *agg_slave;
1135         uint16_t slave_id, i, j;
1136
1137         bond_mode_8023ad_stop(bond_dev);
1138
1139         for (i = 0; i < internals->active_slave_count; i++) {
1140                 slave_id = internals->active_slaves[i];
1141                 slave = &bond_mode_8023ad_ports[slave_id];
1142                 rte_eth_macaddr_get(slave_id, &slave_addr);
1143
1144                 if (rte_is_same_ether_addr(&slave_addr, &slave->actor.system))
1145                         continue;
1146
1147                 rte_ether_addr_copy(&slave_addr, &slave->actor.system);
1148                 /* Do nothing if this port is not an aggregator. In other case
1149                  * Set NTT flag on every port that use this aggregator. */
1150                 if (slave->aggregator_port_id != slave_id)
1151                         continue;
1152
1153                 for (j = 0; j < internals->active_slave_count; j++) {
1154                         agg_slave = &bond_mode_8023ad_ports[internals->active_slaves[j]];
1155                         if (agg_slave->aggregator_port_id == slave_id)
1156                                 SM_FLAG_SET(agg_slave, NTT);
1157                 }
1158         }
1159
1160         if (bond_dev->data->dev_started)
1161                 bond_mode_8023ad_start(bond_dev);
1162 }
1163
1164 static void
1165 bond_mode_8023ad_conf_get(struct rte_eth_dev *dev,
1166                 struct rte_eth_bond_8023ad_conf *conf)
1167 {
1168         struct bond_dev_private *internals = dev->data->dev_private;
1169         struct mode8023ad_private *mode4 = &internals->mode4;
1170         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1171
1172         conf->fast_periodic_ms = mode4->fast_periodic_timeout / ms_ticks;
1173         conf->slow_periodic_ms = mode4->slow_periodic_timeout / ms_ticks;
1174         conf->short_timeout_ms = mode4->short_timeout / ms_ticks;
1175         conf->long_timeout_ms = mode4->long_timeout / ms_ticks;
1176         conf->aggregate_wait_timeout_ms = mode4->aggregate_wait_timeout / ms_ticks;
1177         conf->tx_period_ms = mode4->tx_period_timeout / ms_ticks;
1178         conf->update_timeout_ms = mode4->update_timeout_us / 1000;
1179         conf->rx_marker_period_ms = mode4->rx_marker_timeout / ms_ticks;
1180         conf->slowrx_cb = mode4->slowrx_cb;
1181         conf->agg_selection = mode4->agg_selection;
1182 }
1183
1184 static void
1185 bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf)
1186 {
1187         conf->fast_periodic_ms = BOND_8023AD_FAST_PERIODIC_MS;
1188         conf->slow_periodic_ms = BOND_8023AD_SLOW_PERIODIC_MS;
1189         conf->short_timeout_ms = BOND_8023AD_SHORT_TIMEOUT_MS;
1190         conf->long_timeout_ms = BOND_8023AD_LONG_TIMEOUT_MS;
1191         conf->aggregate_wait_timeout_ms = BOND_8023AD_AGGREGATE_WAIT_TIMEOUT_MS;
1192         conf->tx_period_ms = BOND_8023AD_TX_MACHINE_PERIOD_MS;
1193         conf->rx_marker_period_ms = BOND_8023AD_RX_MARKER_PERIOD_MS;
1194         conf->update_timeout_ms = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS;
1195         conf->slowrx_cb = NULL;
1196         conf->agg_selection = AGG_STABLE;
1197 }
1198
1199 static void
1200 bond_mode_8023ad_conf_assign(struct mode8023ad_private *mode4,
1201                 struct rte_eth_bond_8023ad_conf *conf)
1202 {
1203         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1204
1205         mode4->fast_periodic_timeout = conf->fast_periodic_ms * ms_ticks;
1206         mode4->slow_periodic_timeout = conf->slow_periodic_ms * ms_ticks;
1207         mode4->short_timeout = conf->short_timeout_ms * ms_ticks;
1208         mode4->long_timeout = conf->long_timeout_ms * ms_ticks;
1209         mode4->aggregate_wait_timeout = conf->aggregate_wait_timeout_ms * ms_ticks;
1210         mode4->tx_period_timeout = conf->tx_period_ms * ms_ticks;
1211         mode4->rx_marker_timeout = conf->rx_marker_period_ms * ms_ticks;
1212         mode4->update_timeout_us = conf->update_timeout_ms * 1000;
1213
1214         mode4->dedicated_queues.enabled = 0;
1215         mode4->dedicated_queues.rx_qid = UINT16_MAX;
1216         mode4->dedicated_queues.tx_qid = UINT16_MAX;
1217 }
1218
1219 void
1220 bond_mode_8023ad_setup(struct rte_eth_dev *dev,
1221                 struct rte_eth_bond_8023ad_conf *conf)
1222 {
1223         struct rte_eth_bond_8023ad_conf def_conf;
1224         struct bond_dev_private *internals = dev->data->dev_private;
1225         struct mode8023ad_private *mode4 = &internals->mode4;
1226
1227         if (conf == NULL) {
1228                 conf = &def_conf;
1229                 bond_mode_8023ad_conf_get_default(conf);
1230         }
1231
1232         bond_mode_8023ad_stop(dev);
1233         bond_mode_8023ad_conf_assign(mode4, conf);
1234         mode4->slowrx_cb = conf->slowrx_cb;
1235         mode4->agg_selection = AGG_STABLE;
1236
1237         if (dev->data->dev_started)
1238                 bond_mode_8023ad_start(dev);
1239 }
1240
1241 int
1242 bond_mode_8023ad_enable(struct rte_eth_dev *bond_dev)
1243 {
1244         struct bond_dev_private *internals = bond_dev->data->dev_private;
1245         uint16_t i;
1246
1247         for (i = 0; i < internals->active_slave_count; i++)
1248                 bond_mode_8023ad_activate_slave(bond_dev,
1249                                 internals->active_slaves[i]);
1250
1251         return 0;
1252 }
1253
1254 int
1255 bond_mode_8023ad_start(struct rte_eth_dev *bond_dev)
1256 {
1257         struct bond_dev_private *internals = bond_dev->data->dev_private;
1258         struct mode8023ad_private *mode4 = &internals->mode4;
1259         static const uint64_t us = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS * 1000;
1260
1261         rte_eth_macaddr_get(internals->port_id, &mode4->mac_addr);
1262         if (mode4->slowrx_cb)
1263                 return rte_eal_alarm_set(us, &bond_mode_8023ad_ext_periodic_cb,
1264                                          bond_dev);
1265
1266         return rte_eal_alarm_set(us, &bond_mode_8023ad_periodic_cb, bond_dev);
1267 }
1268
1269 void
1270 bond_mode_8023ad_stop(struct rte_eth_dev *bond_dev)
1271 {
1272         struct bond_dev_private *internals = bond_dev->data->dev_private;
1273         struct mode8023ad_private *mode4 = &internals->mode4;
1274
1275         if (mode4->slowrx_cb) {
1276                 rte_eal_alarm_cancel(&bond_mode_8023ad_ext_periodic_cb,
1277                                      bond_dev);
1278                 return;
1279         }
1280         rte_eal_alarm_cancel(&bond_mode_8023ad_periodic_cb, bond_dev);
1281 }
1282
1283 void
1284 bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
1285                                   uint16_t slave_id, struct rte_mbuf *pkt)
1286 {
1287         struct mode8023ad_private *mode4 = &internals->mode4;
1288         struct port *port = &bond_mode_8023ad_ports[slave_id];
1289         struct marker_header *m_hdr;
1290         uint64_t marker_timer, old_marker_timer;
1291         int retval;
1292         uint8_t wrn, subtype;
1293         /* If packet is a marker, we send response now by reusing given packet
1294          * and update only source MAC, destination MAC is multicast so don't
1295          * update it. Other frames will be handled later by state machines */
1296         subtype = rte_pktmbuf_mtod(pkt,
1297                         struct slow_protocol_frame *)->slow_protocol.subtype;
1298
1299         if (subtype == SLOW_SUBTYPE_MARKER) {
1300                 m_hdr = rte_pktmbuf_mtod(pkt, struct marker_header *);
1301
1302                 if (likely(m_hdr->marker.tlv_type_marker != MARKER_TLV_TYPE_INFO)) {
1303                         wrn = WRN_UNKNOWN_MARKER_TYPE;
1304                         goto free_out;
1305                 }
1306
1307                 /* Setup marker timer. Do it in loop in case concurrent access. */
1308                 do {
1309                         old_marker_timer = port->rx_marker_timer;
1310                         if (!timer_is_expired(&old_marker_timer)) {
1311                                 wrn = WRN_RX_MARKER_TO_FAST;
1312                                 goto free_out;
1313                         }
1314
1315                         timer_set(&marker_timer, mode4->rx_marker_timeout);
1316                         retval = rte_atomic64_cmpset(&port->rx_marker_timer,
1317                                 old_marker_timer, marker_timer);
1318                 } while (unlikely(retval == 0));
1319
1320                 m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP;
1321                 rte_eth_macaddr_get(slave_id, &m_hdr->eth_hdr.s_addr);
1322
1323                 if (internals->mode4.dedicated_queues.enabled == 0) {
1324                         int retval = rte_ring_enqueue(port->tx_ring, pkt);
1325                         if (retval != 0) {
1326                                 /* reset timer */
1327                                 port->rx_marker_timer = 0;
1328                                 wrn = WRN_TX_QUEUE_FULL;
1329                                 goto free_out;
1330                         }
1331                 } else {
1332                         /* Send packet directly to the slow queue */
1333                         uint16_t tx_count = rte_eth_tx_burst(slave_id,
1334                                         internals->mode4.dedicated_queues.tx_qid,
1335                                         &pkt, 1);
1336                         if (tx_count != 1) {
1337                                 /* reset timer */
1338                                 port->rx_marker_timer = 0;
1339                                 wrn = WRN_TX_QUEUE_FULL;
1340                                 goto free_out;
1341                         }
1342                 }
1343         } else if (likely(subtype == SLOW_SUBTYPE_LACP)) {
1344                 if (internals->mode4.dedicated_queues.enabled == 0) {
1345                         int retval = rte_ring_enqueue(port->rx_ring, pkt);
1346                         if (retval != 0) {
1347                                 /* If RX fing full free lacpdu message and drop packet */
1348                                 wrn = WRN_RX_QUEUE_FULL;
1349                                 goto free_out;
1350                         }
1351                 } else
1352                         rx_machine_update(internals, slave_id, pkt);
1353         } else {
1354                 wrn = WRN_UNKNOWN_SLOW_TYPE;
1355                 goto free_out;
1356         }
1357
1358         return;
1359
1360 free_out:
1361         set_warning_flags(port, wrn);
1362         rte_pktmbuf_free(pkt);
1363 }
1364
1365 int
1366 rte_eth_bond_8023ad_conf_get(uint16_t port_id,
1367                 struct rte_eth_bond_8023ad_conf *conf)
1368 {
1369         struct rte_eth_dev *bond_dev;
1370
1371         if (valid_bonded_port_id(port_id) != 0)
1372                 return -EINVAL;
1373
1374         if (conf == NULL)
1375                 return -EINVAL;
1376
1377         bond_dev = &rte_eth_devices[port_id];
1378         bond_mode_8023ad_conf_get(bond_dev, conf);
1379         return 0;
1380 }
1381
1382 int
1383 rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id,
1384                 enum rte_bond_8023ad_agg_selection agg_selection)
1385 {
1386         struct rte_eth_dev *bond_dev;
1387         struct bond_dev_private *internals;
1388         struct mode8023ad_private *mode4;
1389
1390         if (valid_bonded_port_id(port_id) != 0)
1391                 return -EINVAL;
1392
1393         bond_dev = &rte_eth_devices[port_id];
1394         internals = bond_dev->data->dev_private;
1395
1396         if (internals->mode != 4)
1397                 return -EINVAL;
1398
1399         mode4 = &internals->mode4;
1400         if (agg_selection == AGG_COUNT || agg_selection == AGG_BANDWIDTH
1401                         || agg_selection == AGG_STABLE)
1402                 mode4->agg_selection = agg_selection;
1403         return 0;
1404 }
1405
1406 int rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id)
1407 {
1408         struct rte_eth_dev *bond_dev;
1409         struct bond_dev_private *internals;
1410         struct mode8023ad_private *mode4;
1411
1412         if (valid_bonded_port_id(port_id) != 0)
1413                 return -EINVAL;
1414
1415         bond_dev = &rte_eth_devices[port_id];
1416         internals = bond_dev->data->dev_private;
1417
1418         if (internals->mode != 4)
1419                 return -EINVAL;
1420         mode4 = &internals->mode4;
1421
1422         return mode4->agg_selection;
1423 }
1424
1425
1426
1427 static int
1428 bond_8023ad_setup_validate(uint16_t port_id,
1429                 struct rte_eth_bond_8023ad_conf *conf)
1430 {
1431         if (valid_bonded_port_id(port_id) != 0)
1432                 return -EINVAL;
1433
1434         if (conf != NULL) {
1435                 /* Basic sanity check */
1436                 if (conf->slow_periodic_ms == 0 ||
1437                                 conf->fast_periodic_ms >= conf->slow_periodic_ms ||
1438                                 conf->long_timeout_ms == 0 ||
1439                                 conf->short_timeout_ms >= conf->long_timeout_ms ||
1440                                 conf->aggregate_wait_timeout_ms == 0 ||
1441                                 conf->tx_period_ms == 0 ||
1442                                 conf->rx_marker_period_ms == 0 ||
1443                                 conf->update_timeout_ms == 0) {
1444                         RTE_BOND_LOG(ERR, "given mode 4 configuration is invalid");
1445                         return -EINVAL;
1446                 }
1447         }
1448
1449         return 0;
1450 }
1451
1452
1453 int
1454 rte_eth_bond_8023ad_setup(uint16_t port_id,
1455                 struct rte_eth_bond_8023ad_conf *conf)
1456 {
1457         struct rte_eth_dev *bond_dev;
1458         int err;
1459
1460         err = bond_8023ad_setup_validate(port_id, conf);
1461         if (err != 0)
1462                 return err;
1463
1464         bond_dev = &rte_eth_devices[port_id];
1465         bond_mode_8023ad_setup(bond_dev, conf);
1466
1467         return 0;
1468 }
1469
1470
1471
1472
1473
1474 int
1475 rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id,
1476                 struct rte_eth_bond_8023ad_slave_info *info)
1477 {
1478         struct rte_eth_dev *bond_dev;
1479         struct bond_dev_private *internals;
1480         struct port *port;
1481
1482         if (info == NULL || valid_bonded_port_id(port_id) != 0 ||
1483                         rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1484                 return -EINVAL;
1485
1486         bond_dev = &rte_eth_devices[port_id];
1487
1488         internals = bond_dev->data->dev_private;
1489         if (find_slave_by_id(internals->active_slaves,
1490                         internals->active_slave_count, slave_id) ==
1491                                 internals->active_slave_count)
1492                 return -EINVAL;
1493
1494         port = &bond_mode_8023ad_ports[slave_id];
1495         info->selected = port->selected;
1496
1497         info->actor_state = port->actor_state;
1498         rte_memcpy(&info->actor, &port->actor, sizeof(port->actor));
1499
1500         info->partner_state = port->partner_state;
1501         rte_memcpy(&info->partner, &port->partner, sizeof(port->partner));
1502
1503         info->agg_port_id = port->aggregator_port_id;
1504         return 0;
1505 }
1506
1507 static int
1508 bond_8023ad_ext_validate(uint16_t port_id, uint16_t slave_id)
1509 {
1510         struct rte_eth_dev *bond_dev;
1511         struct bond_dev_private *internals;
1512         struct mode8023ad_private *mode4;
1513
1514         if (rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1515                 return -EINVAL;
1516
1517         bond_dev = &rte_eth_devices[port_id];
1518
1519         if (!bond_dev->data->dev_started)
1520                 return -EINVAL;
1521
1522         internals = bond_dev->data->dev_private;
1523         if (find_slave_by_id(internals->active_slaves,
1524                         internals->active_slave_count, slave_id) ==
1525                                 internals->active_slave_count)
1526                 return -EINVAL;
1527
1528         mode4 = &internals->mode4;
1529         if (mode4->slowrx_cb == NULL)
1530                 return -EINVAL;
1531
1532         return 0;
1533 }
1534
1535 int
1536 rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id,
1537                                 int enabled)
1538 {
1539         struct port *port;
1540         int res;
1541
1542         res = bond_8023ad_ext_validate(port_id, slave_id);
1543         if (res != 0)
1544                 return res;
1545
1546         port = &bond_mode_8023ad_ports[slave_id];
1547
1548         if (enabled)
1549                 ACTOR_STATE_SET(port, COLLECTING);
1550         else
1551                 ACTOR_STATE_CLR(port, COLLECTING);
1552
1553         return 0;
1554 }
1555
1556 int
1557 rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id,
1558                                 int enabled)
1559 {
1560         struct port *port;
1561         int res;
1562
1563         res = bond_8023ad_ext_validate(port_id, slave_id);
1564         if (res != 0)
1565                 return res;
1566
1567         port = &bond_mode_8023ad_ports[slave_id];
1568
1569         if (enabled)
1570                 ACTOR_STATE_SET(port, DISTRIBUTING);
1571         else
1572                 ACTOR_STATE_CLR(port, DISTRIBUTING);
1573
1574         return 0;
1575 }
1576
1577 int
1578 rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id)
1579 {
1580         struct port *port;
1581         int err;
1582
1583         err = bond_8023ad_ext_validate(port_id, slave_id);
1584         if (err != 0)
1585                 return err;
1586
1587         port = &bond_mode_8023ad_ports[slave_id];
1588         return ACTOR_STATE(port, DISTRIBUTING);
1589 }
1590
1591 int
1592 rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id)
1593 {
1594         struct port *port;
1595         int err;
1596
1597         err = bond_8023ad_ext_validate(port_id, slave_id);
1598         if (err != 0)
1599                 return err;
1600
1601         port = &bond_mode_8023ad_ports[slave_id];
1602         return ACTOR_STATE(port, COLLECTING);
1603 }
1604
1605 int
1606 rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id,
1607                 struct rte_mbuf *lacp_pkt)
1608 {
1609         struct port *port;
1610         int res;
1611
1612         res = bond_8023ad_ext_validate(port_id, slave_id);
1613         if (res != 0)
1614                 return res;
1615
1616         port = &bond_mode_8023ad_ports[slave_id];
1617
1618         if (rte_pktmbuf_pkt_len(lacp_pkt) < sizeof(struct lacpdu_header))
1619                 return -EINVAL;
1620
1621         struct lacpdu_header *lacp;
1622
1623         /* only enqueue LACPDUs */
1624         lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
1625         if (lacp->lacpdu.subtype != SLOW_SUBTYPE_LACP)
1626                 return -EINVAL;
1627
1628         MODE4_DEBUG("sending LACP frame\n");
1629
1630         return rte_ring_enqueue(port->tx_ring, lacp_pkt);
1631 }
1632
1633 static void
1634 bond_mode_8023ad_ext_periodic_cb(void *arg)
1635 {
1636         struct rte_eth_dev *bond_dev = arg;
1637         struct bond_dev_private *internals = bond_dev->data->dev_private;
1638         struct mode8023ad_private *mode4 = &internals->mode4;
1639         struct port *port;
1640         void *pkt = NULL;
1641         uint16_t i, slave_id;
1642
1643         for (i = 0; i < internals->active_slave_count; i++) {
1644                 slave_id = internals->active_slaves[i];
1645                 port = &bond_mode_8023ad_ports[slave_id];
1646
1647                 if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) {
1648                         struct rte_mbuf *lacp_pkt = pkt;
1649                         struct lacpdu_header *lacp;
1650
1651                         lacp = rte_pktmbuf_mtod(lacp_pkt,
1652                                                 struct lacpdu_header *);
1653                         RTE_VERIFY(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
1654
1655                         /* This is LACP frame so pass it to rx callback.
1656                          * Callback is responsible for freeing mbuf.
1657                          */
1658                         mode4->slowrx_cb(slave_id, lacp_pkt);
1659                 }
1660         }
1661
1662         rte_eal_alarm_set(internals->mode4.update_timeout_us,
1663                         bond_mode_8023ad_ext_periodic_cb, arg);
1664 }
1665
1666 int
1667 rte_eth_bond_8023ad_dedicated_queues_enable(uint16_t port)
1668 {
1669         int retval = 0;
1670         struct rte_eth_dev *dev;
1671         struct bond_dev_private *internals;
1672
1673         if (valid_bonded_port_id(port) != 0)
1674                 return -EINVAL;
1675
1676         dev = &rte_eth_devices[port];
1677         internals = dev->data->dev_private;
1678
1679         if (bond_8023ad_slow_pkt_hw_filter_supported(port) != 0)
1680                 return -1;
1681
1682         /* Device must be stopped to set up slow queue */
1683         if (dev->data->dev_started)
1684                 return -1;
1685
1686         internals->mode4.dedicated_queues.enabled = 1;
1687
1688         bond_ethdev_mode_set(dev, internals->mode);
1689         return retval;
1690 }
1691
1692 int
1693 rte_eth_bond_8023ad_dedicated_queues_disable(uint16_t port)
1694 {
1695         int retval = 0;
1696         struct rte_eth_dev *dev;
1697         struct bond_dev_private *internals;
1698
1699         if (valid_bonded_port_id(port) != 0)
1700                 return -EINVAL;
1701
1702         dev = &rte_eth_devices[port];
1703         internals = dev->data->dev_private;
1704
1705         /* Device must be stopped to set up slow queue */
1706         if (dev->data->dev_started)
1707                 return -1;
1708
1709         internals->mode4.dedicated_queues.enabled = 0;
1710
1711         bond_ethdev_mode_set(dev, internals->mode);
1712
1713         return retval;
1714 }