drivers/net/bonding/rte_eth_bond_8023ad.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <stddef.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_alarm.h>
  10 #include <rte_malloc.h>
  11 #include <rte_errno.h>
  12 #include <rte_cycles.h>
  13 #include <rte_compat.h>
  14
  15 #include "rte_eth_bond_private.h"
  16
  17 static void bond_mode_8023ad_ext_periodic_cb(void *arg);
  18 #ifdef RTE_LIBRTE_BOND_DEBUG_8023AD
  19
  20 #define MODE4_DEBUG(fmt, ...)                           \
  21         rte_log(RTE_LOG_DEBUG, bond_logtype,            \
  22                 "%6u [Port %u: %s] " fmt,               \
  23                 bond_dbg_get_time_diff_ms(), slave_id,  \
  24                 __func__, ##__VA_ARGS__)
  25
  26 static uint64_t start_time;
  27
  28 static unsigned
  29 bond_dbg_get_time_diff_ms(void)
  30 {
  31         uint64_t now;
  32
  33         now = rte_rdtsc();
  34         if (start_time == 0)
  35                 start_time = now;
  36
  37         return ((now - start_time) * 1000) / rte_get_tsc_hz();
  38 }
  39
  40 static void
  41 bond_print_lacp(struct lacpdu *l)
  42 {
  43         char a_address[18];
  44         char p_address[18];
  45         char a_state[256] = { 0 };
  46         char p_state[256] = { 0 };
  47
  48         static const char * const state_labels[] = {
  49                 "ACT", "TIMEOUT", "AGG", "SYNC", "COL", "DIST", "DEF", "EXP"
  50         };
  51
  52         int a_len = 0;
  53         int p_len = 0;
  54         uint8_t i;
  55         uint8_t *addr;
  56
  57         addr = l->actor.port_params.system.addr_bytes;
  58         snprintf(a_address, sizeof(a_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  59                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  60
  61         addr = l->partner.port_params.system.addr_bytes;
  62         snprintf(p_address, sizeof(p_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  63                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  64
  65         for (i = 0; i < 8; i++) {
  66                 if ((l->actor.state >> i) & 1) {
  67                         a_len += snprintf(&a_state[a_len], RTE_DIM(a_state) - a_len, "%s ",
  68                                 state_labels[i]);
  69                 }
  70
  71                 if ((l->partner.state >> i) & 1) {
  72                         p_len += snprintf(&p_state[p_len], RTE_DIM(p_state) - p_len, "%s ",
  73                                 state_labels[i]);
  74                 }
  75         }
  76
  77         if (a_len && a_state[a_len-1] == ' ')
  78                 a_state[a_len-1] = '\0';
  79
  80         if (p_len && p_state[p_len-1] == ' ')
  81                 p_state[p_len-1] = '\0';
  82
  83         RTE_BOND_LOG(DEBUG,
  84                      "LACP: {\n"
  85                      "  subtype= %02X\n"
  86                      "  ver_num=%02X\n"
  87                      "  actor={ tlv=%02X, len=%02X\n"
  88                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  89                      "       state={ %s }\n"
  90                      "  }\n"
  91                      "  partner={ tlv=%02X, len=%02X\n"
  92                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  93                      "       state={ %s }\n"
  94                      "  }\n"
  95                      "  collector={info=%02X, length=%02X, max_delay=%04X\n, "
  96                      "type_term=%02X, terminator_length = %02X }",
  97                      l->subtype,
  98                      l->version_number,
  99                      l->actor.tlv_type_info,
 100                      l->actor.info_length,
 101                      l->actor.port_params.system_priority,
 102                      a_address,
 103                      l->actor.port_params.key,
 104                      l->actor.port_params.port_priority,
 105                      l->actor.port_params.port_number,
 106                      a_state,
 107                      l->partner.tlv_type_info,
 108                      l->partner.info_length,
 109                      l->partner.port_params.system_priority,
 110                      p_address,
 111                      l->partner.port_params.key,
 112                      l->partner.port_params.port_priority,
 113                      l->partner.port_params.port_number,
 114                      p_state,
 115                      l->tlv_type_collector_info,
 116                      l->collector_info_length,
 117                      l->collector_max_delay,
 118                      l->tlv_type_terminator,
 119                      l->terminator_length);
 120
 121 }
 122
 123 #define BOND_PRINT_LACP(lacpdu) bond_print_lacp(lacpdu)
 124 #else
 125 #define BOND_PRINT_LACP(lacpdu) do { } while (0)
 126 #define MODE4_DEBUG(fmt, ...) do { } while (0)
 127 #endif
 128
 129 static const struct rte_ether_addr lacp_mac_addr = {
 130         .addr_bytes = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 }
 131 };
 132
 133 struct port bond_mode_8023ad_ports[RTE_MAX_ETHPORTS];
 134
 135 static void
 136 timer_cancel(uint64_t *timer)
 137 {
 138         *timer = 0;
 139 }
 140
 141 static void
 142 timer_set(uint64_t *timer, uint64_t timeout)
 143 {
 144         *timer = rte_rdtsc() + timeout;
 145 }
 146
 147 /* Forces given timer to be in expired state. */
 148 static void
 149 timer_force_expired(uint64_t *timer)
 150 {
 151         *timer = rte_rdtsc();
 152 }
 153
 154 static bool
 155 timer_is_stopped(uint64_t *timer)
 156 {
 157         return *timer == 0;
 158 }
 159
 160 static bool
 161 timer_is_expired(uint64_t *timer)
 162 {
 163         return *timer < rte_rdtsc();
 164 }
 165
 166 /* Timer is in running state if it is not stopped nor expired */
 167 static bool
 168 timer_is_running(uint64_t *timer)
 169 {
 170         return !timer_is_stopped(timer) && !timer_is_expired(timer);
 171 }
 172
 173 static void
 174 set_warning_flags(struct port *port, uint16_t flags)
 175 {
 176         int retval;
 177         uint16_t old;
 178         uint16_t new_flag = 0;
 179
 180         do {
 181                 old = port->warnings_to_show;
 182                 new_flag = old | flags;
 183                 retval = rte_atomic16_cmpset(&port->warnings_to_show, old, new_flag);
 184         } while (unlikely(retval == 0));
 185 }
 186
 187 static void
 188 show_warnings(uint16_t slave_id)
 189 {
 190         struct port *port = &bond_mode_8023ad_ports[slave_id];
 191         uint8_t warnings;
 192
 193         do {
 194                 warnings = port->warnings_to_show;
 195         } while (rte_atomic16_cmpset(&port->warnings_to_show, warnings, 0) == 0);
 196
 197         if (!warnings)
 198                 return;
 199
 200         if (!timer_is_expired(&port->warning_timer))
 201                 return;
 202
 203
 204         timer_set(&port->warning_timer, BOND_8023AD_WARNINGS_PERIOD_MS *
 205                         rte_get_tsc_hz() / 1000);
 206
 207         if (warnings & WRN_RX_QUEUE_FULL) {
 208                 RTE_BOND_LOG(DEBUG,
 209                              "Slave %u: failed to enqueue LACP packet into RX ring.\n"
 210                              "Receive and transmit functions must be invoked on bonded"
 211                              "interface at least 10 times per second or LACP will notwork correctly",
 212                              slave_id);
 213         }
 214
 215         if (warnings & WRN_TX_QUEUE_FULL) {
 216                 RTE_BOND_LOG(DEBUG,
 217                              "Slave %u: failed to enqueue LACP packet into TX ring.\n"
 218                              "Receive and transmit functions must be invoked on bonded"
 219                              "interface at least 10 times per second or LACP will not work correctly",
 220                              slave_id);
 221         }
 222
 223         if (warnings & WRN_RX_MARKER_TO_FAST)
 224                 RTE_BOND_LOG(INFO, "Slave %u: marker to early - ignoring.",
 225                              slave_id);
 226
 227         if (warnings & WRN_UNKNOWN_SLOW_TYPE) {
 228                 RTE_BOND_LOG(INFO,
 229                         "Slave %u: ignoring unknown slow protocol frame type",
 230                              slave_id);
 231         }
 232
 233         if (warnings & WRN_UNKNOWN_MARKER_TYPE)
 234                 RTE_BOND_LOG(INFO, "Slave %u: ignoring unknown marker type",
 235                              slave_id);
 236
 237         if (warnings & WRN_NOT_LACP_CAPABLE)
 238                 MODE4_DEBUG("Port %u is not LACP capable!\n", slave_id);
 239 }
 240
 241 static void
 242 record_default(struct port *port)
 243 {
 244         /* Record default parameters for partner. Partner admin parameters
 245          * are not implemented so set them to arbitrary default (last known) and
 246          * mark actor that parner is in defaulted state. */
 247         port->partner_state = STATE_LACP_ACTIVE;
 248         ACTOR_STATE_SET(port, DEFAULTED);
 249 }
 250
 251 /** Function handles rx state machine.
 252  *
 253  * This function implements Receive State Machine from point 5.4.12 in
 254  * 802.1AX documentation. It should be called periodically.
 255  *
 256  * @param lacpdu                LACPDU received.
 257  * @param port                  Port on which LACPDU was received.
 258  */
 259 static void
 260 rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 261                 struct lacpdu *lacp)
 262 {
 263         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 264         uint64_t timeout;
 265
 266         if (SM_FLAG(port, BEGIN)) {
 267                 /* Initialize stuff */
 268                 MODE4_DEBUG("-> INITIALIZE\n");
 269                 SM_FLAG_CLR(port, MOVED);
 270                 port->selected = UNSELECTED;
 271
 272                 record_default(port);
 273
 274                 ACTOR_STATE_CLR(port, EXPIRED);
 275                 timer_cancel(&port->current_while_timer);
 276
 277                 /* DISABLED: On initialization partner is out of sync */
 278                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 279
 280                 /* LACP DISABLED stuff if LACP not enabled on this port */
 281                 if (!SM_FLAG(port, LACP_ENABLED))
 282                         PARTNER_STATE_CLR(port, AGGREGATION);
 283                 else
 284                         PARTNER_STATE_SET(port, AGGREGATION);
 285         }
 286
 287         if (!SM_FLAG(port, LACP_ENABLED)) {
 288                 /* Update parameters only if state changed */
 289                 if (!timer_is_stopped(&port->current_while_timer)) {
 290                         port->selected = UNSELECTED;
 291                         record_default(port);
 292                         PARTNER_STATE_CLR(port, AGGREGATION);
 293                         ACTOR_STATE_CLR(port, EXPIRED);
 294                         timer_cancel(&port->current_while_timer);
 295                 }
 296                 return;
 297         }
 298
 299         if (lacp) {
 300                 MODE4_DEBUG("LACP -> CURRENT\n");
 301                 BOND_PRINT_LACP(lacp);
 302                 /* Update selected flag. If partner parameters are defaulted assume they
 303                  * are match. If not defaulted  compare LACP actor with ports parner
 304                  * params. */
 305                 if (!ACTOR_STATE(port, DEFAULTED) &&
 306                         (ACTOR_STATE(port, AGGREGATION) != PARTNER_STATE(port, AGGREGATION)
 307                         || memcmp(&port->partner, &lacp->actor.port_params,
 308                                 sizeof(port->partner)) != 0)) {
 309                         MODE4_DEBUG("selected <- UNSELECTED\n");
 310                         port->selected = UNSELECTED;
 311                 }
 312
 313                 /* Record this PDU actor params as partner params */
 314                 memcpy(&port->partner, &lacp->actor.port_params,
 315                         sizeof(struct port_params));
 316                 port->partner_state = lacp->actor.state;
 317
 318                 /* Partner parameters are not defaulted any more */
 319                 ACTOR_STATE_CLR(port, DEFAULTED);
 320
 321                 /* If LACP partner params match this port actor params */
 322                 agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 323                 bool match = port->actor.system_priority ==
 324                         lacp->partner.port_params.system_priority &&
 325                         rte_is_same_ether_addr(&agg->actor.system,
 326                         &lacp->partner.port_params.system) &&
 327                         port->actor.port_priority ==
 328                         lacp->partner.port_params.port_priority &&
 329                         port->actor.port_number ==
 330                         lacp->partner.port_params.port_number;
 331
 332                 /* Update NTT if partners information are outdated (xored and masked
 333                  * bits are set)*/
 334                 uint8_t state_mask = STATE_LACP_ACTIVE | STATE_LACP_SHORT_TIMEOUT |
 335                         STATE_SYNCHRONIZATION | STATE_AGGREGATION;
 336
 337                 if (((port->actor_state ^ lacp->partner.state) & state_mask) ||
 338                                 match == false) {
 339                         SM_FLAG_SET(port, NTT);
 340                 }
 341
 342                 /* If LACP partner params match this port actor params */
 343                 if (match == true && ACTOR_STATE(port, AGGREGATION) ==
 344                                 PARTNER_STATE(port,     AGGREGATION))
 345                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 346                 else if (!PARTNER_STATE(port, AGGREGATION) && ACTOR_STATE(port,
 347                                 AGGREGATION))
 348                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 349                 else
 350                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 351
 352                 if (ACTOR_STATE(port, LACP_SHORT_TIMEOUT))
 353                         timeout = internals->mode4.short_timeout;
 354                 else
 355                         timeout = internals->mode4.long_timeout;
 356
 357                 timer_set(&port->current_while_timer, timeout);
 358                 ACTOR_STATE_CLR(port, EXPIRED);
 359                 return; /* No state change */
 360         }
 361
 362         /* If CURRENT state timer is not running (stopped or expired)
 363          * transit to EXPIRED state from DISABLED or CURRENT */
 364         if (!timer_is_running(&port->current_while_timer)) {
 365                 ACTOR_STATE_SET(port, EXPIRED);
 366                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 367                 PARTNER_STATE_SET(port, LACP_SHORT_TIMEOUT);
 368                 timer_set(&port->current_while_timer, internals->mode4.short_timeout);
 369         }
 370 }
 371
 372 /**
 373  * Function handles periodic tx state machine.
 374  *
 375  * Function implements Periodic Transmission state machine from point 5.4.13
 376  * in 802.1AX documentation. It should be called periodically.
 377  *
 378  * @param port                  Port to handle state machine.
 379  */
 380 static void
 381 periodic_machine(struct bond_dev_private *internals, uint16_t slave_id)
 382 {
 383         struct port *port = &bond_mode_8023ad_ports[slave_id];
 384         /* Calculate if either site is LACP enabled */
 385         uint64_t timeout;
 386         uint8_t active = ACTOR_STATE(port, LACP_ACTIVE) ||
 387                 PARTNER_STATE(port, LACP_ACTIVE);
 388
 389         uint8_t is_partner_fast, was_partner_fast;
 390         /* No periodic is on BEGIN, LACP DISABLE or when both sides are pasive */
 391         if (SM_FLAG(port, BEGIN) || !SM_FLAG(port, LACP_ENABLED) || !active) {
 392                 timer_cancel(&port->periodic_timer);
 393                 timer_force_expired(&port->tx_machine_timer);
 394                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 395
 396                 MODE4_DEBUG("-> NO_PERIODIC ( %s%s%s)\n",
 397                         SM_FLAG(port, BEGIN) ? "begind " : "",
 398                         SM_FLAG(port, LACP_ENABLED) ? "" : "LACP disabled ",
 399                         active ? "LACP active " : "LACP pasive ");
 400                 return;
 401         }
 402
 403         is_partner_fast = PARTNER_STATE(port, LACP_SHORT_TIMEOUT);
 404         was_partner_fast = SM_FLAG(port, PARTNER_SHORT_TIMEOUT);
 405
 406         /* If periodic timer is not started, transit from NO PERIODIC to FAST/SLOW.
 407          * Other case: check if timer expire or partners settings changed. */
 408         if (!timer_is_stopped(&port->periodic_timer)) {
 409                 if (timer_is_expired(&port->periodic_timer)) {
 410                         SM_FLAG_SET(port, NTT);
 411                 } else if (is_partner_fast != was_partner_fast) {
 412                         /* Partners timeout  was slow and now it is fast -> send LACP.
 413                          * In other case (was fast and now it is slow) just switch
 414                          * timeout to slow without forcing send of LACP (because standard
 415                          * say so)*/
 416                         if (is_partner_fast)
 417                                 SM_FLAG_SET(port, NTT);
 418                 } else
 419                         return; /* Nothing changed */
 420         }
 421
 422         /* Handle state transition to FAST/SLOW LACP timeout */
 423         if (is_partner_fast) {
 424                 timeout = internals->mode4.fast_periodic_timeout;
 425                 SM_FLAG_SET(port, PARTNER_SHORT_TIMEOUT);
 426         } else {
 427                 timeout = internals->mode4.slow_periodic_timeout;
 428                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 429         }
 430
 431         timer_set(&port->periodic_timer, timeout);
 432 }
 433
 434 /**
 435  * Function handles mux state machine.
 436  *
 437  * Function implements Mux Machine from point 5.4.15 in 802.1AX documentation.
 438  * It should be called periodically.
 439  *
 440  * @param port                  Port to handle state machine.
 441  */
 442 static void
 443 mux_machine(struct bond_dev_private *internals, uint16_t slave_id)
 444 {
 445         struct port *port = &bond_mode_8023ad_ports[slave_id];
 446
 447         /* Save current state for later use */
 448         const uint8_t state_mask = STATE_SYNCHRONIZATION | STATE_DISTRIBUTING |
 449                 STATE_COLLECTING;
 450
 451         /* Enter DETACHED state on BEGIN condition or from any other state if
 452          * port was unselected */
 453         if (SM_FLAG(port, BEGIN) ||
 454                         port->selected == UNSELECTED || (port->selected == STANDBY &&
 455                                 (port->actor_state & state_mask) != 0)) {
 456                 /* detach mux from aggregator */
 457                 port->actor_state &= ~state_mask;
 458                 /* Set ntt to true if BEGIN condition or transition from any other state
 459                  * which is indicated that wait_while_timer was started */
 460                 if (SM_FLAG(port, BEGIN) ||
 461                                 !timer_is_stopped(&port->wait_while_timer)) {
 462                         SM_FLAG_SET(port, NTT);
 463                         MODE4_DEBUG("-> DETACHED\n");
 464                 }
 465                 timer_cancel(&port->wait_while_timer);
 466         }
 467
 468         if (timer_is_stopped(&port->wait_while_timer)) {
 469                 if (port->selected == SELECTED || port->selected == STANDBY) {
 470                         timer_set(&port->wait_while_timer,
 471                                 internals->mode4.aggregate_wait_timeout);
 472
 473                         MODE4_DEBUG("DETACHED -> WAITING\n");
 474                 }
 475                 /* Waiting state entered */
 476                 return;
 477         }
 478
 479         /* Transit next state if port is ready */
 480         if (!timer_is_expired(&port->wait_while_timer))
 481                 return;
 482
 483         if ((ACTOR_STATE(port, DISTRIBUTING) || ACTOR_STATE(port, COLLECTING)) &&
 484                 !PARTNER_STATE(port, SYNCHRONIZATION)) {
 485                 /* If in COLLECTING or DISTRIBUTING state and partner becomes out of
 486                  * sync transit to ATACHED state.  */
 487                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 488                 ACTOR_STATE_CLR(port, COLLECTING);
 489                 /* Clear actor sync to activate transit ATACHED in condition bellow */
 490                 ACTOR_STATE_CLR(port, SYNCHRONIZATION);
 491                 MODE4_DEBUG("Out of sync -> ATTACHED\n");
 492         }
 493
 494         if (!ACTOR_STATE(port, SYNCHRONIZATION)) {
 495                 /* attach mux to aggregator */
 496                 RTE_ASSERT((port->actor_state & (STATE_COLLECTING |
 497                         STATE_DISTRIBUTING)) == 0);
 498
 499                 ACTOR_STATE_SET(port, SYNCHRONIZATION);
 500                 SM_FLAG_SET(port, NTT);
 501                 MODE4_DEBUG("ATTACHED Entered\n");
 502         } else if (!ACTOR_STATE(port, COLLECTING)) {
 503                 /* Start collecting if in sync */
 504                 if (PARTNER_STATE(port, SYNCHRONIZATION)) {
 505                         MODE4_DEBUG("ATTACHED -> COLLECTING\n");
 506                         ACTOR_STATE_SET(port, COLLECTING);
 507                         SM_FLAG_SET(port, NTT);
 508                 }
 509         } else if (ACTOR_STATE(port, COLLECTING)) {
 510                 /* Check if partner is in COLLECTING state. If so this port can
 511                  * distribute frames to it */
 512                 if (!ACTOR_STATE(port, DISTRIBUTING)) {
 513                         if (PARTNER_STATE(port, COLLECTING)) {
 514                                 /* Enable  DISTRIBUTING if partner is collecting */
 515                                 ACTOR_STATE_SET(port, DISTRIBUTING);
 516                                 SM_FLAG_SET(port, NTT);
 517                                 MODE4_DEBUG("COLLECTING -> DISTRIBUTING\n");
 518                                 RTE_BOND_LOG(INFO,
 519                                         "Bond %u: slave id %u distributing started.",
 520                                         internals->port_id, slave_id);
 521                         }
 522                 } else {
 523                         if (!PARTNER_STATE(port, COLLECTING)) {
 524                                 /* Disable DISTRIBUTING (enter COLLECTING state) if partner
 525                                  * is not collecting */
 526                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 527                                 SM_FLAG_SET(port, NTT);
 528                                 MODE4_DEBUG("DISTRIBUTING -> COLLECTING\n");
 529                                 RTE_BOND_LOG(INFO,
 530                                         "Bond %u: slave id %u distributing stopped.",
 531                                         internals->port_id, slave_id);
 532                         }
 533                 }
 534         }
 535 }
 536
 537 /**
 538  * Function handles transmit state machine.
 539  *
 540  * Function implements Transmit Machine from point 5.4.16 in 802.1AX
 541  * documentation.
 542  *
 543  * @param port
 544  */
 545 static void
 546 tx_machine(struct bond_dev_private *internals, uint16_t slave_id)
 547 {
 548         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 549
 550         struct rte_mbuf *lacp_pkt = NULL;
 551         struct lacpdu_header *hdr;
 552         struct lacpdu *lacpdu;
 553
 554         /* If periodic timer is not running periodic machine is in NO PERIODIC and
 555          * according to 802.3ax standard tx machine should not transmit any frames
 556          * and set ntt to false. */
 557         if (timer_is_stopped(&port->periodic_timer))
 558                 SM_FLAG_CLR(port, NTT);
 559
 560         if (!SM_FLAG(port, NTT))
 561                 return;
 562
 563         if (!timer_is_expired(&port->tx_machine_timer))
 564                 return;
 565
 566         lacp_pkt = rte_pktmbuf_alloc(port->mbuf_pool);
 567         if (lacp_pkt == NULL) {
 568                 RTE_BOND_LOG(ERR, "Failed to allocate LACP packet from pool");
 569                 return;
 570         }
 571
 572         lacp_pkt->data_len = sizeof(*hdr);
 573         lacp_pkt->pkt_len = sizeof(*hdr);
 574
 575         hdr = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 576
 577         /* Source and destination MAC */
 578         rte_ether_addr_copy(&lacp_mac_addr, &hdr->eth_hdr.d_addr);
 579         rte_eth_macaddr_get(slave_id, &hdr->eth_hdr.s_addr);
 580         hdr->eth_hdr.ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_SLOW);
 581
 582         lacpdu = &hdr->lacpdu;
 583         memset(lacpdu, 0, sizeof(*lacpdu));
 584
 585         /* Initialize LACP part */
 586         lacpdu->subtype = SLOW_SUBTYPE_LACP;
 587         lacpdu->version_number = 1;
 588
 589         /* ACTOR */
 590         lacpdu->actor.tlv_type_info = TLV_TYPE_ACTOR_INFORMATION;
 591         lacpdu->actor.info_length = sizeof(struct lacpdu_actor_partner_params);
 592         memcpy(&hdr->lacpdu.actor.port_params, &port->actor,
 593                         sizeof(port->actor));
 594         agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 595         rte_ether_addr_copy(&agg->actor.system,
 596                         &hdr->lacpdu.actor.port_params.system);
 597         lacpdu->actor.state = port->actor_state;
 598
 599         /* PARTNER */
 600         lacpdu->partner.tlv_type_info = TLV_TYPE_PARTNER_INFORMATION;
 601         lacpdu->partner.info_length = sizeof(struct lacpdu_actor_partner_params);
 602         memcpy(&lacpdu->partner.port_params, &port->partner,
 603                         sizeof(struct port_params));
 604         lacpdu->partner.state = port->partner_state;
 605
 606         /* Other fields */
 607         lacpdu->tlv_type_collector_info = TLV_TYPE_COLLECTOR_INFORMATION;
 608         lacpdu->collector_info_length = 0x10;
 609         lacpdu->collector_max_delay = 0;
 610
 611         lacpdu->tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION;
 612         lacpdu->terminator_length = 0;
 613
 614         MODE4_DEBUG("Sending LACP frame\n");
 615         BOND_PRINT_LACP(lacpdu);
 616
 617         if (internals->mode4.dedicated_queues.enabled == 0) {
 618                 int retval = rte_ring_enqueue(port->tx_ring, lacp_pkt);
 619                 if (retval != 0) {
 620                         /* If TX ring full, drop packet and free message.
 621                            Retransmission will happen in next function call. */
 622                         rte_pktmbuf_free(lacp_pkt);
 623                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 624                         return;
 625                 }
 626         } else {
 627                 uint16_t pkts_sent = rte_eth_tx_burst(slave_id,
 628                                 internals->mode4.dedicated_queues.tx_qid,
 629                                 &lacp_pkt, 1);
 630                 if (pkts_sent != 1) {
 631                         rte_pktmbuf_free(lacp_pkt);
 632                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 633                         return;
 634                 }
 635         }
 636
 637
 638         timer_set(&port->tx_machine_timer, internals->mode4.tx_period_timeout);
 639         SM_FLAG_CLR(port, NTT);
 640 }
 641
 642 static uint8_t
 643 max_index(uint64_t *a, int n)
 644 {
 645         if (n <= 0)
 646                 return -1;
 647
 648         int i, max_i = 0;
 649         uint64_t max = a[0];
 650
 651         for (i = 1; i < n; ++i) {
 652                 if (a[i] > max) {
 653                         max = a[i];
 654                         max_i = i;
 655                 }
 656         }
 657
 658         return max_i;
 659 }
 660
 661 /**
 662  * Function assigns port to aggregator.
 663  *
 664  * @param bond_dev_private      Pointer to bond_dev_private structure.
 665  * @param port_pos                      Port to assign.
 666  */
 667 static void
 668 selection_logic(struct bond_dev_private *internals, uint16_t slave_id)
 669 {
 670         struct port *agg, *port;
 671         uint16_t slaves_count, new_agg_id, i, j = 0;
 672         uint16_t *slaves;
 673         uint64_t agg_bandwidth[8] = {0};
 674         uint64_t agg_count[8] = {0};
 675         uint16_t default_slave = 0;
 676         uint8_t mode_count_id, mode_band_id;
 677         struct rte_eth_link link_info;
 678         int ret;
 679
 680         slaves = internals->active_slaves;
 681         slaves_count = internals->active_slave_count;
 682         port = &bond_mode_8023ad_ports[slave_id];
 683
 684         /* Search for aggregator suitable for this port */
 685         for (i = 0; i < slaves_count; ++i) {
 686                 agg = &bond_mode_8023ad_ports[slaves[i]];
 687                 /* Skip ports that are not aggreagators */
 688                 if (agg->aggregator_port_id != slaves[i])
 689                         continue;
 690
 691                 ret = rte_eth_link_get_nowait(slaves[i], &link_info);
 692                 if (ret < 0) {
 693                         RTE_BOND_LOG(ERR,
 694                                 "Slave (port %u) link get failed: %s\n",
 695                                 slaves[i], rte_strerror(-ret));
 696                         continue;
 697                 }
 698                 agg_count[agg->aggregator_port_id] += 1;
 699                 agg_bandwidth[agg->aggregator_port_id] += link_info.link_speed;
 700
 701                 /* Actors system ID is not checked since all slave device have the same
 702                  * ID (MAC address). */
 703                 if ((agg->actor.key == port->actor.key &&
 704                         agg->partner.system_priority == port->partner.system_priority &&
 705                         rte_is_same_ether_addr(&agg->partner.system,
 706                                         &port->partner.system) == 1
 707                         && (agg->partner.key == port->partner.key)) &&
 708                         rte_is_zero_ether_addr(&port->partner.system) != 1 &&
 709                         (agg->actor.key &
 710                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) != 0) {
 711
 712                         if (j == 0)
 713                                 default_slave = i;
 714                         j++;
 715                 }
 716         }
 717
 718         switch (internals->mode4.agg_selection) {
 719         case AGG_COUNT:
 720                 mode_count_id = max_index(
 721                                 (uint64_t *)agg_count, slaves_count);
 722                 new_agg_id = mode_count_id;
 723                 break;
 724         case AGG_BANDWIDTH:
 725                 mode_band_id = max_index(
 726                                 (uint64_t *)agg_bandwidth, slaves_count);
 727                 new_agg_id = mode_band_id;
 728                 break;
 729         case AGG_STABLE:
 730                 if (default_slave == slaves_count)
 731                         new_agg_id = slave_id;
 732                 else
 733                         new_agg_id = slaves[default_slave];
 734                 break;
 735         default:
 736                 if (default_slave == slaves_count)
 737                         new_agg_id = slave_id;
 738                 else
 739                         new_agg_id = slaves[default_slave];
 740                 break;
 741         }
 742
 743         if (new_agg_id != port->aggregator_port_id) {
 744                 port->aggregator_port_id = new_agg_id;
 745
 746                 MODE4_DEBUG("-> SELECTED: ID=%3u\n"
 747                         "\t%s aggregator ID=%3u\n",
 748                         port->aggregator_port_id,
 749                         port->aggregator_port_id == slave_id ?
 750                                 "aggregator not found, using default" : "aggregator found",
 751                         port->aggregator_port_id);
 752         }
 753
 754         port->selected = SELECTED;
 755 }
 756
 757 /* Function maps DPDK speed to bonding speed stored in key field */
 758 static uint16_t
 759 link_speed_key(uint16_t speed) {
 760         uint16_t key_speed;
 761
 762         switch (speed) {
 763         case ETH_SPEED_NUM_NONE:
 764                 key_speed = 0x00;
 765                 break;
 766         case ETH_SPEED_NUM_10M:
 767                 key_speed = BOND_LINK_SPEED_KEY_10M;
 768                 break;
 769         case ETH_SPEED_NUM_100M:
 770                 key_speed = BOND_LINK_SPEED_KEY_100M;
 771                 break;
 772         case ETH_SPEED_NUM_1G:
 773                 key_speed = BOND_LINK_SPEED_KEY_1000M;
 774                 break;
 775         case ETH_SPEED_NUM_10G:
 776                 key_speed = BOND_LINK_SPEED_KEY_10G;
 777                 break;
 778         case ETH_SPEED_NUM_20G:
 779                 key_speed = BOND_LINK_SPEED_KEY_20G;
 780                 break;
 781         case ETH_SPEED_NUM_40G:
 782                 key_speed = BOND_LINK_SPEED_KEY_40G;
 783                 break;
 784         default:
 785                 /* Unknown speed*/
 786                 key_speed = 0xFFFF;
 787         }
 788
 789         return key_speed;
 790 }
 791
 792 static void
 793 rx_machine_update(struct bond_dev_private *internals, uint16_t slave_id,
 794                 struct rte_mbuf *lacp_pkt) {
 795         struct lacpdu_header *lacp;
 796         struct lacpdu_actor_partner_params *partner;
 797
 798         if (lacp_pkt != NULL) {
 799                 lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 800                 RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
 801
 802                 partner = &lacp->lacpdu.partner;
 803                 if (rte_is_same_ether_addr(&partner->port_params.system,
 804                         &internals->mode4.mac_addr)) {
 805                         /* This LACP frame is sending to the bonding port
 806                          * so pass it to rx_machine.
 807                          */
 808                         rx_machine(internals, slave_id, &lacp->lacpdu);
 809                 }
 810                 rte_pktmbuf_free(lacp_pkt);
 811         } else
 812                 rx_machine(internals, slave_id, NULL);
 813 }
 814
 815 static void
 816 bond_mode_8023ad_periodic_cb(void *arg)
 817 {
 818         struct rte_eth_dev *bond_dev = arg;
 819         struct bond_dev_private *internals = bond_dev->data->dev_private;
 820         struct port *port;
 821         struct rte_eth_link link_info;
 822         struct rte_ether_addr slave_addr;
 823         struct rte_mbuf *lacp_pkt = NULL;
 824         uint16_t slave_id;
 825         uint16_t i;
 826
 827
 828         /* Update link status on each port */
 829         for (i = 0; i < internals->active_slave_count; i++) {
 830                 uint16_t key;
 831                 int ret;
 832
 833                 slave_id = internals->active_slaves[i];
 834                 ret = rte_eth_link_get_nowait(slave_id, &link_info);
 835                 if (ret < 0) {
 836                         RTE_BOND_LOG(ERR,
 837                                 "Slave (port %u) link get failed: %s\n",
 838                                 slave_id, rte_strerror(-ret));
 839                 }
 840
 841                 if (ret >= 0 && link_info.link_status != 0) {
 842                         key = link_speed_key(link_info.link_speed) << 1;
 843                         if (link_info.link_duplex == ETH_LINK_FULL_DUPLEX)
 844                                 key |= BOND_LINK_FULL_DUPLEX_KEY;
 845                 } else {
 846                         key = 0;
 847                 }
 848
 849                 rte_eth_macaddr_get(slave_id, &slave_addr);
 850                 port = &bond_mode_8023ad_ports[slave_id];
 851
 852                 key = rte_cpu_to_be_16(key);
 853                 if (key != port->actor.key) {
 854                         if (!(key & rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)))
 855                                 set_warning_flags(port, WRN_NOT_LACP_CAPABLE);
 856
 857                         port->actor.key = key;
 858                         SM_FLAG_SET(port, NTT);
 859                 }
 860
 861                 if (!rte_is_same_ether_addr(&port->actor.system, &slave_addr)) {
 862                         rte_ether_addr_copy(&slave_addr, &port->actor.system);
 863                         if (port->aggregator_port_id == slave_id)
 864                                 SM_FLAG_SET(port, NTT);
 865                 }
 866         }
 867
 868         for (i = 0; i < internals->active_slave_count; i++) {
 869                 slave_id = internals->active_slaves[i];
 870                 port = &bond_mode_8023ad_ports[slave_id];
 871
 872                 if ((port->actor.key &
 873                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) == 0) {
 874
 875                         SM_FLAG_SET(port, BEGIN);
 876
 877                         /* LACP is disabled on half duples or link is down */
 878                         if (SM_FLAG(port, LACP_ENABLED)) {
 879                                 /* If port was enabled set it to BEGIN state */
 880                                 SM_FLAG_CLR(port, LACP_ENABLED);
 881                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 882                                 ACTOR_STATE_CLR(port, COLLECTING);
 883                         }
 884
 885                         /* Skip this port processing */
 886                         continue;
 887                 }
 888
 889                 SM_FLAG_SET(port, LACP_ENABLED);
 890
 891                 if (internals->mode4.dedicated_queues.enabled == 0) {
 892                         /* Find LACP packet to this port. Do not check subtype,
 893                          * it is done in function that queued packet
 894                          */
 895                         int retval = rte_ring_dequeue(port->rx_ring,
 896                                         (void **)&lacp_pkt);
 897
 898                         if (retval != 0)
 899                                 lacp_pkt = NULL;
 900
 901                         rx_machine_update(internals, slave_id, lacp_pkt);
 902                 } else {
 903                         uint16_t rx_count = rte_eth_rx_burst(slave_id,
 904                                         internals->mode4.dedicated_queues.rx_qid,
 905                                         &lacp_pkt, 1);
 906
 907                         if (rx_count == 1)
 908                                 bond_mode_8023ad_handle_slow_pkt(internals,
 909                                                 slave_id, lacp_pkt);
 910                         else
 911                                 rx_machine_update(internals, slave_id, NULL);
 912                 }
 913
 914                 periodic_machine(internals, slave_id);
 915                 mux_machine(internals, slave_id);
 916                 tx_machine(internals, slave_id);
 917                 selection_logic(internals, slave_id);
 918
 919                 SM_FLAG_CLR(port, BEGIN);
 920                 show_warnings(slave_id);
 921         }
 922
 923         rte_eal_alarm_set(internals->mode4.update_timeout_us,
 924                         bond_mode_8023ad_periodic_cb, arg);
 925 }
 926
 927 static int
 928 bond_mode_8023ad_register_lacp_mac(uint16_t slave_id)
 929 {
 930         int ret;
 931
 932         ret = rte_eth_allmulticast_enable(slave_id);
 933         if (ret != 0) {
 934                 RTE_BOND_LOG(ERR,
 935                         "failed to enable allmulti mode for port %u: %s",
 936                         slave_id, rte_strerror(-ret));
 937         }
 938         if (rte_eth_allmulticast_get(slave_id)) {
 939                 RTE_BOND_LOG(DEBUG, "forced allmulti for port %u",
 940                              slave_id);
 941                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 942                                 BOND_8023AD_FORCED_ALLMULTI;
 943                 return 0;
 944         }
 945
 946         ret = rte_eth_promiscuous_enable(slave_id);
 947         if (ret != 0) {
 948                 RTE_BOND_LOG(ERR,
 949                         "failed to enable promiscuous mode for port %u: %s",
 950                         slave_id, rte_strerror(-ret));
 951         }
 952         if (rte_eth_promiscuous_get(slave_id)) {
 953                 RTE_BOND_LOG(DEBUG, "forced promiscuous for port %u",
 954                              slave_id);
 955                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 956                                 BOND_8023AD_FORCED_PROMISC;
 957                 return 0;
 958         }
 959
 960         return -1;
 961 }
 962
 963 static void
 964 bond_mode_8023ad_unregister_lacp_mac(uint16_t slave_id)
 965 {
 966         int ret;
 967
 968         switch (bond_mode_8023ad_ports[slave_id].forced_rx_flags) {
 969         case BOND_8023AD_FORCED_ALLMULTI:
 970                 RTE_BOND_LOG(DEBUG, "unset allmulti for port %u", slave_id);
 971                 ret = rte_eth_allmulticast_disable(slave_id);
 972                 if (ret != 0)
 973                         RTE_BOND_LOG(ERR,
 974                                 "failed to disable allmulti mode for port %u: %s",
 975                                 slave_id, rte_strerror(-ret));
 976                 break;
 977
 978         case BOND_8023AD_FORCED_PROMISC:
 979                 RTE_BOND_LOG(DEBUG, "unset promisc for port %u", slave_id);
 980                 ret = rte_eth_promiscuous_disable(slave_id);
 981                 if (ret != 0)
 982                         RTE_BOND_LOG(ERR,
 983                                 "failed to disable promiscuous mode for port %u: %s",
 984                                 slave_id, rte_strerror(-ret));
 985                 break;
 986
 987         default:
 988                 break;
 989         }
 990 }
 991
 992 void
 993 bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev,
 994                                 uint16_t slave_id)
 995 {
 996         struct bond_dev_private *internals = bond_dev->data->dev_private;
 997
 998         struct port *port = &bond_mode_8023ad_ports[slave_id];
 999         struct port_params initial = {
1000                         .system = { { 0 } },
1001                         .system_priority = rte_cpu_to_be_16(0xFFFF),
1002                         .key = rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY),
1003                         .port_priority = rte_cpu_to_be_16(0x00FF),
1004                         .port_number = 0,
1005         };
1006
1007         char mem_name[RTE_ETH_NAME_MAX_LEN];
1008         int socket_id;
1009         unsigned element_size;
1010         uint32_t total_tx_desc;
1011         struct bond_tx_queue *bd_tx_q;
1012         uint16_t q_id;
1013
1014         /* Given slave mus not be in active list */
1015         RTE_ASSERT(find_slave_by_id(internals->active_slaves,
1016         internals->active_slave_count, slave_id) == internals->active_slave_count);
1017         RTE_SET_USED(internals); /* used only for assert when enabled */
1018
1019         memcpy(&port->actor, &initial, sizeof(struct port_params));
1020         /* Standard requires that port ID must be grater than 0.
1021          * Add 1 do get corresponding port_number */
1022         port->actor.port_number = rte_cpu_to_be_16(slave_id + 1);
1023
1024         memcpy(&port->partner, &initial, sizeof(struct port_params));
1025
1026         /* default states */
1027         port->actor_state = STATE_AGGREGATION | STATE_LACP_ACTIVE | STATE_DEFAULTED;
1028         port->partner_state = STATE_LACP_ACTIVE | STATE_AGGREGATION;
1029         port->sm_flags = SM_FLAGS_BEGIN;
1030
1031         /* use this port as agregator */
1032         port->aggregator_port_id = slave_id;
1033
1034         if (bond_mode_8023ad_register_lacp_mac(slave_id) < 0) {
1035                 RTE_BOND_LOG(WARNING, "slave %u is most likely broken and won't receive LACP packets",
1036                              slave_id);
1037         }
1038
1039         timer_cancel(&port->warning_timer);
1040
1041         if (port->mbuf_pool != NULL)
1042                 return;
1043
1044         RTE_ASSERT(port->rx_ring == NULL);
1045         RTE_ASSERT(port->tx_ring == NULL);
1046
1047         socket_id = rte_eth_dev_socket_id(slave_id);
1048         if (socket_id == (int)LCORE_ID_ANY)
1049                 socket_id = rte_socket_id();
1050
1051         element_size = sizeof(struct slow_protocol_frame) +
1052                                 RTE_PKTMBUF_HEADROOM;
1053
1054         /* The size of the mempool should be at least:
1055          * the sum of the TX descriptors + BOND_MODE_8023AX_SLAVE_TX_PKTS */
1056         total_tx_desc = BOND_MODE_8023AX_SLAVE_TX_PKTS;
1057         for (q_id = 0; q_id < bond_dev->data->nb_tx_queues; q_id++) {
1058                 bd_tx_q = (struct bond_tx_queue*)bond_dev->data->tx_queues[q_id];
1059                 total_tx_desc += bd_tx_q->nb_tx_desc;
1060         }
1061
1062         snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_pool", slave_id);
1063         port->mbuf_pool = rte_pktmbuf_pool_create(mem_name, total_tx_desc,
1064                 RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ?
1065                         32 : RTE_MEMPOOL_CACHE_MAX_SIZE,
1066                 0, element_size, socket_id);
1067
1068         /* Any memory allocation failure in initialization is critical because
1069          * resources can't be free, so reinitialization is impossible. */
1070         if (port->mbuf_pool == NULL) {
1071                 rte_panic("Slave %u: Failed to create memory pool '%s': %s\n",
1072                         slave_id, mem_name, rte_strerror(rte_errno));
1073         }
1074
1075         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_rx", slave_id);
1076         port->rx_ring = rte_ring_create(mem_name,
1077                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_RX_PKTS), socket_id, 0);
1078
1079         if (port->rx_ring == NULL) {
1080                 rte_panic("Slave %u: Failed to create rx ring '%s': %s\n", slave_id,
1081                         mem_name, rte_strerror(rte_errno));
1082         }
1083
1084         /* TX ring is at least one pkt longer to make room for marker packet. */
1085         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_tx", slave_id);
1086         port->tx_ring = rte_ring_create(mem_name,
1087                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_TX_PKTS + 1), socket_id, 0);
1088
1089         if (port->tx_ring == NULL) {
1090                 rte_panic("Slave %u: Failed to create tx ring '%s': %s\n", slave_id,
1091                         mem_name, rte_strerror(rte_errno));
1092         }
1093 }
1094
1095 int
1096 bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev __rte_unused,
1097                 uint16_t slave_id)
1098 {
1099         void *pkt = NULL;
1100         struct port *port = NULL;
1101         uint8_t old_partner_state;
1102
1103         port = &bond_mode_8023ad_ports[slave_id];
1104
1105         ACTOR_STATE_CLR(port, AGGREGATION);
1106         port->selected = UNSELECTED;
1107
1108         old_partner_state = port->partner_state;
1109         record_default(port);
1110
1111         bond_mode_8023ad_unregister_lacp_mac(slave_id);
1112
1113         /* If partner timeout state changes then disable timer */
1114         if (!((old_partner_state ^ port->partner_state) &
1115                         STATE_LACP_SHORT_TIMEOUT))
1116                 timer_cancel(&port->current_while_timer);
1117
1118         PARTNER_STATE_CLR(port, AGGREGATION);
1119         ACTOR_STATE_CLR(port, EXPIRED);
1120
1121         /* flush rx/tx rings */
1122         while (rte_ring_dequeue(port->rx_ring, &pkt) == 0)
1123                 rte_pktmbuf_free((struct rte_mbuf *)pkt);
1124
1125         while (rte_ring_dequeue(port->tx_ring, &pkt) == 0)
1126                         rte_pktmbuf_free((struct rte_mbuf *)pkt);
1127         return 0;
1128 }
1129
1130 void
1131 bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev)
1132 {
1133         struct bond_dev_private *internals = bond_dev->data->dev_private;
1134         struct rte_ether_addr slave_addr;
1135         struct port *slave, *agg_slave;
1136         uint16_t slave_id, i, j;
1137
1138         bond_mode_8023ad_stop(bond_dev);
1139
1140         for (i = 0; i < internals->active_slave_count; i++) {
1141                 slave_id = internals->active_slaves[i];
1142                 slave = &bond_mode_8023ad_ports[slave_id];
1143                 rte_eth_macaddr_get(slave_id, &slave_addr);
1144
1145                 if (rte_is_same_ether_addr(&slave_addr, &slave->actor.system))
1146                         continue;
1147
1148                 rte_ether_addr_copy(&slave_addr, &slave->actor.system);
1149                 /* Do nothing if this port is not an aggregator. In other case
1150                  * Set NTT flag on every port that use this aggregator. */
1151                 if (slave->aggregator_port_id != slave_id)
1152                         continue;
1153
1154                 for (j = 0; j < internals->active_slave_count; j++) {
1155                         agg_slave = &bond_mode_8023ad_ports[internals->active_slaves[j]];
1156                         if (agg_slave->aggregator_port_id == slave_id)
1157                                 SM_FLAG_SET(agg_slave, NTT);
1158                 }
1159         }
1160
1161         if (bond_dev->data->dev_started)
1162                 bond_mode_8023ad_start(bond_dev);
1163 }
1164
1165 static void
1166 bond_mode_8023ad_conf_get(struct rte_eth_dev *dev,
1167                 struct rte_eth_bond_8023ad_conf *conf)
1168 {
1169         struct bond_dev_private *internals = dev->data->dev_private;
1170         struct mode8023ad_private *mode4 = &internals->mode4;
1171         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1172
1173         conf->fast_periodic_ms = mode4->fast_periodic_timeout / ms_ticks;
1174         conf->slow_periodic_ms = mode4->slow_periodic_timeout / ms_ticks;
1175         conf->short_timeout_ms = mode4->short_timeout / ms_ticks;
1176         conf->long_timeout_ms = mode4->long_timeout / ms_ticks;
1177         conf->aggregate_wait_timeout_ms = mode4->aggregate_wait_timeout / ms_ticks;
1178         conf->tx_period_ms = mode4->tx_period_timeout / ms_ticks;
1179         conf->update_timeout_ms = mode4->update_timeout_us / 1000;
1180         conf->rx_marker_period_ms = mode4->rx_marker_timeout / ms_ticks;
1181         conf->slowrx_cb = mode4->slowrx_cb;
1182         conf->agg_selection = mode4->agg_selection;
1183 }
1184
1185 static void
1186 bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf)
1187 {
1188         conf->fast_periodic_ms = BOND_8023AD_FAST_PERIODIC_MS;
1189         conf->slow_periodic_ms = BOND_8023AD_SLOW_PERIODIC_MS;
1190         conf->short_timeout_ms = BOND_8023AD_SHORT_TIMEOUT_MS;
1191         conf->long_timeout_ms = BOND_8023AD_LONG_TIMEOUT_MS;
1192         conf->aggregate_wait_timeout_ms = BOND_8023AD_AGGREGATE_WAIT_TIMEOUT_MS;
1193         conf->tx_period_ms = BOND_8023AD_TX_MACHINE_PERIOD_MS;
1194         conf->rx_marker_period_ms = BOND_8023AD_RX_MARKER_PERIOD_MS;
1195         conf->update_timeout_ms = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS;
1196         conf->slowrx_cb = NULL;
1197         conf->agg_selection = AGG_STABLE;
1198 }
1199
1200 static void
1201 bond_mode_8023ad_conf_assign(struct mode8023ad_private *mode4,
1202                 struct rte_eth_bond_8023ad_conf *conf)
1203 {
1204         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1205
1206         mode4->fast_periodic_timeout = conf->fast_periodic_ms * ms_ticks;
1207         mode4->slow_periodic_timeout = conf->slow_periodic_ms * ms_ticks;
1208         mode4->short_timeout = conf->short_timeout_ms * ms_ticks;
1209         mode4->long_timeout = conf->long_timeout_ms * ms_ticks;
1210         mode4->aggregate_wait_timeout = conf->aggregate_wait_timeout_ms * ms_ticks;
1211         mode4->tx_period_timeout = conf->tx_period_ms * ms_ticks;
1212         mode4->rx_marker_timeout = conf->rx_marker_period_ms * ms_ticks;
1213         mode4->update_timeout_us = conf->update_timeout_ms * 1000;
1214
1215         mode4->dedicated_queues.enabled = 0;
1216         mode4->dedicated_queues.rx_qid = UINT16_MAX;
1217         mode4->dedicated_queues.tx_qid = UINT16_MAX;
1218 }
1219
1220 void
1221 bond_mode_8023ad_setup(struct rte_eth_dev *dev,
1222                 struct rte_eth_bond_8023ad_conf *conf)
1223 {
1224         struct rte_eth_bond_8023ad_conf def_conf;
1225         struct bond_dev_private *internals = dev->data->dev_private;
1226         struct mode8023ad_private *mode4 = &internals->mode4;
1227
1228         if (conf == NULL) {
1229                 conf = &def_conf;
1230                 bond_mode_8023ad_conf_get_default(conf);
1231         }
1232
1233         bond_mode_8023ad_stop(dev);
1234         bond_mode_8023ad_conf_assign(mode4, conf);
1235         mode4->slowrx_cb = conf->slowrx_cb;
1236         mode4->agg_selection = AGG_STABLE;
1237
1238         if (dev->data->dev_started)
1239                 bond_mode_8023ad_start(dev);
1240 }
1241
1242 int
1243 bond_mode_8023ad_enable(struct rte_eth_dev *bond_dev)
1244 {
1245         struct bond_dev_private *internals = bond_dev->data->dev_private;
1246         uint16_t i;
1247
1248         for (i = 0; i < internals->active_slave_count; i++)
1249                 bond_mode_8023ad_activate_slave(bond_dev,
1250                                 internals->active_slaves[i]);
1251
1252         return 0;
1253 }
1254
1255 int
1256 bond_mode_8023ad_start(struct rte_eth_dev *bond_dev)
1257 {
1258         struct bond_dev_private *internals = bond_dev->data->dev_private;
1259         struct mode8023ad_private *mode4 = &internals->mode4;
1260         static const uint64_t us = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS * 1000;
1261
1262         rte_eth_macaddr_get(internals->port_id, &mode4->mac_addr);
1263         if (mode4->slowrx_cb)
1264                 return rte_eal_alarm_set(us, &bond_mode_8023ad_ext_periodic_cb,
1265                                          bond_dev);
1266
1267         return rte_eal_alarm_set(us, &bond_mode_8023ad_periodic_cb, bond_dev);
1268 }
1269
1270 void
1271 bond_mode_8023ad_stop(struct rte_eth_dev *bond_dev)
1272 {
1273         struct bond_dev_private *internals = bond_dev->data->dev_private;
1274         struct mode8023ad_private *mode4 = &internals->mode4;
1275
1276         if (mode4->slowrx_cb) {
1277                 rte_eal_alarm_cancel(&bond_mode_8023ad_ext_periodic_cb,
1278                                      bond_dev);
1279                 return;
1280         }
1281         rte_eal_alarm_cancel(&bond_mode_8023ad_periodic_cb, bond_dev);
1282 }
1283
1284 void
1285 bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
1286                                   uint16_t slave_id, struct rte_mbuf *pkt)
1287 {
1288         struct mode8023ad_private *mode4 = &internals->mode4;
1289         struct port *port = &bond_mode_8023ad_ports[slave_id];
1290         struct marker_header *m_hdr;
1291         uint64_t marker_timer, old_marker_timer;
1292         int retval;
1293         uint8_t wrn, subtype;
1294         /* If packet is a marker, we send response now by reusing given packet
1295          * and update only source MAC, destination MAC is multicast so don't
1296          * update it. Other frames will be handled later by state machines */
1297         subtype = rte_pktmbuf_mtod(pkt,
1298                         struct slow_protocol_frame *)->slow_protocol.subtype;
1299
1300         if (subtype == SLOW_SUBTYPE_MARKER) {
1301                 m_hdr = rte_pktmbuf_mtod(pkt, struct marker_header *);
1302
1303                 if (likely(m_hdr->marker.tlv_type_marker != MARKER_TLV_TYPE_INFO)) {
1304                         wrn = WRN_UNKNOWN_MARKER_TYPE;
1305                         goto free_out;
1306                 }
1307
1308                 /* Setup marker timer. Do it in loop in case concurrent access. */
1309                 do {
1310                         old_marker_timer = port->rx_marker_timer;
1311                         if (!timer_is_expired(&old_marker_timer)) {
1312                                 wrn = WRN_RX_MARKER_TO_FAST;
1313                                 goto free_out;
1314                         }
1315
1316                         timer_set(&marker_timer, mode4->rx_marker_timeout);
1317                         retval = rte_atomic64_cmpset(&port->rx_marker_timer,
1318                                 old_marker_timer, marker_timer);
1319                 } while (unlikely(retval == 0));
1320
1321                 m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP;
1322                 rte_eth_macaddr_get(slave_id, &m_hdr->eth_hdr.s_addr);
1323
1324                 if (internals->mode4.dedicated_queues.enabled == 0) {
1325                         int retval = rte_ring_enqueue(port->tx_ring, pkt);
1326                         if (retval != 0) {
1327                                 /* reset timer */
1328                                 port->rx_marker_timer = 0;
1329                                 wrn = WRN_TX_QUEUE_FULL;
1330                                 goto free_out;
1331                         }
1332                 } else {
1333                         /* Send packet directly to the slow queue */
1334                         uint16_t tx_count = rte_eth_tx_burst(slave_id,
1335                                         internals->mode4.dedicated_queues.tx_qid,
1336                                         &pkt, 1);
1337                         if (tx_count != 1) {
1338                                 /* reset timer */
1339                                 port->rx_marker_timer = 0;
1340                                 wrn = WRN_TX_QUEUE_FULL;
1341                                 goto free_out;
1342                         }
1343                 }
1344         } else if (likely(subtype == SLOW_SUBTYPE_LACP)) {
1345                 if (internals->mode4.dedicated_queues.enabled == 0) {
1346                         int retval = rte_ring_enqueue(port->rx_ring, pkt);
1347                         if (retval != 0) {
1348                                 /* If RX fing full free lacpdu message and drop packet */
1349                                 wrn = WRN_RX_QUEUE_FULL;
1350                                 goto free_out;
1351                         }
1352                 } else
1353                         rx_machine_update(internals, slave_id, pkt);
1354         } else {
1355                 wrn = WRN_UNKNOWN_SLOW_TYPE;
1356                 goto free_out;
1357         }
1358
1359         return;
1360
1361 free_out:
1362         set_warning_flags(port, wrn);
1363         rte_pktmbuf_free(pkt);
1364 }
1365
1366 int
1367 rte_eth_bond_8023ad_conf_get(uint16_t port_id,
1368                 struct rte_eth_bond_8023ad_conf *conf)
1369 {
1370         struct rte_eth_dev *bond_dev;
1371
1372         if (valid_bonded_port_id(port_id) != 0)
1373                 return -EINVAL;
1374
1375         if (conf == NULL)
1376                 return -EINVAL;
1377
1378         bond_dev = &rte_eth_devices[port_id];
1379         bond_mode_8023ad_conf_get(bond_dev, conf);
1380         return 0;
1381 }
1382
1383 int
1384 rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id,
1385                 enum rte_bond_8023ad_agg_selection agg_selection)
1386 {
1387         struct rte_eth_dev *bond_dev;
1388         struct bond_dev_private *internals;
1389         struct mode8023ad_private *mode4;
1390
1391         bond_dev = &rte_eth_devices[port_id];
1392         internals = bond_dev->data->dev_private;
1393
1394         if (valid_bonded_port_id(port_id) != 0)
1395                 return -EINVAL;
1396         if (internals->mode != 4)
1397                 return -EINVAL;
1398
1399         mode4 = &internals->mode4;
1400         if (agg_selection == AGG_COUNT || agg_selection == AGG_BANDWIDTH
1401                         || agg_selection == AGG_STABLE)
1402                 mode4->agg_selection = agg_selection;
1403         return 0;
1404 }
1405
1406 int rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id)
1407 {
1408         struct rte_eth_dev *bond_dev;
1409         struct bond_dev_private *internals;
1410         struct mode8023ad_private *mode4;
1411
1412         bond_dev = &rte_eth_devices[port_id];
1413         internals = bond_dev->data->dev_private;
1414
1415         if (valid_bonded_port_id(port_id) != 0)
1416                 return -EINVAL;
1417         if (internals->mode != 4)
1418                 return -EINVAL;
1419         mode4 = &internals->mode4;
1420
1421         return mode4->agg_selection;
1422 }
1423
1424
1425
1426 static int
1427 bond_8023ad_setup_validate(uint16_t port_id,
1428                 struct rte_eth_bond_8023ad_conf *conf)
1429 {
1430         if (valid_bonded_port_id(port_id) != 0)
1431                 return -EINVAL;
1432
1433         if (conf != NULL) {
1434                 /* Basic sanity check */
1435                 if (conf->slow_periodic_ms == 0 ||
1436                                 conf->fast_periodic_ms >= conf->slow_periodic_ms ||
1437                                 conf->long_timeout_ms == 0 ||
1438                                 conf->short_timeout_ms >= conf->long_timeout_ms ||
1439                                 conf->aggregate_wait_timeout_ms == 0 ||
1440                                 conf->tx_period_ms == 0 ||
1441                                 conf->rx_marker_period_ms == 0 ||
1442                                 conf->update_timeout_ms == 0) {
1443                         RTE_BOND_LOG(ERR, "given mode 4 configuration is invalid");
1444                         return -EINVAL;
1445                 }
1446         }
1447
1448         return 0;
1449 }
1450
1451
1452 int
1453 rte_eth_bond_8023ad_setup(uint16_t port_id,
1454                 struct rte_eth_bond_8023ad_conf *conf)
1455 {
1456         struct rte_eth_dev *bond_dev;
1457         int err;
1458
1459         err = bond_8023ad_setup_validate(port_id, conf);
1460         if (err != 0)
1461                 return err;
1462
1463         bond_dev = &rte_eth_devices[port_id];
1464         bond_mode_8023ad_setup(bond_dev, conf);
1465
1466         return 0;
1467 }
1468
1469
1470
1471
1472
1473 int
1474 rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id,
1475                 struct rte_eth_bond_8023ad_slave_info *info)
1476 {
1477         struct rte_eth_dev *bond_dev;
1478         struct bond_dev_private *internals;
1479         struct port *port;
1480
1481         if (info == NULL || valid_bonded_port_id(port_id) != 0 ||
1482                         rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1483                 return -EINVAL;
1484
1485         bond_dev = &rte_eth_devices[port_id];
1486
1487         internals = bond_dev->data->dev_private;
1488         if (find_slave_by_id(internals->active_slaves,
1489                         internals->active_slave_count, slave_id) ==
1490                                 internals->active_slave_count)
1491                 return -EINVAL;
1492
1493         port = &bond_mode_8023ad_ports[slave_id];
1494         info->selected = port->selected;
1495
1496         info->actor_state = port->actor_state;
1497         rte_memcpy(&info->actor, &port->actor, sizeof(port->actor));
1498
1499         info->partner_state = port->partner_state;
1500         rte_memcpy(&info->partner, &port->partner, sizeof(port->partner));
1501
1502         info->agg_port_id = port->aggregator_port_id;
1503         return 0;
1504 }
1505
1506 static int
1507 bond_8023ad_ext_validate(uint16_t port_id, uint16_t slave_id)
1508 {
1509         struct rte_eth_dev *bond_dev;
1510         struct bond_dev_private *internals;
1511         struct mode8023ad_private *mode4;
1512
1513         if (rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1514                 return -EINVAL;
1515
1516         bond_dev = &rte_eth_devices[port_id];
1517
1518         if (!bond_dev->data->dev_started)
1519                 return -EINVAL;
1520
1521         internals = bond_dev->data->dev_private;
1522         if (find_slave_by_id(internals->active_slaves,
1523                         internals->active_slave_count, slave_id) ==
1524                                 internals->active_slave_count)
1525                 return -EINVAL;
1526
1527         mode4 = &internals->mode4;
1528         if (mode4->slowrx_cb == NULL)
1529                 return -EINVAL;
1530
1531         return 0;
1532 }
1533
1534 int
1535 rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id,
1536                                 int enabled)
1537 {
1538         struct port *port;
1539         int res;
1540
1541         res = bond_8023ad_ext_validate(port_id, slave_id);
1542         if (res != 0)
1543                 return res;
1544
1545         port = &bond_mode_8023ad_ports[slave_id];
1546
1547         if (enabled)
1548                 ACTOR_STATE_SET(port, COLLECTING);
1549         else
1550                 ACTOR_STATE_CLR(port, COLLECTING);
1551
1552         return 0;
1553 }
1554
1555 int
1556 rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id,
1557                                 int enabled)
1558 {
1559         struct port *port;
1560         int res;
1561
1562         res = bond_8023ad_ext_validate(port_id, slave_id);
1563         if (res != 0)
1564                 return res;
1565
1566         port = &bond_mode_8023ad_ports[slave_id];
1567
1568         if (enabled)
1569                 ACTOR_STATE_SET(port, DISTRIBUTING);
1570         else
1571                 ACTOR_STATE_CLR(port, DISTRIBUTING);
1572
1573         return 0;
1574 }
1575
1576 int
1577 rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id)
1578 {
1579         struct port *port;
1580         int err;
1581
1582         err = bond_8023ad_ext_validate(port_id, slave_id);
1583         if (err != 0)
1584                 return err;
1585
1586         port = &bond_mode_8023ad_ports[slave_id];
1587         return ACTOR_STATE(port, DISTRIBUTING);
1588 }
1589
1590 int
1591 rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id)
1592 {
1593         struct port *port;
1594         int err;
1595
1596         err = bond_8023ad_ext_validate(port_id, slave_id);
1597         if (err != 0)
1598                 return err;
1599
1600         port = &bond_mode_8023ad_ports[slave_id];
1601         return ACTOR_STATE(port, COLLECTING);
1602 }
1603
1604 int
1605 rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id,
1606                 struct rte_mbuf *lacp_pkt)
1607 {
1608         struct port *port;
1609         int res;
1610
1611         res = bond_8023ad_ext_validate(port_id, slave_id);
1612         if (res != 0)
1613                 return res;
1614
1615         port = &bond_mode_8023ad_ports[slave_id];
1616
1617         if (rte_pktmbuf_pkt_len(lacp_pkt) < sizeof(struct lacpdu_header))
1618                 return -EINVAL;
1619
1620         struct lacpdu_header *lacp;
1621
1622         /* only enqueue LACPDUs */
1623         lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
1624         if (lacp->lacpdu.subtype != SLOW_SUBTYPE_LACP)
1625                 return -EINVAL;
1626
1627         MODE4_DEBUG("sending LACP frame\n");
1628
1629         return rte_ring_enqueue(port->tx_ring, lacp_pkt);
1630 }
1631
1632 static void
1633 bond_mode_8023ad_ext_periodic_cb(void *arg)
1634 {
1635         struct rte_eth_dev *bond_dev = arg;
1636         struct bond_dev_private *internals = bond_dev->data->dev_private;
1637         struct mode8023ad_private *mode4 = &internals->mode4;
1638         struct port *port;
1639         void *pkt = NULL;
1640         uint16_t i, slave_id;
1641
1642         for (i = 0; i < internals->active_slave_count; i++) {
1643                 slave_id = internals->active_slaves[i];
1644                 port = &bond_mode_8023ad_ports[slave_id];
1645
1646                 if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) {
1647                         struct rte_mbuf *lacp_pkt = pkt;
1648                         struct lacpdu_header *lacp;
1649
1650                         lacp = rte_pktmbuf_mtod(lacp_pkt,
1651                                                 struct lacpdu_header *);
1652                         RTE_VERIFY(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
1653
1654                         /* This is LACP frame so pass it to rx callback.
1655                          * Callback is responsible for freeing mbuf.
1656                          */
1657                         mode4->slowrx_cb(slave_id, lacp_pkt);
1658                 }
1659         }
1660
1661         rte_eal_alarm_set(internals->mode4.update_timeout_us,
1662                         bond_mode_8023ad_ext_periodic_cb, arg);
1663 }
1664
1665 int
1666 rte_eth_bond_8023ad_dedicated_queues_enable(uint16_t port)
1667 {
1668         int retval = 0;
1669         struct rte_eth_dev *dev = &rte_eth_devices[port];
1670         struct bond_dev_private *internals = (struct bond_dev_private *)
1671                 dev->data->dev_private;
1672
1673         if (check_for_bonded_ethdev(dev) != 0)
1674                 return -1;
1675
1676         if (bond_8023ad_slow_pkt_hw_filter_supported(port) != 0)
1677                 return -1;
1678
1679         /* Device must be stopped to set up slow queue */
1680         if (dev->data->dev_started)
1681                 return -1;
1682
1683         internals->mode4.dedicated_queues.enabled = 1;
1684
1685         bond_ethdev_mode_set(dev, internals->mode);
1686         return retval;
1687 }
1688
1689 int
1690 rte_eth_bond_8023ad_dedicated_queues_disable(uint16_t port)
1691 {
1692         int retval = 0;
1693         struct rte_eth_dev *dev = &rte_eth_devices[port];
1694         struct bond_dev_private *internals = (struct bond_dev_private *)
1695                 dev->data->dev_private;
1696
1697         if (check_for_bonded_ethdev(dev) != 0)
1698                 return -1;
1699
1700         /* Device must be stopped to set up slow queue */
1701         if (dev->data->dev_started)
1702                 return -1;
1703
1704         internals->mode4.dedicated_queues.enabled = 0;
1705
1706         bond_ethdev_mode_set(dev, internals->mode);
1707
1708         return retval;
1709 }