drivers/net/bonding/rte_eth_bond_8023ad.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <stddef.h>
   6 #include <string.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_alarm.h>
  10 #include <rte_malloc.h>
  11 #include <rte_errno.h>
  12 #include <rte_cycles.h>
  13 #include <rte_compat.h>
  14
  15 #include "rte_eth_bond_private.h"
  16
  17 static void bond_mode_8023ad_ext_periodic_cb(void *arg);
  18 #ifdef RTE_LIBRTE_BOND_DEBUG_8023AD
  19
  20 #define MODE4_DEBUG(fmt, ...)                           \
  21         rte_log(RTE_LOG_DEBUG, bond_logtype,            \
  22                 "%6u [Port %u: %s] " fmt,               \
  23                 bond_dbg_get_time_diff_ms(), slave_id,  \
  24                 __func__, ##__VA_ARGS__)
  25
  26 static uint64_t start_time;
  27
  28 static unsigned
  29 bond_dbg_get_time_diff_ms(void)
  30 {
  31         uint64_t now;
  32
  33         now = rte_rdtsc();
  34         if (start_time == 0)
  35                 start_time = now;
  36
  37         return ((now - start_time) * 1000) / rte_get_tsc_hz();
  38 }
  39
  40 static void
  41 bond_print_lacp(struct lacpdu *l)
  42 {
  43         char a_address[18];
  44         char p_address[18];
  45         char a_state[256] = { 0 };
  46         char p_state[256] = { 0 };
  47
  48         static const char * const state_labels[] = {
  49                 "ACT", "TIMEOUT", "AGG", "SYNC", "COL", "DIST", "DEF", "EXP"
  50         };
  51
  52         int a_len = 0;
  53         int p_len = 0;
  54         uint8_t i;
  55         uint8_t *addr;
  56
  57         addr = l->actor.port_params.system.addr_bytes;
  58         snprintf(a_address, sizeof(a_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  59                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  60
  61         addr = l->partner.port_params.system.addr_bytes;
  62         snprintf(p_address, sizeof(p_address), "%02X:%02X:%02X:%02X:%02X:%02X",
  63                 addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]);
  64
  65         for (i = 0; i < 8; i++) {
  66                 if ((l->actor.state >> i) & 1) {
  67                         a_len += snprintf(&a_state[a_len], RTE_DIM(a_state) - a_len, "%s ",
  68                                 state_labels[i]);
  69                 }
  70
  71                 if ((l->partner.state >> i) & 1) {
  72                         p_len += snprintf(&p_state[p_len], RTE_DIM(p_state) - p_len, "%s ",
  73                                 state_labels[i]);
  74                 }
  75         }
  76
  77         if (a_len && a_state[a_len-1] == ' ')
  78                 a_state[a_len-1] = '\0';
  79
  80         if (p_len && p_state[p_len-1] == ' ')
  81                 p_state[p_len-1] = '\0';
  82
  83         RTE_BOND_LOG(DEBUG,
  84                      "LACP: {\n"
  85                      "  subtype= %02X\n"
  86                      "  ver_num=%02X\n"
  87                      "  actor={ tlv=%02X, len=%02X\n"
  88                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  89                      "       state={ %s }\n"
  90                      "  }\n"
  91                      "  partner={ tlv=%02X, len=%02X\n"
  92                      "    pri=%04X, system=%s, key=%04X, p_pri=%04X p_num=%04X\n"
  93                      "       state={ %s }\n"
  94                      "  }\n"
  95                      "  collector={info=%02X, length=%02X, max_delay=%04X\n, "
  96                      "type_term=%02X, terminator_length = %02X }",
  97                      l->subtype,
  98                      l->version_number,
  99                      l->actor.tlv_type_info,
 100                      l->actor.info_length,
 101                      l->actor.port_params.system_priority,
 102                      a_address,
 103                      l->actor.port_params.key,
 104                      l->actor.port_params.port_priority,
 105                      l->actor.port_params.port_number,
 106                      a_state,
 107                      l->partner.tlv_type_info,
 108                      l->partner.info_length,
 109                      l->partner.port_params.system_priority,
 110                      p_address,
 111                      l->partner.port_params.key,
 112                      l->partner.port_params.port_priority,
 113                      l->partner.port_params.port_number,
 114                      p_state,
 115                      l->tlv_type_collector_info,
 116                      l->collector_info_length,
 117                      l->collector_max_delay,
 118                      l->tlv_type_terminator,
 119                      l->terminator_length);
 120
 121 }
 122
 123 #define BOND_PRINT_LACP(lacpdu) bond_print_lacp(lacpdu)
 124 #else
 125 #define BOND_PRINT_LACP(lacpdu) do { } while (0)
 126 #define MODE4_DEBUG(fmt, ...) do { } while (0)
 127 #endif
 128
 129 static const struct rte_ether_addr lacp_mac_addr = {
 130         .addr_bytes = { 0x01, 0x80, 0xC2, 0x00, 0x00, 0x02 }
 131 };
 132
 133 struct port bond_mode_8023ad_ports[RTE_MAX_ETHPORTS];
 134
 135 static void
 136 timer_cancel(uint64_t *timer)
 137 {
 138         *timer = 0;
 139 }
 140
 141 static void
 142 timer_set(uint64_t *timer, uint64_t timeout)
 143 {
 144         *timer = rte_rdtsc() + timeout;
 145 }
 146
 147 /* Forces given timer to be in expired state. */
 148 static void
 149 timer_force_expired(uint64_t *timer)
 150 {
 151         *timer = rte_rdtsc();
 152 }
 153
 154 static bool
 155 timer_is_stopped(uint64_t *timer)
 156 {
 157         return *timer == 0;
 158 }
 159
 160 static bool
 161 timer_is_expired(uint64_t *timer)
 162 {
 163         return *timer < rte_rdtsc();
 164 }
 165
 166 /* Timer is in running state if it is not stopped nor expired */
 167 static bool
 168 timer_is_running(uint64_t *timer)
 169 {
 170         return !timer_is_stopped(timer) && !timer_is_expired(timer);
 171 }
 172
 173 static void
 174 set_warning_flags(struct port *port, uint16_t flags)
 175 {
 176         int retval;
 177         uint16_t old;
 178         uint16_t new_flag = 0;
 179
 180         do {
 181                 old = port->warnings_to_show;
 182                 new_flag = old | flags;
 183                 retval = rte_atomic16_cmpset(&port->warnings_to_show, old, new_flag);
 184         } while (unlikely(retval == 0));
 185 }
 186
 187 static void
 188 show_warnings(uint16_t slave_id)
 189 {
 190         struct port *port = &bond_mode_8023ad_ports[slave_id];
 191         uint8_t warnings;
 192
 193         do {
 194                 warnings = port->warnings_to_show;
 195         } while (rte_atomic16_cmpset(&port->warnings_to_show, warnings, 0) == 0);
 196
 197         if (!warnings)
 198                 return;
 199
 200         if (!timer_is_expired(&port->warning_timer))
 201                 return;
 202
 203
 204         timer_set(&port->warning_timer, BOND_8023AD_WARNINGS_PERIOD_MS *
 205                         rte_get_tsc_hz() / 1000);
 206
 207         if (warnings & WRN_RX_QUEUE_FULL) {
 208                 RTE_BOND_LOG(DEBUG,
 209                              "Slave %u: failed to enqueue LACP packet into RX ring.\n"
 210                              "Receive and transmit functions must be invoked on bonded"
 211                              "interface at least 10 times per second or LACP will notwork correctly",
 212                              slave_id);
 213         }
 214
 215         if (warnings & WRN_TX_QUEUE_FULL) {
 216                 RTE_BOND_LOG(DEBUG,
 217                              "Slave %u: failed to enqueue LACP packet into TX ring.\n"
 218                              "Receive and transmit functions must be invoked on bonded"
 219                              "interface at least 10 times per second or LACP will not work correctly",
 220                              slave_id);
 221         }
 222
 223         if (warnings & WRN_RX_MARKER_TO_FAST)
 224                 RTE_BOND_LOG(INFO, "Slave %u: marker to early - ignoring.",
 225                              slave_id);
 226
 227         if (warnings & WRN_UNKNOWN_SLOW_TYPE) {
 228                 RTE_BOND_LOG(INFO,
 229                         "Slave %u: ignoring unknown slow protocol frame type",
 230                              slave_id);
 231         }
 232
 233         if (warnings & WRN_UNKNOWN_MARKER_TYPE)
 234                 RTE_BOND_LOG(INFO, "Slave %u: ignoring unknown marker type",
 235                              slave_id);
 236
 237         if (warnings & WRN_NOT_LACP_CAPABLE)
 238                 MODE4_DEBUG("Port %u is not LACP capable!\n", slave_id);
 239 }
 240
 241 static void
 242 record_default(struct port *port)
 243 {
 244         /* Record default parameters for partner. Partner admin parameters
 245          * are not implemented so set them to arbitrary default (last known) and
 246          * mark actor that parner is in defaulted state. */
 247         port->partner_state = STATE_LACP_ACTIVE;
 248         ACTOR_STATE_SET(port, DEFAULTED);
 249 }
 250
 251 /** Function handles rx state machine.
 252  *
 253  * This function implements Receive State Machine from point 5.4.12 in
 254  * 802.1AX documentation. It should be called periodically.
 255  *
 256  * @param lacpdu                LACPDU received.
 257  * @param port                  Port on which LACPDU was received.
 258  */
 259 static void
 260 rx_machine(struct bond_dev_private *internals, uint16_t slave_id,
 261                 struct lacpdu *lacp)
 262 {
 263         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 264         uint64_t timeout;
 265
 266         if (SM_FLAG(port, BEGIN)) {
 267                 /* Initialize stuff */
 268                 MODE4_DEBUG("-> INITIALIZE\n");
 269                 SM_FLAG_CLR(port, MOVED);
 270                 port->selected = UNSELECTED;
 271
 272                 record_default(port);
 273
 274                 ACTOR_STATE_CLR(port, EXPIRED);
 275                 timer_cancel(&port->current_while_timer);
 276
 277                 /* DISABLED: On initialization partner is out of sync */
 278                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 279
 280                 /* LACP DISABLED stuff if LACP not enabled on this port */
 281                 if (!SM_FLAG(port, LACP_ENABLED))
 282                         PARTNER_STATE_CLR(port, AGGREGATION);
 283                 else
 284                         PARTNER_STATE_SET(port, AGGREGATION);
 285         }
 286
 287         if (!SM_FLAG(port, LACP_ENABLED)) {
 288                 /* Update parameters only if state changed */
 289                 if (!timer_is_stopped(&port->current_while_timer)) {
 290                         port->selected = UNSELECTED;
 291                         record_default(port);
 292                         PARTNER_STATE_CLR(port, AGGREGATION);
 293                         ACTOR_STATE_CLR(port, EXPIRED);
 294                         timer_cancel(&port->current_while_timer);
 295                 }
 296                 return;
 297         }
 298
 299         if (lacp) {
 300                 MODE4_DEBUG("LACP -> CURRENT\n");
 301                 BOND_PRINT_LACP(lacp);
 302                 /* Update selected flag. If partner parameters are defaulted assume they
 303                  * are match. If not defaulted  compare LACP actor with ports parner
 304                  * params. */
 305                 if (!ACTOR_STATE(port, DEFAULTED) &&
 306                         (ACTOR_STATE(port, AGGREGATION) != PARTNER_STATE(port, AGGREGATION)
 307                         || memcmp(&port->partner, &lacp->actor.port_params,
 308                                 sizeof(port->partner)) != 0)) {
 309                         MODE4_DEBUG("selected <- UNSELECTED\n");
 310                         port->selected = UNSELECTED;
 311                 }
 312
 313                 /* Record this PDU actor params as partner params */
 314                 memcpy(&port->partner, &lacp->actor.port_params,
 315                         sizeof(struct port_params));
 316                 port->partner_state = lacp->actor.state;
 317
 318                 /* Partner parameters are not defaulted any more */
 319                 ACTOR_STATE_CLR(port, DEFAULTED);
 320
 321                 /* If LACP partner params match this port actor params */
 322                 agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 323                 bool match = port->actor.system_priority ==
 324                         lacp->partner.port_params.system_priority &&
 325                         rte_is_same_ether_addr(&agg->actor.system,
 326                         &lacp->partner.port_params.system) &&
 327                         port->actor.port_priority ==
 328                         lacp->partner.port_params.port_priority &&
 329                         port->actor.port_number ==
 330                         lacp->partner.port_params.port_number;
 331
 332                 /* Update NTT if partners information are outdated (xored and masked
 333                  * bits are set)*/
 334                 uint8_t state_mask = STATE_LACP_ACTIVE | STATE_LACP_SHORT_TIMEOUT |
 335                         STATE_SYNCHRONIZATION | STATE_AGGREGATION;
 336
 337                 if (((port->actor_state ^ lacp->partner.state) & state_mask) ||
 338                                 match == false) {
 339                         SM_FLAG_SET(port, NTT);
 340                 }
 341
 342                 /* If LACP partner params match this port actor params */
 343                 if (match == true && ACTOR_STATE(port, AGGREGATION) ==
 344                                 PARTNER_STATE(port,     AGGREGATION))
 345                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 346                 else if (!PARTNER_STATE(port, AGGREGATION) && ACTOR_STATE(port,
 347                                 AGGREGATION))
 348                         PARTNER_STATE_SET(port, SYNCHRONIZATION);
 349                 else
 350                         PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 351
 352                 if (ACTOR_STATE(port, LACP_SHORT_TIMEOUT))
 353                         timeout = internals->mode4.short_timeout;
 354                 else
 355                         timeout = internals->mode4.long_timeout;
 356
 357                 timer_set(&port->current_while_timer, timeout);
 358                 ACTOR_STATE_CLR(port, EXPIRED);
 359                 return; /* No state change */
 360         }
 361
 362         /* If CURRENT state timer is not running (stopped or expired)
 363          * transit to EXPIRED state from DISABLED or CURRENT */
 364         if (!timer_is_running(&port->current_while_timer)) {
 365                 ACTOR_STATE_SET(port, EXPIRED);
 366                 PARTNER_STATE_CLR(port, SYNCHRONIZATION);
 367                 PARTNER_STATE_SET(port, LACP_SHORT_TIMEOUT);
 368                 timer_set(&port->current_while_timer, internals->mode4.short_timeout);
 369         }
 370 }
 371
 372 /**
 373  * Function handles periodic tx state machine.
 374  *
 375  * Function implements Periodic Transmission state machine from point 5.4.13
 376  * in 802.1AX documentation. It should be called periodically.
 377  *
 378  * @param port                  Port to handle state machine.
 379  */
 380 static void
 381 periodic_machine(struct bond_dev_private *internals, uint16_t slave_id)
 382 {
 383         struct port *port = &bond_mode_8023ad_ports[slave_id];
 384         /* Calculate if either site is LACP enabled */
 385         uint64_t timeout;
 386         uint8_t active = ACTOR_STATE(port, LACP_ACTIVE) ||
 387                 PARTNER_STATE(port, LACP_ACTIVE);
 388
 389         uint8_t is_partner_fast, was_partner_fast;
 390         /* No periodic is on BEGIN, LACP DISABLE or when both sides are pasive */
 391         if (SM_FLAG(port, BEGIN) || !SM_FLAG(port, LACP_ENABLED) || !active) {
 392                 timer_cancel(&port->periodic_timer);
 393                 timer_force_expired(&port->tx_machine_timer);
 394                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 395
 396                 MODE4_DEBUG("-> NO_PERIODIC ( %s%s%s)\n",
 397                         SM_FLAG(port, BEGIN) ? "begind " : "",
 398                         SM_FLAG(port, LACP_ENABLED) ? "" : "LACP disabled ",
 399                         active ? "LACP active " : "LACP pasive ");
 400                 return;
 401         }
 402
 403         is_partner_fast = PARTNER_STATE(port, LACP_SHORT_TIMEOUT);
 404         was_partner_fast = SM_FLAG(port, PARTNER_SHORT_TIMEOUT);
 405
 406         /* If periodic timer is not started, transit from NO PERIODIC to FAST/SLOW.
 407          * Other case: check if timer expire or partners settings changed. */
 408         if (!timer_is_stopped(&port->periodic_timer)) {
 409                 if (timer_is_expired(&port->periodic_timer)) {
 410                         SM_FLAG_SET(port, NTT);
 411                 } else if (is_partner_fast != was_partner_fast) {
 412                         /* Partners timeout  was slow and now it is fast -> send LACP.
 413                          * In other case (was fast and now it is slow) just switch
 414                          * timeout to slow without forcing send of LACP (because standard
 415                          * say so)*/
 416                         if (is_partner_fast)
 417                                 SM_FLAG_SET(port, NTT);
 418                 } else
 419                         return; /* Nothing changed */
 420         }
 421
 422         /* Handle state transition to FAST/SLOW LACP timeout */
 423         if (is_partner_fast) {
 424                 timeout = internals->mode4.fast_periodic_timeout;
 425                 SM_FLAG_SET(port, PARTNER_SHORT_TIMEOUT);
 426         } else {
 427                 timeout = internals->mode4.slow_periodic_timeout;
 428                 SM_FLAG_CLR(port, PARTNER_SHORT_TIMEOUT);
 429         }
 430
 431         timer_set(&port->periodic_timer, timeout);
 432 }
 433
 434 /**
 435  * Function handles mux state machine.
 436  *
 437  * Function implements Mux Machine from point 5.4.15 in 802.1AX documentation.
 438  * It should be called periodically.
 439  *
 440  * @param port                  Port to handle state machine.
 441  */
 442 static void
 443 mux_machine(struct bond_dev_private *internals, uint16_t slave_id)
 444 {
 445         struct port *port = &bond_mode_8023ad_ports[slave_id];
 446
 447         /* Save current state for later use */
 448         const uint8_t state_mask = STATE_SYNCHRONIZATION | STATE_DISTRIBUTING |
 449                 STATE_COLLECTING;
 450
 451         /* Enter DETACHED state on BEGIN condition or from any other state if
 452          * port was unselected */
 453         if (SM_FLAG(port, BEGIN) ||
 454                         port->selected == UNSELECTED || (port->selected == STANDBY &&
 455                                 (port->actor_state & state_mask) != 0)) {
 456                 /* detach mux from aggregator */
 457                 port->actor_state &= ~state_mask;
 458                 /* Set ntt to true if BEGIN condition or transition from any other state
 459                  * which is indicated that wait_while_timer was started */
 460                 if (SM_FLAG(port, BEGIN) ||
 461                                 !timer_is_stopped(&port->wait_while_timer)) {
 462                         SM_FLAG_SET(port, NTT);
 463                         MODE4_DEBUG("-> DETACHED\n");
 464                 }
 465                 timer_cancel(&port->wait_while_timer);
 466         }
 467
 468         if (timer_is_stopped(&port->wait_while_timer)) {
 469                 if (port->selected == SELECTED || port->selected == STANDBY) {
 470                         timer_set(&port->wait_while_timer,
 471                                 internals->mode4.aggregate_wait_timeout);
 472
 473                         MODE4_DEBUG("DETACHED -> WAITING\n");
 474                 }
 475                 /* Waiting state entered */
 476                 return;
 477         }
 478
 479         /* Transit next state if port is ready */
 480         if (!timer_is_expired(&port->wait_while_timer))
 481                 return;
 482
 483         if ((ACTOR_STATE(port, DISTRIBUTING) || ACTOR_STATE(port, COLLECTING)) &&
 484                 !PARTNER_STATE(port, SYNCHRONIZATION)) {
 485                 /* If in COLLECTING or DISTRIBUTING state and partner becomes out of
 486                  * sync transit to ATACHED state.  */
 487                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 488                 ACTOR_STATE_CLR(port, COLLECTING);
 489                 /* Clear actor sync to activate transit ATACHED in condition bellow */
 490                 ACTOR_STATE_CLR(port, SYNCHRONIZATION);
 491                 MODE4_DEBUG("Out of sync -> ATTACHED\n");
 492         }
 493
 494         if (!ACTOR_STATE(port, SYNCHRONIZATION)) {
 495                 /* attach mux to aggregator */
 496                 RTE_ASSERT((port->actor_state & (STATE_COLLECTING |
 497                         STATE_DISTRIBUTING)) == 0);
 498
 499                 ACTOR_STATE_SET(port, SYNCHRONIZATION);
 500                 SM_FLAG_SET(port, NTT);
 501                 MODE4_DEBUG("ATTACHED Entered\n");
 502         } else if (!ACTOR_STATE(port, COLLECTING)) {
 503                 /* Start collecting if in sync */
 504                 if (PARTNER_STATE(port, SYNCHRONIZATION)) {
 505                         MODE4_DEBUG("ATTACHED -> COLLECTING\n");
 506                         ACTOR_STATE_SET(port, COLLECTING);
 507                         SM_FLAG_SET(port, NTT);
 508                 }
 509         } else if (ACTOR_STATE(port, COLLECTING)) {
 510                 /* Check if partner is in COLLECTING state. If so this port can
 511                  * distribute frames to it */
 512                 if (!ACTOR_STATE(port, DISTRIBUTING)) {
 513                         if (PARTNER_STATE(port, COLLECTING)) {
 514                                 /* Enable  DISTRIBUTING if partner is collecting */
 515                                 ACTOR_STATE_SET(port, DISTRIBUTING);
 516                                 SM_FLAG_SET(port, NTT);
 517                                 MODE4_DEBUG("COLLECTING -> DISTRIBUTING\n");
 518                                 RTE_BOND_LOG(INFO,
 519                                         "Bond %u: slave id %u distributing started.",
 520                                         internals->port_id, slave_id);
 521                         }
 522                 } else {
 523                         if (!PARTNER_STATE(port, COLLECTING)) {
 524                                 /* Disable DISTRIBUTING (enter COLLECTING state) if partner
 525                                  * is not collecting */
 526                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 527                                 SM_FLAG_SET(port, NTT);
 528                                 MODE4_DEBUG("DISTRIBUTING -> COLLECTING\n");
 529                                 RTE_BOND_LOG(INFO,
 530                                         "Bond %u: slave id %u distributing stopped.",
 531                                         internals->port_id, slave_id);
 532                         }
 533                 }
 534         }
 535 }
 536
 537 /**
 538  * Function handles transmit state machine.
 539  *
 540  * Function implements Transmit Machine from point 5.4.16 in 802.1AX
 541  * documentation.
 542  *
 543  * @param port
 544  */
 545 static void
 546 tx_machine(struct bond_dev_private *internals, uint16_t slave_id)
 547 {
 548         struct port *agg, *port = &bond_mode_8023ad_ports[slave_id];
 549
 550         struct rte_mbuf *lacp_pkt = NULL;
 551         struct lacpdu_header *hdr;
 552         struct lacpdu *lacpdu;
 553
 554         /* If periodic timer is not running periodic machine is in NO PERIODIC and
 555          * according to 802.3ax standard tx machine should not transmit any frames
 556          * and set ntt to false. */
 557         if (timer_is_stopped(&port->periodic_timer))
 558                 SM_FLAG_CLR(port, NTT);
 559
 560         if (!SM_FLAG(port, NTT))
 561                 return;
 562
 563         if (!timer_is_expired(&port->tx_machine_timer))
 564                 return;
 565
 566         lacp_pkt = rte_pktmbuf_alloc(port->mbuf_pool);
 567         if (lacp_pkt == NULL) {
 568                 RTE_BOND_LOG(ERR, "Failed to allocate LACP packet from pool");
 569                 return;
 570         }
 571
 572         lacp_pkt->data_len = sizeof(*hdr);
 573         lacp_pkt->pkt_len = sizeof(*hdr);
 574
 575         hdr = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 576
 577         /* Source and destination MAC */
 578         rte_ether_addr_copy(&lacp_mac_addr, &hdr->eth_hdr.d_addr);
 579         rte_eth_macaddr_get(slave_id, &hdr->eth_hdr.s_addr);
 580         hdr->eth_hdr.ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_SLOW);
 581
 582         lacpdu = &hdr->lacpdu;
 583         memset(lacpdu, 0, sizeof(*lacpdu));
 584
 585         /* Initialize LACP part */
 586         lacpdu->subtype = SLOW_SUBTYPE_LACP;
 587         lacpdu->version_number = 1;
 588
 589         /* ACTOR */
 590         lacpdu->actor.tlv_type_info = TLV_TYPE_ACTOR_INFORMATION;
 591         lacpdu->actor.info_length = sizeof(struct lacpdu_actor_partner_params);
 592         memcpy(&hdr->lacpdu.actor.port_params, &port->actor,
 593                         sizeof(port->actor));
 594         agg = &bond_mode_8023ad_ports[port->aggregator_port_id];
 595         rte_ether_addr_copy(&agg->actor.system,
 596                         &hdr->lacpdu.actor.port_params.system);
 597         lacpdu->actor.state = port->actor_state;
 598
 599         /* PARTNER */
 600         lacpdu->partner.tlv_type_info = TLV_TYPE_PARTNER_INFORMATION;
 601         lacpdu->partner.info_length = sizeof(struct lacpdu_actor_partner_params);
 602         memcpy(&lacpdu->partner.port_params, &port->partner,
 603                         sizeof(struct port_params));
 604         lacpdu->partner.state = port->partner_state;
 605
 606         /* Other fields */
 607         lacpdu->tlv_type_collector_info = TLV_TYPE_COLLECTOR_INFORMATION;
 608         lacpdu->collector_info_length = 0x10;
 609         lacpdu->collector_max_delay = 0;
 610
 611         lacpdu->tlv_type_terminator = TLV_TYPE_TERMINATOR_INFORMATION;
 612         lacpdu->terminator_length = 0;
 613
 614         MODE4_DEBUG("Sending LACP frame\n");
 615         BOND_PRINT_LACP(lacpdu);
 616
 617         if (internals->mode4.dedicated_queues.enabled == 0) {
 618                 int retval = rte_ring_enqueue(port->tx_ring, lacp_pkt);
 619                 if (retval != 0) {
 620                         /* If TX ring full, drop packet and free message.
 621                            Retransmission will happen in next function call. */
 622                         rte_pktmbuf_free(lacp_pkt);
 623                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 624                         return;
 625                 }
 626         } else {
 627                 uint16_t pkts_sent = rte_eth_tx_burst(slave_id,
 628                                 internals->mode4.dedicated_queues.tx_qid,
 629                                 &lacp_pkt, 1);
 630                 if (pkts_sent != 1) {
 631                         rte_pktmbuf_free(lacp_pkt);
 632                         set_warning_flags(port, WRN_TX_QUEUE_FULL);
 633                         return;
 634                 }
 635         }
 636
 637
 638         timer_set(&port->tx_machine_timer, internals->mode4.tx_period_timeout);
 639         SM_FLAG_CLR(port, NTT);
 640 }
 641
 642 static uint16_t
 643 max_index(uint64_t *a, int n)
 644 {
 645         if (n <= 0)
 646                 return -1;
 647
 648         int i, max_i = 0;
 649         uint64_t max = a[0];
 650
 651         for (i = 1; i < n; ++i) {
 652                 if (a[i] > max) {
 653                         max = a[i];
 654                         max_i = i;
 655                 }
 656         }
 657
 658         return max_i;
 659 }
 660
 661 /**
 662  * Function assigns port to aggregator.
 663  *
 664  * @param bond_dev_private      Pointer to bond_dev_private structure.
 665  * @param port_pos                      Port to assign.
 666  */
 667 static void
 668 selection_logic(struct bond_dev_private *internals, uint16_t slave_id)
 669 {
 670         struct port *agg, *port;
 671         uint16_t slaves_count, new_agg_id, i, j = 0;
 672         uint16_t *slaves;
 673         uint64_t agg_bandwidth[RTE_MAX_ETHPORTS] = {0};
 674         uint64_t agg_count[RTE_MAX_ETHPORTS] = {0};
 675         uint16_t default_slave = 0;
 676         uint16_t mode_count_id;
 677         uint16_t mode_band_id;
 678         struct rte_eth_link link_info;
 679         int ret;
 680
 681         slaves = internals->active_slaves;
 682         slaves_count = internals->active_slave_count;
 683         port = &bond_mode_8023ad_ports[slave_id];
 684
 685         /* Search for aggregator suitable for this port */
 686         for (i = 0; i < slaves_count; ++i) {
 687                 agg = &bond_mode_8023ad_ports[slaves[i]];
 688                 /* Skip ports that are not aggreagators */
 689                 if (agg->aggregator_port_id != slaves[i])
 690                         continue;
 691
 692                 ret = rte_eth_link_get_nowait(slaves[i], &link_info);
 693                 if (ret < 0) {
 694                         RTE_BOND_LOG(ERR,
 695                                 "Slave (port %u) link get failed: %s\n",
 696                                 slaves[i], rte_strerror(-ret));
 697                         continue;
 698                 }
 699                 agg_count[agg->aggregator_port_id] += 1;
 700                 agg_bandwidth[agg->aggregator_port_id] += link_info.link_speed;
 701
 702                 /* Actors system ID is not checked since all slave device have the same
 703                  * ID (MAC address). */
 704                 if ((agg->actor.key == port->actor.key &&
 705                         agg->partner.system_priority == port->partner.system_priority &&
 706                         rte_is_same_ether_addr(&agg->partner.system,
 707                                         &port->partner.system) == 1
 708                         && (agg->partner.key == port->partner.key)) &&
 709                         rte_is_zero_ether_addr(&port->partner.system) != 1 &&
 710                         (agg->actor.key &
 711                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) != 0) {
 712
 713                         if (j == 0)
 714                                 default_slave = i;
 715                         j++;
 716                 }
 717         }
 718
 719         switch (internals->mode4.agg_selection) {
 720         case AGG_COUNT:
 721                 mode_count_id = max_index(agg_count, slaves_count);
 722                 new_agg_id = mode_count_id;
 723                 break;
 724         case AGG_BANDWIDTH:
 725                 mode_band_id = max_index(agg_bandwidth, slaves_count);
 726                 new_agg_id = mode_band_id;
 727                 break;
 728         case AGG_STABLE:
 729                 if (default_slave == slaves_count)
 730                         new_agg_id = slave_id;
 731                 else
 732                         new_agg_id = slaves[default_slave];
 733                 break;
 734         default:
 735                 if (default_slave == slaves_count)
 736                         new_agg_id = slave_id;
 737                 else
 738                         new_agg_id = slaves[default_slave];
 739                 break;
 740         }
 741
 742         if (new_agg_id != port->aggregator_port_id) {
 743                 port->aggregator_port_id = new_agg_id;
 744
 745                 MODE4_DEBUG("-> SELECTED: ID=%3u\n"
 746                         "\t%s aggregator ID=%3u\n",
 747                         port->aggregator_port_id,
 748                         port->aggregator_port_id == slave_id ?
 749                                 "aggregator not found, using default" : "aggregator found",
 750                         port->aggregator_port_id);
 751         }
 752
 753         port->selected = SELECTED;
 754 }
 755
 756 /* Function maps DPDK speed to bonding speed stored in key field */
 757 static uint16_t
 758 link_speed_key(uint16_t speed) {
 759         uint16_t key_speed;
 760
 761         switch (speed) {
 762         case ETH_SPEED_NUM_NONE:
 763                 key_speed = 0x00;
 764                 break;
 765         case ETH_SPEED_NUM_10M:
 766                 key_speed = BOND_LINK_SPEED_KEY_10M;
 767                 break;
 768         case ETH_SPEED_NUM_100M:
 769                 key_speed = BOND_LINK_SPEED_KEY_100M;
 770                 break;
 771         case ETH_SPEED_NUM_1G:
 772                 key_speed = BOND_LINK_SPEED_KEY_1000M;
 773                 break;
 774         case ETH_SPEED_NUM_10G:
 775                 key_speed = BOND_LINK_SPEED_KEY_10G;
 776                 break;
 777         case ETH_SPEED_NUM_20G:
 778                 key_speed = BOND_LINK_SPEED_KEY_20G;
 779                 break;
 780         case ETH_SPEED_NUM_40G:
 781                 key_speed = BOND_LINK_SPEED_KEY_40G;
 782                 break;
 783         default:
 784                 /* Unknown speed*/
 785                 key_speed = 0xFFFF;
 786         }
 787
 788         return key_speed;
 789 }
 790
 791 static void
 792 rx_machine_update(struct bond_dev_private *internals, uint16_t slave_id,
 793                 struct rte_mbuf *lacp_pkt) {
 794         struct lacpdu_header *lacp;
 795         struct lacpdu_actor_partner_params *partner;
 796
 797         if (lacp_pkt != NULL) {
 798                 lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
 799                 RTE_ASSERT(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
 800
 801                 partner = &lacp->lacpdu.partner;
 802                 if (rte_is_same_ether_addr(&partner->port_params.system,
 803                         &internals->mode4.mac_addr)) {
 804                         /* This LACP frame is sending to the bonding port
 805                          * so pass it to rx_machine.
 806                          */
 807                         rx_machine(internals, slave_id, &lacp->lacpdu);
 808                 }
 809                 rte_pktmbuf_free(lacp_pkt);
 810         } else
 811                 rx_machine(internals, slave_id, NULL);
 812 }
 813
 814 static void
 815 bond_mode_8023ad_periodic_cb(void *arg)
 816 {
 817         struct rte_eth_dev *bond_dev = arg;
 818         struct bond_dev_private *internals = bond_dev->data->dev_private;
 819         struct port *port;
 820         struct rte_eth_link link_info;
 821         struct rte_ether_addr slave_addr;
 822         struct rte_mbuf *lacp_pkt = NULL;
 823         uint16_t slave_id;
 824         uint16_t i;
 825
 826
 827         /* Update link status on each port */
 828         for (i = 0; i < internals->active_slave_count; i++) {
 829                 uint16_t key;
 830                 int ret;
 831
 832                 slave_id = internals->active_slaves[i];
 833                 ret = rte_eth_link_get_nowait(slave_id, &link_info);
 834                 if (ret < 0) {
 835                         RTE_BOND_LOG(ERR,
 836                                 "Slave (port %u) link get failed: %s\n",
 837                                 slave_id, rte_strerror(-ret));
 838                 }
 839
 840                 if (ret >= 0 && link_info.link_status != 0) {
 841                         key = link_speed_key(link_info.link_speed) << 1;
 842                         if (link_info.link_duplex == ETH_LINK_FULL_DUPLEX)
 843                                 key |= BOND_LINK_FULL_DUPLEX_KEY;
 844                 } else {
 845                         key = 0;
 846                 }
 847
 848                 rte_eth_macaddr_get(slave_id, &slave_addr);
 849                 port = &bond_mode_8023ad_ports[slave_id];
 850
 851                 key = rte_cpu_to_be_16(key);
 852                 if (key != port->actor.key) {
 853                         if (!(key & rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)))
 854                                 set_warning_flags(port, WRN_NOT_LACP_CAPABLE);
 855
 856                         port->actor.key = key;
 857                         SM_FLAG_SET(port, NTT);
 858                 }
 859
 860                 if (!rte_is_same_ether_addr(&port->actor.system, &slave_addr)) {
 861                         rte_ether_addr_copy(&slave_addr, &port->actor.system);
 862                         if (port->aggregator_port_id == slave_id)
 863                                 SM_FLAG_SET(port, NTT);
 864                 }
 865         }
 866
 867         for (i = 0; i < internals->active_slave_count; i++) {
 868                 slave_id = internals->active_slaves[i];
 869                 port = &bond_mode_8023ad_ports[slave_id];
 870
 871                 if ((port->actor.key &
 872                                 rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY)) == 0) {
 873
 874                         SM_FLAG_SET(port, BEGIN);
 875
 876                         /* LACP is disabled on half duples or link is down */
 877                         if (SM_FLAG(port, LACP_ENABLED)) {
 878                                 /* If port was enabled set it to BEGIN state */
 879                                 SM_FLAG_CLR(port, LACP_ENABLED);
 880                                 ACTOR_STATE_CLR(port, DISTRIBUTING);
 881                                 ACTOR_STATE_CLR(port, COLLECTING);
 882                         }
 883
 884                         /* Skip this port processing */
 885                         continue;
 886                 }
 887
 888                 SM_FLAG_SET(port, LACP_ENABLED);
 889
 890                 if (internals->mode4.dedicated_queues.enabled == 0) {
 891                         /* Find LACP packet to this port. Do not check subtype,
 892                          * it is done in function that queued packet
 893                          */
 894                         int retval = rte_ring_dequeue(port->rx_ring,
 895                                         (void **)&lacp_pkt);
 896
 897                         if (retval != 0)
 898                                 lacp_pkt = NULL;
 899
 900                         rx_machine_update(internals, slave_id, lacp_pkt);
 901                 } else {
 902                         uint16_t rx_count = rte_eth_rx_burst(slave_id,
 903                                         internals->mode4.dedicated_queues.rx_qid,
 904                                         &lacp_pkt, 1);
 905
 906                         if (rx_count == 1)
 907                                 bond_mode_8023ad_handle_slow_pkt(internals,
 908                                                 slave_id, lacp_pkt);
 909                         else
 910                                 rx_machine_update(internals, slave_id, NULL);
 911                 }
 912
 913                 periodic_machine(internals, slave_id);
 914                 mux_machine(internals, slave_id);
 915                 tx_machine(internals, slave_id);
 916                 selection_logic(internals, slave_id);
 917
 918                 SM_FLAG_CLR(port, BEGIN);
 919                 show_warnings(slave_id);
 920         }
 921
 922         rte_eal_alarm_set(internals->mode4.update_timeout_us,
 923                         bond_mode_8023ad_periodic_cb, arg);
 924 }
 925
 926 static int
 927 bond_mode_8023ad_register_lacp_mac(uint16_t slave_id)
 928 {
 929         int ret;
 930
 931         ret = rte_eth_allmulticast_enable(slave_id);
 932         if (ret != 0) {
 933                 RTE_BOND_LOG(ERR,
 934                         "failed to enable allmulti mode for port %u: %s",
 935                         slave_id, rte_strerror(-ret));
 936         }
 937         if (rte_eth_allmulticast_get(slave_id)) {
 938                 RTE_BOND_LOG(DEBUG, "forced allmulti for port %u",
 939                              slave_id);
 940                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 941                                 BOND_8023AD_FORCED_ALLMULTI;
 942                 return 0;
 943         }
 944
 945         ret = rte_eth_promiscuous_enable(slave_id);
 946         if (ret != 0) {
 947                 RTE_BOND_LOG(ERR,
 948                         "failed to enable promiscuous mode for port %u: %s",
 949                         slave_id, rte_strerror(-ret));
 950         }
 951         if (rte_eth_promiscuous_get(slave_id)) {
 952                 RTE_BOND_LOG(DEBUG, "forced promiscuous for port %u",
 953                              slave_id);
 954                 bond_mode_8023ad_ports[slave_id].forced_rx_flags =
 955                                 BOND_8023AD_FORCED_PROMISC;
 956                 return 0;
 957         }
 958
 959         return -1;
 960 }
 961
 962 static void
 963 bond_mode_8023ad_unregister_lacp_mac(uint16_t slave_id)
 964 {
 965         int ret;
 966
 967         switch (bond_mode_8023ad_ports[slave_id].forced_rx_flags) {
 968         case BOND_8023AD_FORCED_ALLMULTI:
 969                 RTE_BOND_LOG(DEBUG, "unset allmulti for port %u", slave_id);
 970                 ret = rte_eth_allmulticast_disable(slave_id);
 971                 if (ret != 0)
 972                         RTE_BOND_LOG(ERR,
 973                                 "failed to disable allmulti mode for port %u: %s",
 974                                 slave_id, rte_strerror(-ret));
 975                 break;
 976
 977         case BOND_8023AD_FORCED_PROMISC:
 978                 RTE_BOND_LOG(DEBUG, "unset promisc for port %u", slave_id);
 979                 ret = rte_eth_promiscuous_disable(slave_id);
 980                 if (ret != 0)
 981                         RTE_BOND_LOG(ERR,
 982                                 "failed to disable promiscuous mode for port %u: %s",
 983                                 slave_id, rte_strerror(-ret));
 984                 break;
 985
 986         default:
 987                 break;
 988         }
 989 }
 990
 991 void
 992 bond_mode_8023ad_activate_slave(struct rte_eth_dev *bond_dev,
 993                                 uint16_t slave_id)
 994 {
 995         struct bond_dev_private *internals = bond_dev->data->dev_private;
 996
 997         struct port *port = &bond_mode_8023ad_ports[slave_id];
 998         struct port_params initial = {
 999                         .system = { { 0 } },
1000                         .system_priority = rte_cpu_to_be_16(0xFFFF),
1001                         .key = rte_cpu_to_be_16(BOND_LINK_FULL_DUPLEX_KEY),
1002                         .port_priority = rte_cpu_to_be_16(0x00FF),
1003                         .port_number = 0,
1004         };
1005
1006         char mem_name[RTE_ETH_NAME_MAX_LEN];
1007         int socket_id;
1008         unsigned element_size;
1009         uint32_t total_tx_desc;
1010         struct bond_tx_queue *bd_tx_q;
1011         uint16_t q_id;
1012
1013         /* Given slave mus not be in active list */
1014         RTE_ASSERT(find_slave_by_id(internals->active_slaves,
1015         internals->active_slave_count, slave_id) == internals->active_slave_count);
1016         RTE_SET_USED(internals); /* used only for assert when enabled */
1017
1018         memcpy(&port->actor, &initial, sizeof(struct port_params));
1019         /* Standard requires that port ID must be grater than 0.
1020          * Add 1 do get corresponding port_number */
1021         port->actor.port_number = rte_cpu_to_be_16(slave_id + 1);
1022
1023         memcpy(&port->partner, &initial, sizeof(struct port_params));
1024
1025         /* default states */
1026         port->actor_state = STATE_AGGREGATION | STATE_LACP_ACTIVE | STATE_DEFAULTED;
1027         port->partner_state = STATE_LACP_ACTIVE | STATE_AGGREGATION;
1028         port->sm_flags = SM_FLAGS_BEGIN;
1029
1030         /* use this port as agregator */
1031         port->aggregator_port_id = slave_id;
1032
1033         if (bond_mode_8023ad_register_lacp_mac(slave_id) < 0) {
1034                 RTE_BOND_LOG(WARNING, "slave %u is most likely broken and won't receive LACP packets",
1035                              slave_id);
1036         }
1037
1038         timer_cancel(&port->warning_timer);
1039
1040         if (port->mbuf_pool != NULL)
1041                 return;
1042
1043         RTE_ASSERT(port->rx_ring == NULL);
1044         RTE_ASSERT(port->tx_ring == NULL);
1045
1046         socket_id = rte_eth_dev_socket_id(slave_id);
1047         if (socket_id == (int)LCORE_ID_ANY)
1048                 socket_id = rte_socket_id();
1049
1050         element_size = sizeof(struct slow_protocol_frame) +
1051                                 RTE_PKTMBUF_HEADROOM;
1052
1053         /* The size of the mempool should be at least:
1054          * the sum of the TX descriptors + BOND_MODE_8023AX_SLAVE_TX_PKTS */
1055         total_tx_desc = BOND_MODE_8023AX_SLAVE_TX_PKTS;
1056         for (q_id = 0; q_id < bond_dev->data->nb_tx_queues; q_id++) {
1057                 bd_tx_q = (struct bond_tx_queue*)bond_dev->data->tx_queues[q_id];
1058                 total_tx_desc += bd_tx_q->nb_tx_desc;
1059         }
1060
1061         snprintf(mem_name, RTE_DIM(mem_name), "slave_port%u_pool", slave_id);
1062         port->mbuf_pool = rte_pktmbuf_pool_create(mem_name, total_tx_desc,
1063                 RTE_MEMPOOL_CACHE_MAX_SIZE >= 32 ?
1064                         32 : RTE_MEMPOOL_CACHE_MAX_SIZE,
1065                 0, element_size, socket_id);
1066
1067         /* Any memory allocation failure in initialization is critical because
1068          * resources can't be free, so reinitialization is impossible. */
1069         if (port->mbuf_pool == NULL) {
1070                 rte_panic("Slave %u: Failed to create memory pool '%s': %s\n",
1071                         slave_id, mem_name, rte_strerror(rte_errno));
1072         }
1073
1074         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_rx", slave_id);
1075         port->rx_ring = rte_ring_create(mem_name,
1076                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_RX_PKTS), socket_id, 0);
1077
1078         if (port->rx_ring == NULL) {
1079                 rte_panic("Slave %u: Failed to create rx ring '%s': %s\n", slave_id,
1080                         mem_name, rte_strerror(rte_errno));
1081         }
1082
1083         /* TX ring is at least one pkt longer to make room for marker packet. */
1084         snprintf(mem_name, RTE_DIM(mem_name), "slave_%u_tx", slave_id);
1085         port->tx_ring = rte_ring_create(mem_name,
1086                         rte_align32pow2(BOND_MODE_8023AX_SLAVE_TX_PKTS + 1), socket_id, 0);
1087
1088         if (port->tx_ring == NULL) {
1089                 rte_panic("Slave %u: Failed to create tx ring '%s': %s\n", slave_id,
1090                         mem_name, rte_strerror(rte_errno));
1091         }
1092 }
1093
1094 int
1095 bond_mode_8023ad_deactivate_slave(struct rte_eth_dev *bond_dev __rte_unused,
1096                 uint16_t slave_id)
1097 {
1098         void *pkt = NULL;
1099         struct port *port = NULL;
1100         uint8_t old_partner_state;
1101
1102         port = &bond_mode_8023ad_ports[slave_id];
1103
1104         ACTOR_STATE_CLR(port, AGGREGATION);
1105         port->selected = UNSELECTED;
1106
1107         old_partner_state = port->partner_state;
1108         record_default(port);
1109
1110         bond_mode_8023ad_unregister_lacp_mac(slave_id);
1111
1112         /* If partner timeout state changes then disable timer */
1113         if (!((old_partner_state ^ port->partner_state) &
1114                         STATE_LACP_SHORT_TIMEOUT))
1115                 timer_cancel(&port->current_while_timer);
1116
1117         PARTNER_STATE_CLR(port, AGGREGATION);
1118         ACTOR_STATE_CLR(port, EXPIRED);
1119
1120         /* flush rx/tx rings */
1121         while (rte_ring_dequeue(port->rx_ring, &pkt) == 0)
1122                 rte_pktmbuf_free((struct rte_mbuf *)pkt);
1123
1124         while (rte_ring_dequeue(port->tx_ring, &pkt) == 0)
1125                         rte_pktmbuf_free((struct rte_mbuf *)pkt);
1126         return 0;
1127 }
1128
1129 void
1130 bond_mode_8023ad_mac_address_update(struct rte_eth_dev *bond_dev)
1131 {
1132         struct bond_dev_private *internals = bond_dev->data->dev_private;
1133         struct rte_ether_addr slave_addr;
1134         struct port *slave, *agg_slave;
1135         uint16_t slave_id, i, j;
1136
1137         bond_mode_8023ad_stop(bond_dev);
1138
1139         for (i = 0; i < internals->active_slave_count; i++) {
1140                 slave_id = internals->active_slaves[i];
1141                 slave = &bond_mode_8023ad_ports[slave_id];
1142                 rte_eth_macaddr_get(slave_id, &slave_addr);
1143
1144                 if (rte_is_same_ether_addr(&slave_addr, &slave->actor.system))
1145                         continue;
1146
1147                 rte_ether_addr_copy(&slave_addr, &slave->actor.system);
1148                 /* Do nothing if this port is not an aggregator. In other case
1149                  * Set NTT flag on every port that use this aggregator. */
1150                 if (slave->aggregator_port_id != slave_id)
1151                         continue;
1152
1153                 for (j = 0; j < internals->active_slave_count; j++) {
1154                         agg_slave = &bond_mode_8023ad_ports[internals->active_slaves[j]];
1155                         if (agg_slave->aggregator_port_id == slave_id)
1156                                 SM_FLAG_SET(agg_slave, NTT);
1157                 }
1158         }
1159
1160         if (bond_dev->data->dev_started)
1161                 bond_mode_8023ad_start(bond_dev);
1162 }
1163
1164 static void
1165 bond_mode_8023ad_conf_get(struct rte_eth_dev *dev,
1166                 struct rte_eth_bond_8023ad_conf *conf)
1167 {
1168         struct bond_dev_private *internals = dev->data->dev_private;
1169         struct mode8023ad_private *mode4 = &internals->mode4;
1170         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1171
1172         conf->fast_periodic_ms = mode4->fast_periodic_timeout / ms_ticks;
1173         conf->slow_periodic_ms = mode4->slow_periodic_timeout / ms_ticks;
1174         conf->short_timeout_ms = mode4->short_timeout / ms_ticks;
1175         conf->long_timeout_ms = mode4->long_timeout / ms_ticks;
1176         conf->aggregate_wait_timeout_ms = mode4->aggregate_wait_timeout / ms_ticks;
1177         conf->tx_period_ms = mode4->tx_period_timeout / ms_ticks;
1178         conf->update_timeout_ms = mode4->update_timeout_us / 1000;
1179         conf->rx_marker_period_ms = mode4->rx_marker_timeout / ms_ticks;
1180         conf->slowrx_cb = mode4->slowrx_cb;
1181         conf->agg_selection = mode4->agg_selection;
1182 }
1183
1184 static void
1185 bond_mode_8023ad_conf_get_default(struct rte_eth_bond_8023ad_conf *conf)
1186 {
1187         conf->fast_periodic_ms = BOND_8023AD_FAST_PERIODIC_MS;
1188         conf->slow_periodic_ms = BOND_8023AD_SLOW_PERIODIC_MS;
1189         conf->short_timeout_ms = BOND_8023AD_SHORT_TIMEOUT_MS;
1190         conf->long_timeout_ms = BOND_8023AD_LONG_TIMEOUT_MS;
1191         conf->aggregate_wait_timeout_ms = BOND_8023AD_AGGREGATE_WAIT_TIMEOUT_MS;
1192         conf->tx_period_ms = BOND_8023AD_TX_MACHINE_PERIOD_MS;
1193         conf->rx_marker_period_ms = BOND_8023AD_RX_MARKER_PERIOD_MS;
1194         conf->update_timeout_ms = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS;
1195         conf->slowrx_cb = NULL;
1196         conf->agg_selection = AGG_STABLE;
1197 }
1198
1199 static void
1200 bond_mode_8023ad_conf_assign(struct mode8023ad_private *mode4,
1201                 struct rte_eth_bond_8023ad_conf *conf)
1202 {
1203         uint64_t ms_ticks = rte_get_tsc_hz() / 1000;
1204
1205         mode4->fast_periodic_timeout = conf->fast_periodic_ms * ms_ticks;
1206         mode4->slow_periodic_timeout = conf->slow_periodic_ms * ms_ticks;
1207         mode4->short_timeout = conf->short_timeout_ms * ms_ticks;
1208         mode4->long_timeout = conf->long_timeout_ms * ms_ticks;
1209         mode4->aggregate_wait_timeout = conf->aggregate_wait_timeout_ms * ms_ticks;
1210         mode4->tx_period_timeout = conf->tx_period_ms * ms_ticks;
1211         mode4->rx_marker_timeout = conf->rx_marker_period_ms * ms_ticks;
1212         mode4->update_timeout_us = conf->update_timeout_ms * 1000;
1213
1214         mode4->dedicated_queues.enabled = 0;
1215         mode4->dedicated_queues.rx_qid = UINT16_MAX;
1216         mode4->dedicated_queues.tx_qid = UINT16_MAX;
1217 }
1218
1219 void
1220 bond_mode_8023ad_setup(struct rte_eth_dev *dev,
1221                 struct rte_eth_bond_8023ad_conf *conf)
1222 {
1223         struct rte_eth_bond_8023ad_conf def_conf;
1224         struct bond_dev_private *internals = dev->data->dev_private;
1225         struct mode8023ad_private *mode4 = &internals->mode4;
1226
1227         if (conf == NULL) {
1228                 conf = &def_conf;
1229                 bond_mode_8023ad_conf_get_default(conf);
1230         }
1231
1232         bond_mode_8023ad_stop(dev);
1233         bond_mode_8023ad_conf_assign(mode4, conf);
1234         mode4->slowrx_cb = conf->slowrx_cb;
1235         mode4->agg_selection = AGG_STABLE;
1236
1237         if (dev->data->dev_started)
1238                 bond_mode_8023ad_start(dev);
1239 }
1240
1241 int
1242 bond_mode_8023ad_enable(struct rte_eth_dev *bond_dev)
1243 {
1244         struct bond_dev_private *internals = bond_dev->data->dev_private;
1245         uint16_t i;
1246
1247         for (i = 0; i < internals->active_slave_count; i++)
1248                 bond_mode_8023ad_activate_slave(bond_dev,
1249                                 internals->active_slaves[i]);
1250
1251         return 0;
1252 }
1253
1254 int
1255 bond_mode_8023ad_start(struct rte_eth_dev *bond_dev)
1256 {
1257         struct bond_dev_private *internals = bond_dev->data->dev_private;
1258         struct mode8023ad_private *mode4 = &internals->mode4;
1259         static const uint64_t us = BOND_MODE_8023AX_UPDATE_TIMEOUT_MS * 1000;
1260
1261         rte_eth_macaddr_get(internals->port_id, &mode4->mac_addr);
1262         if (mode4->slowrx_cb)
1263                 return rte_eal_alarm_set(us, &bond_mode_8023ad_ext_periodic_cb,
1264                                          bond_dev);
1265
1266         return rte_eal_alarm_set(us, &bond_mode_8023ad_periodic_cb, bond_dev);
1267 }
1268
1269 void
1270 bond_mode_8023ad_stop(struct rte_eth_dev *bond_dev)
1271 {
1272         struct bond_dev_private *internals = bond_dev->data->dev_private;
1273         struct mode8023ad_private *mode4 = &internals->mode4;
1274
1275         if (mode4->slowrx_cb) {
1276                 rte_eal_alarm_cancel(&bond_mode_8023ad_ext_periodic_cb,
1277                                      bond_dev);
1278                 return;
1279         }
1280         rte_eal_alarm_cancel(&bond_mode_8023ad_periodic_cb, bond_dev);
1281 }
1282
1283 void
1284 bond_mode_8023ad_handle_slow_pkt(struct bond_dev_private *internals,
1285                                   uint16_t slave_id, struct rte_mbuf *pkt)
1286 {
1287         struct mode8023ad_private *mode4 = &internals->mode4;
1288         struct port *port = &bond_mode_8023ad_ports[slave_id];
1289         struct marker_header *m_hdr;
1290         uint64_t marker_timer, old_marker_timer;
1291         int retval;
1292         uint8_t wrn, subtype;
1293         /* If packet is a marker, we send response now by reusing given packet
1294          * and update only source MAC, destination MAC is multicast so don't
1295          * update it. Other frames will be handled later by state machines */
1296         subtype = rte_pktmbuf_mtod(pkt,
1297                         struct slow_protocol_frame *)->slow_protocol.subtype;
1298
1299         if (subtype == SLOW_SUBTYPE_MARKER) {
1300                 m_hdr = rte_pktmbuf_mtod(pkt, struct marker_header *);
1301
1302                 if (likely(m_hdr->marker.tlv_type_marker != MARKER_TLV_TYPE_INFO)) {
1303                         wrn = WRN_UNKNOWN_MARKER_TYPE;
1304                         goto free_out;
1305                 }
1306
1307                 /* Setup marker timer. Do it in loop in case concurrent access. */
1308                 do {
1309                         old_marker_timer = port->rx_marker_timer;
1310                         if (!timer_is_expired(&old_marker_timer)) {
1311                                 wrn = WRN_RX_MARKER_TO_FAST;
1312                                 goto free_out;
1313                         }
1314
1315                         timer_set(&marker_timer, mode4->rx_marker_timeout);
1316                         retval = rte_atomic64_cmpset(&port->rx_marker_timer,
1317                                 old_marker_timer, marker_timer);
1318                 } while (unlikely(retval == 0));
1319
1320                 m_hdr->marker.tlv_type_marker = MARKER_TLV_TYPE_RESP;
1321                 rte_eth_macaddr_get(slave_id, &m_hdr->eth_hdr.s_addr);
1322
1323                 if (internals->mode4.dedicated_queues.enabled == 0) {
1324                         int retval = rte_ring_enqueue(port->tx_ring, pkt);
1325                         if (retval != 0) {
1326                                 /* reset timer */
1327                                 port->rx_marker_timer = 0;
1328                                 wrn = WRN_TX_QUEUE_FULL;
1329                                 goto free_out;
1330                         }
1331                 } else {
1332                         /* Send packet directly to the slow queue */
1333                         uint16_t tx_count = rte_eth_tx_burst(slave_id,
1334                                         internals->mode4.dedicated_queues.tx_qid,
1335                                         &pkt, 1);
1336                         if (tx_count != 1) {
1337                                 /* reset timer */
1338                                 port->rx_marker_timer = 0;
1339                                 wrn = WRN_TX_QUEUE_FULL;
1340                                 goto free_out;
1341                         }
1342                 }
1343         } else if (likely(subtype == SLOW_SUBTYPE_LACP)) {
1344                 if (internals->mode4.dedicated_queues.enabled == 0) {
1345                         int retval = rte_ring_enqueue(port->rx_ring, pkt);
1346                         if (retval != 0) {
1347                                 /* If RX fing full free lacpdu message and drop packet */
1348                                 wrn = WRN_RX_QUEUE_FULL;
1349                                 goto free_out;
1350                         }
1351                 } else
1352                         rx_machine_update(internals, slave_id, pkt);
1353         } else {
1354                 wrn = WRN_UNKNOWN_SLOW_TYPE;
1355                 goto free_out;
1356         }
1357
1358         return;
1359
1360 free_out:
1361         set_warning_flags(port, wrn);
1362         rte_pktmbuf_free(pkt);
1363 }
1364
1365 int
1366 rte_eth_bond_8023ad_conf_get(uint16_t port_id,
1367                 struct rte_eth_bond_8023ad_conf *conf)
1368 {
1369         struct rte_eth_dev *bond_dev;
1370
1371         if (valid_bonded_port_id(port_id) != 0)
1372                 return -EINVAL;
1373
1374         if (conf == NULL)
1375                 return -EINVAL;
1376
1377         bond_dev = &rte_eth_devices[port_id];
1378         bond_mode_8023ad_conf_get(bond_dev, conf);
1379         return 0;
1380 }
1381
1382 int
1383 rte_eth_bond_8023ad_agg_selection_set(uint16_t port_id,
1384                 enum rte_bond_8023ad_agg_selection agg_selection)
1385 {
1386         struct rte_eth_dev *bond_dev;
1387         struct bond_dev_private *internals;
1388         struct mode8023ad_private *mode4;
1389
1390         bond_dev = &rte_eth_devices[port_id];
1391         internals = bond_dev->data->dev_private;
1392
1393         if (valid_bonded_port_id(port_id) != 0)
1394                 return -EINVAL;
1395         if (internals->mode != 4)
1396                 return -EINVAL;
1397
1398         mode4 = &internals->mode4;
1399         if (agg_selection == AGG_COUNT || agg_selection == AGG_BANDWIDTH
1400                         || agg_selection == AGG_STABLE)
1401                 mode4->agg_selection = agg_selection;
1402         return 0;
1403 }
1404
1405 int rte_eth_bond_8023ad_agg_selection_get(uint16_t port_id)
1406 {
1407         struct rte_eth_dev *bond_dev;
1408         struct bond_dev_private *internals;
1409         struct mode8023ad_private *mode4;
1410
1411         bond_dev = &rte_eth_devices[port_id];
1412         internals = bond_dev->data->dev_private;
1413
1414         if (valid_bonded_port_id(port_id) != 0)
1415                 return -EINVAL;
1416         if (internals->mode != 4)
1417                 return -EINVAL;
1418         mode4 = &internals->mode4;
1419
1420         return mode4->agg_selection;
1421 }
1422
1423
1424
1425 static int
1426 bond_8023ad_setup_validate(uint16_t port_id,
1427                 struct rte_eth_bond_8023ad_conf *conf)
1428 {
1429         if (valid_bonded_port_id(port_id) != 0)
1430                 return -EINVAL;
1431
1432         if (conf != NULL) {
1433                 /* Basic sanity check */
1434                 if (conf->slow_periodic_ms == 0 ||
1435                                 conf->fast_periodic_ms >= conf->slow_periodic_ms ||
1436                                 conf->long_timeout_ms == 0 ||
1437                                 conf->short_timeout_ms >= conf->long_timeout_ms ||
1438                                 conf->aggregate_wait_timeout_ms == 0 ||
1439                                 conf->tx_period_ms == 0 ||
1440                                 conf->rx_marker_period_ms == 0 ||
1441                                 conf->update_timeout_ms == 0) {
1442                         RTE_BOND_LOG(ERR, "given mode 4 configuration is invalid");
1443                         return -EINVAL;
1444                 }
1445         }
1446
1447         return 0;
1448 }
1449
1450
1451 int
1452 rte_eth_bond_8023ad_setup(uint16_t port_id,
1453                 struct rte_eth_bond_8023ad_conf *conf)
1454 {
1455         struct rte_eth_dev *bond_dev;
1456         int err;
1457
1458         err = bond_8023ad_setup_validate(port_id, conf);
1459         if (err != 0)
1460                 return err;
1461
1462         bond_dev = &rte_eth_devices[port_id];
1463         bond_mode_8023ad_setup(bond_dev, conf);
1464
1465         return 0;
1466 }
1467
1468
1469
1470
1471
1472 int
1473 rte_eth_bond_8023ad_slave_info(uint16_t port_id, uint16_t slave_id,
1474                 struct rte_eth_bond_8023ad_slave_info *info)
1475 {
1476         struct rte_eth_dev *bond_dev;
1477         struct bond_dev_private *internals;
1478         struct port *port;
1479
1480         if (info == NULL || valid_bonded_port_id(port_id) != 0 ||
1481                         rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1482                 return -EINVAL;
1483
1484         bond_dev = &rte_eth_devices[port_id];
1485
1486         internals = bond_dev->data->dev_private;
1487         if (find_slave_by_id(internals->active_slaves,
1488                         internals->active_slave_count, slave_id) ==
1489                                 internals->active_slave_count)
1490                 return -EINVAL;
1491
1492         port = &bond_mode_8023ad_ports[slave_id];
1493         info->selected = port->selected;
1494
1495         info->actor_state = port->actor_state;
1496         rte_memcpy(&info->actor, &port->actor, sizeof(port->actor));
1497
1498         info->partner_state = port->partner_state;
1499         rte_memcpy(&info->partner, &port->partner, sizeof(port->partner));
1500
1501         info->agg_port_id = port->aggregator_port_id;
1502         return 0;
1503 }
1504
1505 static int
1506 bond_8023ad_ext_validate(uint16_t port_id, uint16_t slave_id)
1507 {
1508         struct rte_eth_dev *bond_dev;
1509         struct bond_dev_private *internals;
1510         struct mode8023ad_private *mode4;
1511
1512         if (rte_eth_bond_mode_get(port_id) != BONDING_MODE_8023AD)
1513                 return -EINVAL;
1514
1515         bond_dev = &rte_eth_devices[port_id];
1516
1517         if (!bond_dev->data->dev_started)
1518                 return -EINVAL;
1519
1520         internals = bond_dev->data->dev_private;
1521         if (find_slave_by_id(internals->active_slaves,
1522                         internals->active_slave_count, slave_id) ==
1523                                 internals->active_slave_count)
1524                 return -EINVAL;
1525
1526         mode4 = &internals->mode4;
1527         if (mode4->slowrx_cb == NULL)
1528                 return -EINVAL;
1529
1530         return 0;
1531 }
1532
1533 int
1534 rte_eth_bond_8023ad_ext_collect(uint16_t port_id, uint16_t slave_id,
1535                                 int enabled)
1536 {
1537         struct port *port;
1538         int res;
1539
1540         res = bond_8023ad_ext_validate(port_id, slave_id);
1541         if (res != 0)
1542                 return res;
1543
1544         port = &bond_mode_8023ad_ports[slave_id];
1545
1546         if (enabled)
1547                 ACTOR_STATE_SET(port, COLLECTING);
1548         else
1549                 ACTOR_STATE_CLR(port, COLLECTING);
1550
1551         return 0;
1552 }
1553
1554 int
1555 rte_eth_bond_8023ad_ext_distrib(uint16_t port_id, uint16_t slave_id,
1556                                 int enabled)
1557 {
1558         struct port *port;
1559         int res;
1560
1561         res = bond_8023ad_ext_validate(port_id, slave_id);
1562         if (res != 0)
1563                 return res;
1564
1565         port = &bond_mode_8023ad_ports[slave_id];
1566
1567         if (enabled)
1568                 ACTOR_STATE_SET(port, DISTRIBUTING);
1569         else
1570                 ACTOR_STATE_CLR(port, DISTRIBUTING);
1571
1572         return 0;
1573 }
1574
1575 int
1576 rte_eth_bond_8023ad_ext_distrib_get(uint16_t port_id, uint16_t slave_id)
1577 {
1578         struct port *port;
1579         int err;
1580
1581         err = bond_8023ad_ext_validate(port_id, slave_id);
1582         if (err != 0)
1583                 return err;
1584
1585         port = &bond_mode_8023ad_ports[slave_id];
1586         return ACTOR_STATE(port, DISTRIBUTING);
1587 }
1588
1589 int
1590 rte_eth_bond_8023ad_ext_collect_get(uint16_t port_id, uint16_t slave_id)
1591 {
1592         struct port *port;
1593         int err;
1594
1595         err = bond_8023ad_ext_validate(port_id, slave_id);
1596         if (err != 0)
1597                 return err;
1598
1599         port = &bond_mode_8023ad_ports[slave_id];
1600         return ACTOR_STATE(port, COLLECTING);
1601 }
1602
1603 int
1604 rte_eth_bond_8023ad_ext_slowtx(uint16_t port_id, uint16_t slave_id,
1605                 struct rte_mbuf *lacp_pkt)
1606 {
1607         struct port *port;
1608         int res;
1609
1610         res = bond_8023ad_ext_validate(port_id, slave_id);
1611         if (res != 0)
1612                 return res;
1613
1614         port = &bond_mode_8023ad_ports[slave_id];
1615
1616         if (rte_pktmbuf_pkt_len(lacp_pkt) < sizeof(struct lacpdu_header))
1617                 return -EINVAL;
1618
1619         struct lacpdu_header *lacp;
1620
1621         /* only enqueue LACPDUs */
1622         lacp = rte_pktmbuf_mtod(lacp_pkt, struct lacpdu_header *);
1623         if (lacp->lacpdu.subtype != SLOW_SUBTYPE_LACP)
1624                 return -EINVAL;
1625
1626         MODE4_DEBUG("sending LACP frame\n");
1627
1628         return rte_ring_enqueue(port->tx_ring, lacp_pkt);
1629 }
1630
1631 static void
1632 bond_mode_8023ad_ext_periodic_cb(void *arg)
1633 {
1634         struct rte_eth_dev *bond_dev = arg;
1635         struct bond_dev_private *internals = bond_dev->data->dev_private;
1636         struct mode8023ad_private *mode4 = &internals->mode4;
1637         struct port *port;
1638         void *pkt = NULL;
1639         uint16_t i, slave_id;
1640
1641         for (i = 0; i < internals->active_slave_count; i++) {
1642                 slave_id = internals->active_slaves[i];
1643                 port = &bond_mode_8023ad_ports[slave_id];
1644
1645                 if (rte_ring_dequeue(port->rx_ring, &pkt) == 0) {
1646                         struct rte_mbuf *lacp_pkt = pkt;
1647                         struct lacpdu_header *lacp;
1648
1649                         lacp = rte_pktmbuf_mtod(lacp_pkt,
1650                                                 struct lacpdu_header *);
1651                         RTE_VERIFY(lacp->lacpdu.subtype == SLOW_SUBTYPE_LACP);
1652
1653                         /* This is LACP frame so pass it to rx callback.
1654                          * Callback is responsible for freeing mbuf.
1655                          */
1656                         mode4->slowrx_cb(slave_id, lacp_pkt);
1657                 }
1658         }
1659
1660         rte_eal_alarm_set(internals->mode4.update_timeout_us,
1661                         bond_mode_8023ad_ext_periodic_cb, arg);
1662 }
1663
1664 int
1665 rte_eth_bond_8023ad_dedicated_queues_enable(uint16_t port)
1666 {
1667         int retval = 0;
1668         struct rte_eth_dev *dev = &rte_eth_devices[port];
1669         struct bond_dev_private *internals = (struct bond_dev_private *)
1670                 dev->data->dev_private;
1671
1672         if (check_for_bonded_ethdev(dev) != 0)
1673                 return -1;
1674
1675         if (bond_8023ad_slow_pkt_hw_filter_supported(port) != 0)
1676                 return -1;
1677
1678         /* Device must be stopped to set up slow queue */
1679         if (dev->data->dev_started)
1680                 return -1;
1681
1682         internals->mode4.dedicated_queues.enabled = 1;
1683
1684         bond_ethdev_mode_set(dev, internals->mode);
1685         return retval;
1686 }
1687
1688 int
1689 rte_eth_bond_8023ad_dedicated_queues_disable(uint16_t port)
1690 {
1691         int retval = 0;
1692         struct rte_eth_dev *dev = &rte_eth_devices[port];
1693         struct bond_dev_private *internals = (struct bond_dev_private *)
1694                 dev->data->dev_private;
1695
1696         if (check_for_bonded_ethdev(dev) != 0)
1697                 return -1;
1698
1699         /* Device must be stopped to set up slow queue */
1700         if (dev->data->dev_started)
1701                 return -1;
1702
1703         internals->mode4.dedicated_queues.enabled = 0;
1704
1705         bond_ethdev_mode_set(dev, internals->mode);
1706
1707         return retval;
1708 }