drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .lif_index = lif->index,
  29                         .type = q->type,
  30                         .index = q->index,
  31                         .oper = IONIC_Q_ENABLE,
  32                 },
  33         };
  34
  35         if (qcq->flags & IONIC_QCQ_F_INTR) {
  36                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  37                         IONIC_INTR_MASK_CLEAR);
  38         }
  39
  40         return ionic_adminq_post_wait(lif, &ctx);
  41 }
  42
  43 int
  44 ionic_qcq_disable(struct ionic_qcq *qcq)
  45 {
  46         struct ionic_queue *q = &qcq->q;
  47         struct ionic_lif *lif = q->lif;
  48         struct ionic_dev *idev = &lif->adapter->idev;
  49         struct ionic_admin_ctx ctx = {
  50                 .pending_work = true,
  51                 .cmd.q_control = {
  52                         .opcode = IONIC_CMD_Q_CONTROL,
  53                         .lif_index = lif->index,
  54                         .type = q->type,
  55                         .index = q->index,
  56                         .oper = IONIC_Q_DISABLE,
  57                 },
  58         };
  59
  60         if (qcq->flags & IONIC_QCQ_F_INTR) {
  61                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  62                         IONIC_INTR_MASK_SET);
  63         }
  64
  65         return ionic_adminq_post_wait(lif, &ctx);
  66 }
  67
  68 int
  69 ionic_lif_stop(struct ionic_lif *lif __rte_unused)
  70 {
  71         /* Carrier OFF here */
  72
  73         return 0;
  74 }
  75
  76 void
  77 ionic_lif_reset(struct ionic_lif *lif)
  78 {
  79         struct ionic_dev *idev = &lif->adapter->idev;
  80
  81         IONIC_PRINT_CALL();
  82
  83         ionic_dev_cmd_lif_reset(idev, lif->index);
  84         ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  85 }
  86
  87 static void
  88 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  89 {
  90         struct ionic_lif_stats *ls = &lif->info->stats;
  91         uint32_t i;
  92         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  93                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  94         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  95                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  96
  97         memset(stats, 0, sizeof(*stats));
  98
  99         if (ls == NULL) {
 100                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 101                         lif->port_id);
 102                 return;
 103         }
 104
 105         /* RX */
 106
 107         stats->ipackets = ls->rx_ucast_packets +
 108                 ls->rx_mcast_packets +
 109                 ls->rx_bcast_packets;
 110
 111         stats->ibytes = ls->rx_ucast_bytes +
 112                 ls->rx_mcast_bytes +
 113                 ls->rx_bcast_bytes;
 114
 115         for (i = 0; i < lif->nrxqcqs; i++) {
 116                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 117                 stats->imissed +=
 118                         rx_stats->no_cb_arg +
 119                         rx_stats->bad_cq_status +
 120                         rx_stats->no_room +
 121                         rx_stats->bad_len;
 122         }
 123
 124         stats->imissed +=
 125                 ls->rx_ucast_drop_packets +
 126                 ls->rx_mcast_drop_packets +
 127                 ls->rx_bcast_drop_packets;
 128
 129         stats->imissed +=
 130                 ls->rx_queue_empty +
 131                 ls->rx_dma_error +
 132                 ls->rx_queue_disabled +
 133                 ls->rx_desc_fetch_error +
 134                 ls->rx_desc_data_error;
 135
 136         for (i = 0; i < num_rx_q_counters; i++) {
 137                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 138                 stats->q_ipackets[i] = rx_stats->packets;
 139                 stats->q_ibytes[i] = rx_stats->bytes;
 140                 stats->q_errors[i] =
 141                         rx_stats->no_cb_arg +
 142                         rx_stats->bad_cq_status +
 143                         rx_stats->no_room +
 144                         rx_stats->bad_len;
 145         }
 146
 147         /* TX */
 148
 149         stats->opackets = ls->tx_ucast_packets +
 150                 ls->tx_mcast_packets +
 151                 ls->tx_bcast_packets;
 152
 153         stats->obytes = ls->tx_ucast_bytes +
 154                 ls->tx_mcast_bytes +
 155                 ls->tx_bcast_bytes;
 156
 157         for (i = 0; i < lif->ntxqcqs; i++) {
 158                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 159                 stats->oerrors += tx_stats->drop;
 160         }
 161
 162         stats->oerrors +=
 163                 ls->tx_ucast_drop_packets +
 164                 ls->tx_mcast_drop_packets +
 165                 ls->tx_bcast_drop_packets;
 166
 167         stats->oerrors +=
 168                 ls->tx_dma_error +
 169                 ls->tx_queue_disabled +
 170                 ls->tx_desc_fetch_error +
 171                 ls->tx_desc_data_error;
 172
 173         for (i = 0; i < num_tx_q_counters; i++) {
 174                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 175                 stats->q_opackets[i] = tx_stats->packets;
 176                 stats->q_obytes[i] = tx_stats->bytes;
 177         }
 178 }
 179
 180 void
 181 ionic_lif_get_stats(const struct ionic_lif *lif,
 182                 struct rte_eth_stats *stats)
 183 {
 184         ionic_lif_get_abs_stats(lif, stats);
 185
 186         stats->ipackets  -= lif->stats_base.ipackets;
 187         stats->opackets  -= lif->stats_base.opackets;
 188         stats->ibytes    -= lif->stats_base.ibytes;
 189         stats->obytes    -= lif->stats_base.obytes;
 190         stats->imissed   -= lif->stats_base.imissed;
 191         stats->ierrors   -= lif->stats_base.ierrors;
 192         stats->oerrors   -= lif->stats_base.oerrors;
 193         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 194 }
 195
 196 void
 197 ionic_lif_reset_stats(struct ionic_lif *lif)
 198 {
 199         uint32_t i;
 200
 201         for (i = 0; i < lif->nrxqcqs; i++) {
 202                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 203                         sizeof(struct ionic_rx_stats));
 204                 memset(&lif->txqcqs[i]->stats.tx, 0,
 205                         sizeof(struct ionic_tx_stats));
 206         }
 207
 208         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 209 }
 210
 211 void
 212 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 213 {
 214         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 215         uint64_t *stats64 = (uint64_t *)stats;
 216         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 217         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 218
 219         for (i = 0; i < count; i++)
 220                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 221 }
 222
 223 void
 224 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 225 {
 226         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 227         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 228         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 229
 230         for (i = 0; i < count; i++)
 231                 lif_stats64_base[i] = lif_stats64[i];
 232 }
 233
 234 static int
 235 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 236 {
 237         struct ionic_admin_ctx ctx = {
 238                 .pending_work = true,
 239                 .cmd.rx_filter_add = {
 240                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 241                         .match = IONIC_RX_FILTER_MATCH_MAC,
 242                 },
 243         };
 244         int err;
 245
 246         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 247
 248         err = ionic_adminq_post_wait(lif, &ctx);
 249         if (err)
 250                 return err;
 251
 252         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 253                 ctx.comp.rx_filter_add.filter_id);
 254
 255         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 256 }
 257
 258 static int
 259 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 260 {
 261         struct ionic_admin_ctx ctx = {
 262                 .pending_work = true,
 263                 .cmd.rx_filter_del = {
 264                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 265                 },
 266         };
 267         struct ionic_rx_filter *f;
 268         int err;
 269
 270         IONIC_PRINT_CALL();
 271
 272         rte_spinlock_lock(&lif->rx_filters.lock);
 273
 274         f = ionic_rx_filter_by_addr(lif, addr);
 275         if (!f) {
 276                 rte_spinlock_unlock(&lif->rx_filters.lock);
 277                 return -ENOENT;
 278         }
 279
 280         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 281         ionic_rx_filter_free(f);
 282
 283         rte_spinlock_unlock(&lif->rx_filters.lock);
 284
 285         err = ionic_adminq_post_wait(lif, &ctx);
 286         if (err)
 287                 return err;
 288
 289         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 290                 ctx.cmd.rx_filter_del.filter_id);
 291
 292         return 0;
 293 }
 294
 295 int
 296 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 297                 struct rte_ether_addr *mac_addr,
 298                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 299 {
 300         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 301
 302         IONIC_PRINT_CALL();
 303
 304         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 305 }
 306
 307 void
 308 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index __rte_unused)
 309 {
 310         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 311         struct ionic_adapter *adapter = lif->adapter;
 312
 313         IONIC_PRINT_CALL();
 314
 315         if (index >= adapter->max_mac_addrs) {
 316                 IONIC_PRINT(WARNING,
 317                         "Index %u is above MAC filter limit %u",
 318                         index, adapter->max_mac_addrs);
 319                 return;
 320         }
 321
 322         if (!rte_is_valid_assigned_ether_addr(&eth_dev->data->mac_addrs[index]))
 323                 return;
 324
 325         ionic_lif_addr_del(lif, (const uint8_t *)
 326                 &eth_dev->data->mac_addrs[index]);
 327 }
 328
 329 int
 330 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 331 {
 332         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 333
 334         IONIC_PRINT_CALL();
 335
 336         if (mac_addr == NULL) {
 337                 IONIC_PRINT(NOTICE, "New mac is null");
 338                 return -1;
 339         }
 340
 341         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 342                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 343                         lif->mac_addr);
 344                 ionic_lif_addr_del(lif, lif->mac_addr);
 345                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 346         }
 347
 348         IONIC_PRINT(INFO, "Updating mac addr");
 349
 350         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 351
 352         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 353 }
 354
 355 static int
 356 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 357 {
 358         struct ionic_admin_ctx ctx = {
 359                 .pending_work = true,
 360                 .cmd.rx_filter_add = {
 361                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 362                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 363                         .vlan.vlan = vid,
 364                 },
 365         };
 366         int err;
 367
 368         err = ionic_adminq_post_wait(lif, &ctx);
 369         if (err)
 370                 return err;
 371
 372         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 373                 ctx.comp.rx_filter_add.filter_id);
 374
 375         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 376 }
 377
 378 static int
 379 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 380 {
 381         struct ionic_admin_ctx ctx = {
 382                 .pending_work = true,
 383                 .cmd.rx_filter_del = {
 384                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 385                 },
 386         };
 387         struct ionic_rx_filter *f;
 388         int err;
 389
 390         IONIC_PRINT_CALL();
 391
 392         rte_spinlock_lock(&lif->rx_filters.lock);
 393
 394         f = ionic_rx_filter_by_vlan(lif, vid);
 395         if (!f) {
 396                 rte_spinlock_unlock(&lif->rx_filters.lock);
 397                 return -ENOENT;
 398         }
 399
 400         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 401         ionic_rx_filter_free(f);
 402         rte_spinlock_unlock(&lif->rx_filters.lock);
 403
 404         err = ionic_adminq_post_wait(lif, &ctx);
 405         if (err)
 406                 return err;
 407
 408         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 409                 ctx.cmd.rx_filter_del.filter_id);
 410
 411         return 0;
 412 }
 413
 414 int
 415 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 416                 int on)
 417 {
 418         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 419         int err;
 420
 421         if (on)
 422                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 423         else
 424                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 425
 426         return err;
 427 }
 428
 429 static void
 430 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 431 {
 432         struct ionic_admin_ctx ctx = {
 433                 .pending_work = true,
 434                 .cmd.rx_mode_set = {
 435                         .opcode = IONIC_CMD_RX_MODE_SET,
 436                         .lif_index = lif->index,
 437                         .rx_mode = rx_mode,
 438                 },
 439         };
 440         int err;
 441
 442         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 443                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 444         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 445                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 446         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 447                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 448         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 449                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 450         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 451                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 452
 453         err = ionic_adminq_post_wait(lif, &ctx);
 454         if (err)
 455                 IONIC_PRINT(ERR, "Failure setting RX mode");
 456 }
 457
 458 static void
 459 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 460 {
 461         if (lif->rx_mode != rx_mode) {
 462                 lif->rx_mode = rx_mode;
 463                 ionic_lif_rx_mode(lif, rx_mode);
 464         }
 465 }
 466
 467 int
 468 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 469 {
 470         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 471         uint32_t rx_mode = lif->rx_mode;
 472
 473         IONIC_PRINT_CALL();
 474
 475         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 476
 477         ionic_set_rx_mode(lif, rx_mode);
 478
 479         return 0;
 480 }
 481
 482 int
 483 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 484 {
 485         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 486         uint32_t rx_mode = lif->rx_mode;
 487
 488         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 489
 490         ionic_set_rx_mode(lif, rx_mode);
 491
 492         return 0;
 493 }
 494
 495 int
 496 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 497 {
 498         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 499         uint32_t rx_mode = lif->rx_mode;
 500
 501         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 502
 503         ionic_set_rx_mode(lif, rx_mode);
 504
 505         return 0;
 506 }
 507
 508 int
 509 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 510 {
 511         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 512         uint32_t rx_mode = lif->rx_mode;
 513
 514         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 515
 516         ionic_set_rx_mode(lif, rx_mode);
 517
 518         return 0;
 519 }
 520
 521 int
 522 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 523 {
 524         struct ionic_admin_ctx ctx = {
 525                 .pending_work = true,
 526                 .cmd.lif_setattr = {
 527                         .opcode = IONIC_CMD_LIF_SETATTR,
 528                         .index = lif->index,
 529                         .attr = IONIC_LIF_ATTR_MTU,
 530                         .mtu = new_mtu,
 531                 },
 532         };
 533         int err;
 534
 535         err = ionic_adminq_post_wait(lif, &ctx);
 536         if (err)
 537                 return err;
 538
 539         lif->mtu = new_mtu;
 540
 541         return 0;
 542 }
 543
 544 int
 545 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 546 {
 547         struct ionic_adapter *adapter = lif->adapter;
 548         struct ionic_dev *idev = &adapter->idev;
 549         unsigned long index;
 550
 551         /*
 552          * Note: interrupt handler is called for index = 0 only
 553          * (we use interrupts for the notifyq only anyway,
 554          * which hash index = 0)
 555          */
 556
 557         for (index = 0; index < adapter->nintrs; index++)
 558                 if (!adapter->intrs[index])
 559                         break;
 560
 561         if (index == adapter->nintrs)
 562                 return -ENOSPC;
 563
 564         adapter->intrs[index] = true;
 565
 566         ionic_intr_init(idev, intr, index);
 567
 568         return 0;
 569 }
 570
 571 void
 572 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 573 {
 574         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 575                 lif->adapter->intrs[intr->index] = false;
 576 }
 577
 578 static int
 579 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 580                 uint32_t index,
 581                 const char *base, uint32_t flags,
 582                 uint32_t num_descs,
 583                 uint32_t desc_size,
 584                 uint32_t cq_desc_size,
 585                 uint32_t sg_desc_size,
 586                 uint32_t pid, struct ionic_qcq **qcq)
 587 {
 588         struct ionic_dev *idev = &lif->adapter->idev;
 589         struct ionic_qcq *new;
 590         uint32_t q_size, cq_size, sg_size, total_size;
 591         void *q_base, *cq_base, *sg_base;
 592         rte_iova_t q_base_pa = 0;
 593         rte_iova_t cq_base_pa = 0;
 594         rte_iova_t sg_base_pa = 0;
 595         uint32_t socket_id = rte_socket_id();
 596         int err;
 597
 598         *qcq = NULL;
 599
 600         q_size  = num_descs * desc_size;
 601         cq_size = num_descs * cq_desc_size;
 602         sg_size = num_descs * sg_desc_size;
 603
 604         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 605                 RTE_ALIGN(cq_size, PAGE_SIZE);
 606         /*
 607          * Note: aligning q_size/cq_size is not enough due to cq_base address
 608          * aligning as q_base could be not aligned to the page.
 609          * Adding PAGE_SIZE.
 610          */
 611         total_size += PAGE_SIZE;
 612
 613         if (flags & IONIC_QCQ_F_SG) {
 614                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 615                 total_size += PAGE_SIZE;
 616         }
 617
 618         new = rte_zmalloc("ionic", sizeof(*new), 0);
 619         if (!new) {
 620                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 621                 return -ENOMEM;
 622         }
 623
 624         new->lif = lif;
 625         new->flags = flags;
 626
 627         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 628         if (!new->q.info) {
 629                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 630                 return -ENOMEM;
 631         }
 632
 633         new->q.type = type;
 634
 635         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 636                 desc_size, sg_desc_size, pid);
 637         if (err) {
 638                 IONIC_PRINT(ERR, "Queue initialization failed");
 639                 return err;
 640         }
 641
 642         if (flags & IONIC_QCQ_F_INTR) {
 643                 err = ionic_intr_alloc(lif, &new->intr);
 644                 if (err)
 645                         return err;
 646
 647                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 648                         IONIC_INTR_MASK_SET);
 649         } else {
 650                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 651         }
 652
 653         err = ionic_cq_init(lif, &new->cq, &new->intr,
 654                 num_descs, cq_desc_size);
 655         if (err) {
 656                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 657                 goto err_out_free_intr;
 658         }
 659
 660         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 661                 base /* name */, index /* queue_idx */,
 662                 total_size, IONIC_ALIGN, socket_id);
 663
 664         if (!new->base_z) {
 665                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 666                 err = -ENOMEM;
 667                 goto err_out_free_intr;
 668         }
 669
 670         new->base = new->base_z->addr;
 671         new->base_pa = new->base_z->iova;
 672         new->total_size = total_size;
 673
 674         q_base = new->base;
 675         q_base_pa = new->base_pa;
 676
 677         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 678         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 679
 680         if (flags & IONIC_QCQ_F_SG) {
 681                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 682                         PAGE_SIZE);
 683                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 684                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 685         }
 686
 687         IONIC_PRINT(DEBUG, "Q-Base-PA = %ju CQ-Base-PA = %ju "
 688                 "SG-base-PA = %ju",
 689                 q_base_pa, cq_base_pa, sg_base_pa);
 690
 691         ionic_q_map(&new->q, q_base, q_base_pa);
 692         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 693         ionic_cq_bind(&new->cq, &new->q);
 694
 695         *qcq = new;
 696
 697         return 0;
 698
 699 err_out_free_intr:
 700         if (flags & IONIC_QCQ_F_INTR)
 701                 ionic_intr_free(lif, &new->intr);
 702
 703         return err;
 704 }
 705
 706 void
 707 ionic_qcq_free(struct ionic_qcq *qcq)
 708 {
 709         if (qcq->base_z) {
 710                 qcq->base = NULL;
 711                 qcq->base_pa = 0;
 712                 rte_memzone_free(qcq->base_z);
 713                 qcq->base_z = NULL;
 714         }
 715
 716         if (qcq->q.info) {
 717                 rte_free(qcq->q.info);
 718                 qcq->q.info = NULL;
 719         }
 720
 721         rte_free(qcq);
 722 }
 723
 724 int
 725 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 726                 struct ionic_qcq **qcq)
 727 {
 728         uint32_t flags;
 729         int err = -ENOMEM;
 730
 731         flags = IONIC_QCQ_F_SG;
 732         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 733                 nrxq_descs,
 734                 sizeof(struct ionic_rxq_desc),
 735                 sizeof(struct ionic_rxq_comp),
 736                 sizeof(struct ionic_rxq_sg_desc),
 737                 lif->kern_pid, &lif->rxqcqs[index]);
 738         if (err)
 739                 return err;
 740
 741         *qcq = lif->rxqcqs[index];
 742
 743         return 0;
 744 }
 745
 746 int
 747 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 748                 struct ionic_qcq **qcq)
 749 {
 750         uint32_t flags;
 751         int err = -ENOMEM;
 752
 753         flags = IONIC_QCQ_F_SG;
 754         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 755                 ntxq_descs,
 756                 sizeof(struct ionic_txq_desc),
 757                 sizeof(struct ionic_txq_comp),
 758                 sizeof(struct ionic_txq_sg_desc),
 759                 lif->kern_pid, &lif->txqcqs[index]);
 760         if (err)
 761                 return err;
 762
 763         *qcq = lif->txqcqs[index];
 764
 765         return 0;
 766 }
 767
 768 static int
 769 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 770 {
 771         uint32_t flags;
 772         int err = -ENOMEM;
 773
 774         flags = 0;
 775         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 776                 IONIC_ADMINQ_LENGTH,
 777                 sizeof(struct ionic_admin_cmd),
 778                 sizeof(struct ionic_admin_comp),
 779                 0,
 780                 lif->kern_pid, &lif->adminqcq);
 781         if (err)
 782                 return err;
 783
 784         return 0;
 785 }
 786
 787 static int
 788 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 789 {
 790         uint32_t flags;
 791         int err = -ENOMEM;
 792
 793         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 794
 795         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 796                 flags,
 797                 IONIC_NOTIFYQ_LENGTH,
 798                 sizeof(struct ionic_notifyq_cmd),
 799                 sizeof(union ionic_notifyq_comp),
 800                 0,
 801                 lif->kern_pid, &lif->notifyqcq);
 802         if (err)
 803                 return err;
 804
 805         return 0;
 806 }
 807
 808 static void *
 809 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 810 {
 811         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 812
 813         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 814                 return NULL;
 815
 816         return (void *)&vaddr[page_num << PAGE_SHIFT];
 817 }
 818
 819 int
 820 ionic_lif_alloc(struct ionic_lif *lif)
 821 {
 822         struct ionic_adapter *adapter = lif->adapter;
 823         uint32_t socket_id = rte_socket_id();
 824         int dbpage_num;
 825         int err;
 826
 827         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
 828
 829         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 830
 831         rte_spinlock_init(&lif->adminq_lock);
 832         rte_spinlock_init(&lif->adminq_service_lock);
 833
 834         lif->kern_pid = 0;
 835
 836         dbpage_num = ionic_db_page_num(lif, 0);
 837
 838         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, dbpage_num);
 839         if (!lif->kern_dbpage) {
 840                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 841                 return -ENOMEM;
 842         }
 843
 844         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 845                 adapter->max_ntxqs_per_lif, 0);
 846
 847         if (!lif->txqcqs) {
 848                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 849                 return -ENOMEM;
 850         }
 851
 852         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 853                 adapter->max_nrxqs_per_lif, 0);
 854
 855         if (!lif->rxqcqs) {
 856                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 857                 return -ENOMEM;
 858         }
 859
 860         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 861
 862         err = ionic_notify_qcq_alloc(lif);
 863         if (err) {
 864                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 865                 return err;
 866         }
 867
 868         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 869
 870         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 871
 872         err = ionic_admin_qcq_alloc(lif);
 873         if (err) {
 874                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 875                 return err;
 876         }
 877
 878         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 879
 880         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 881
 882         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 883                 "lif_info", 0 /* queue_idx*/,
 884                 lif->info_sz, IONIC_ALIGN, socket_id);
 885         if (!lif->info_z) {
 886                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 887                 return -ENOMEM;
 888         }
 889
 890         lif->info = lif->info_z->addr;
 891         lif->info_pa = lif->info_z->iova;
 892
 893         return 0;
 894 }
 895
 896 void
 897 ionic_lif_free(struct ionic_lif *lif)
 898 {
 899         if (lif->notifyqcq) {
 900                 ionic_qcq_free(lif->notifyqcq);
 901                 lif->notifyqcq = NULL;
 902         }
 903
 904         if (lif->adminqcq) {
 905                 ionic_qcq_free(lif->adminqcq);
 906                 lif->adminqcq = NULL;
 907         }
 908
 909         if (lif->txqcqs) {
 910                 rte_free(lif->txqcqs);
 911                 lif->txqcqs = NULL;
 912         }
 913
 914         if (lif->rxqcqs) {
 915                 rte_free(lif->rxqcqs);
 916                 lif->rxqcqs = NULL;
 917         }
 918
 919         if (lif->info) {
 920                 rte_memzone_free(lif->info_z);
 921                 lif->info = NULL;
 922         }
 923 }
 924
 925 int
 926 ionic_lif_rss_config(struct ionic_lif *lif,
 927                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 928 {
 929         struct ionic_admin_ctx ctx = {
 930                 .pending_work = true,
 931                 .cmd.lif_setattr = {
 932                         .opcode = IONIC_CMD_LIF_SETATTR,
 933                         .attr = IONIC_LIF_ATTR_RSS,
 934                         .rss.types = types,
 935                         .rss.addr = lif->rss_ind_tbl_pa,
 936                 },
 937         };
 938         unsigned int i;
 939
 940         IONIC_PRINT_CALL();
 941
 942         lif->rss_types = types;
 943
 944         if (key)
 945                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 946
 947         if (indir)
 948                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 949                         lif->rss_ind_tbl[i] = indir[i];
 950
 951         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 952                IONIC_RSS_HASH_KEY_SIZE);
 953
 954         return ionic_adminq_post_wait(lif, &ctx);
 955 }
 956
 957 static int
 958 ionic_lif_rss_setup(struct ionic_lif *lif)
 959 {
 960         size_t tbl_size = sizeof(*lif->rss_ind_tbl) *
 961                 lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 962         static const uint8_t toeplitz_symmetric_key[] = {
 963                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 964                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 965                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 966                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 967                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 968         };
 969         uint32_t socket_id = rte_socket_id();
 970         uint32_t i;
 971         int err;
 972
 973         IONIC_PRINT_CALL();
 974
 975         lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 976                 "rss_ind_tbl",
 977                 0 /* queue_idx*/, tbl_size, IONIC_ALIGN, socket_id);
 978
 979         if (!lif->rss_ind_tbl_z) {
 980                 IONIC_PRINT(ERR, "OOM");
 981                 return -ENOMEM;
 982         }
 983
 984         lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 985         lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
 986
 987         /* Fill indirection table with 'default' values */
 988         for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 989                 lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
 990
 991         err = ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
 992                 toeplitz_symmetric_key, NULL);
 993         if (err)
 994                 return err;
 995
 996         return 0;
 997 }
 998
 999 static void
1000 ionic_lif_rss_teardown(struct ionic_lif *lif)
1001 {
1002         if (!lif->rss_ind_tbl)
1003                 return;
1004
1005         if (lif->rss_ind_tbl_z) {
1006                 /* Disable RSS on the NIC */
1007                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1008
1009                 lif->rss_ind_tbl = NULL;
1010                 lif->rss_ind_tbl_pa = 0;
1011                 rte_memzone_free(lif->rss_ind_tbl_z);
1012                 lif->rss_ind_tbl_z = NULL;
1013         }
1014 }
1015
1016 static void
1017 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1018 {
1019         struct ionic_dev *idev = &lif->adapter->idev;
1020
1021         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1022                 return;
1023
1024         if (qcq->flags & IONIC_QCQ_F_INTR)
1025                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1026                         IONIC_INTR_MASK_SET);
1027
1028         qcq->flags &= ~IONIC_QCQ_F_INITED;
1029 }
1030
1031 void
1032 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1033 {
1034         ionic_lif_qcq_deinit(qcq->lif, qcq);
1035 }
1036
1037 void
1038 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1039 {
1040         ionic_lif_qcq_deinit(qcq->lif, qcq);
1041 }
1042
1043 bool
1044 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1045                 void *cb_arg __rte_unused)
1046 {
1047         struct ionic_admin_comp *cq_desc_base = cq->base;
1048         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1049
1050         if (!color_match(cq_desc->color, cq->done_color))
1051                 return false;
1052
1053         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1054
1055         return true;
1056 }
1057
1058 /* This acts like ionic_napi */
1059 int
1060 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1061                 void *cb_arg)
1062 {
1063         struct ionic_cq *cq = &qcq->cq;
1064         uint32_t work_done;
1065
1066         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1067
1068         return work_done;
1069 }
1070
1071 static void
1072 ionic_link_status_check(struct ionic_lif *lif)
1073 {
1074         struct ionic_adapter *adapter = lif->adapter;
1075         bool link_up;
1076
1077         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1078
1079         if (!lif->info)
1080                 return;
1081
1082         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1083
1084         if ((link_up  && adapter->link_up) ||
1085             (!link_up && !adapter->link_up))
1086                 return;
1087
1088         if (link_up) {
1089                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1090                         lif->info->status.link_speed);
1091                 adapter->link_speed = lif->info->status.link_speed;
1092         } else {
1093                 IONIC_PRINT(DEBUG, "Link down");
1094         }
1095
1096         adapter->link_up = link_up;
1097 }
1098
1099 static bool
1100 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1101 {
1102         union ionic_notifyq_comp *cq_desc_base = cq->base;
1103         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1104         struct ionic_lif *lif = cb_arg;
1105
1106         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1107                 cq_desc->event.eid, cq_desc->event.ecode);
1108
1109         /* Have we run out of new completions to process? */
1110         if (!(cq_desc->event.eid > lif->last_eid))
1111                 return false;
1112
1113         lif->last_eid = cq_desc->event.eid;
1114
1115         switch (cq_desc->event.ecode) {
1116         case IONIC_EVENT_LINK_CHANGE:
1117                 IONIC_PRINT(DEBUG,
1118                         "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
1119                         cq_desc->event.eid,
1120                         cq_desc->link_change.link_status,
1121                         cq_desc->link_change.link_speed);
1122
1123                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1124
1125                 break;
1126         default:
1127                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1128                         cq_desc->event.ecode, cq_desc->event.eid);
1129                 break;
1130         }
1131
1132         return true;
1133 }
1134
1135 int
1136 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1137 {
1138         struct ionic_dev *idev = &lif->adapter->idev;
1139         struct ionic_qcq *qcq = lif->notifyqcq;
1140         uint32_t work_done;
1141
1142         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1143                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1144                 return -1;
1145         }
1146
1147         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1148                 IONIC_INTR_MASK_SET);
1149
1150         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1151
1152         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1153                 ionic_link_status_check(lif);
1154
1155         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1156                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1157
1158         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1159                 IONIC_INTR_MASK_CLEAR);
1160
1161         return 0;
1162 }
1163
1164 static int
1165 ionic_lif_adminq_init(struct ionic_lif *lif)
1166 {
1167         struct ionic_dev *idev = &lif->adapter->idev;
1168         struct ionic_qcq *qcq = lif->adminqcq;
1169         struct ionic_queue *q = &qcq->q;
1170         struct ionic_q_init_comp comp;
1171         int err;
1172
1173         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
1174         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1175         if (err)
1176                 return err;
1177
1178         ionic_dev_cmd_comp(idev, &comp);
1179
1180         q->hw_type = comp.hw_type;
1181         q->hw_index = comp.hw_index;
1182         q->db = ionic_db_map(lif, q);
1183
1184         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1185         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1186         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1187
1188         if (qcq->flags & IONIC_QCQ_F_INTR)
1189                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1190                         IONIC_INTR_MASK_CLEAR);
1191
1192         qcq->flags |= IONIC_QCQ_F_INITED;
1193
1194         return 0;
1195 }
1196
1197 static int
1198 ionic_lif_notifyq_init(struct ionic_lif *lif)
1199 {
1200         struct ionic_dev *idev = &lif->adapter->idev;
1201         struct ionic_qcq *qcq = lif->notifyqcq;
1202         struct ionic_queue *q = &qcq->q;
1203         int err;
1204
1205         struct ionic_admin_ctx ctx = {
1206                 .pending_work = true,
1207                 .cmd.q_init = {
1208                         .opcode = IONIC_CMD_Q_INIT,
1209                         .lif_index = lif->index,
1210                         .type = q->type,
1211                         .index = q->index,
1212                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1213                         .intr_index = qcq->intr.index,
1214                         .pid = q->pid,
1215                         .ring_size = rte_log2_u32(q->num_descs),
1216                         .ring_base = q->base_pa,
1217                 }
1218         };
1219
1220         IONIC_PRINT(DEBUG, "notifyq_init.pid %d", ctx.cmd.q_init.pid);
1221         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1222                 ctx.cmd.q_init.index);
1223         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1224                 ctx.cmd.q_init.ring_base);
1225         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1226                 ctx.cmd.q_init.ring_size);
1227
1228         err = ionic_adminq_post_wait(lif, &ctx);
1229         if (err)
1230                 return err;
1231
1232         q->hw_type = ctx.comp.q_init.hw_type;
1233         q->hw_index = ctx.comp.q_init.hw_index;
1234         q->db = NULL;
1235
1236         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1237         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1238         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1239
1240         if (qcq->flags & IONIC_QCQ_F_INTR)
1241                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1242                         IONIC_INTR_MASK_CLEAR);
1243
1244         qcq->flags |= IONIC_QCQ_F_INITED;
1245
1246         return 0;
1247 }
1248
1249 int
1250 ionic_lif_set_features(struct ionic_lif *lif)
1251 {
1252         struct ionic_admin_ctx ctx = {
1253                 .pending_work = true,
1254                 .cmd.lif_setattr = {
1255                         .opcode = IONIC_CMD_LIF_SETATTR,
1256                         .index = lif->index,
1257                         .attr = IONIC_LIF_ATTR_FEATURES,
1258                         .features = lif->features,
1259                 },
1260         };
1261         int err;
1262
1263         err = ionic_adminq_post_wait(lif, &ctx);
1264         if (err)
1265                 return err;
1266
1267         lif->hw_features = (ctx.cmd.lif_setattr.features &
1268                 ctx.comp.lif_setattr.features);
1269
1270         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1271                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1272         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1273                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1274         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1275                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1276         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1277                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1278         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1279                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1280         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1281                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1282         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1283                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1284         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1285                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1286         if (lif->hw_features & IONIC_ETH_HW_TSO)
1287                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1288         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1289                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1290         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1291                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1292         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1293                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1294         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1295                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1296         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1297                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1298         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1299                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1300         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1301                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1302         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1303                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1304
1305         return 0;
1306 }
1307
1308 int
1309 ionic_lif_txq_init(struct ionic_qcq *qcq)
1310 {
1311         struct ionic_queue *q = &qcq->q;
1312         struct ionic_lif *lif = qcq->lif;
1313         struct ionic_cq *cq = &qcq->cq;
1314         struct ionic_admin_ctx ctx = {
1315                 .pending_work = true,
1316                 .cmd.q_init = {
1317                         .opcode = IONIC_CMD_Q_INIT,
1318                         .lif_index = lif->index,
1319                         .type = q->type,
1320                         .index = q->index,
1321                         .flags = IONIC_QINIT_F_SG,
1322                         .intr_index = cq->bound_intr->index,
1323                         .pid = q->pid,
1324                         .ring_size = rte_log2_u32(q->num_descs),
1325                         .ring_base = q->base_pa,
1326                         .cq_ring_base = cq->base_pa,
1327                         .sg_ring_base = q->sg_base_pa,
1328                 },
1329         };
1330         int err;
1331
1332         IONIC_PRINT(DEBUG, "txq_init.pid %d", ctx.cmd.q_init.pid);
1333         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1334         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1335                 ctx.cmd.q_init.ring_base);
1336         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1337                 ctx.cmd.q_init.ring_size);
1338
1339         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1340         if (err)
1341                 return err;
1342
1343         q->hw_type = ctx.comp.q_init.hw_type;
1344         q->hw_index = ctx.comp.q_init.hw_index;
1345         q->db = ionic_db_map(lif, q);
1346
1347         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1348         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1349         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1350
1351         qcq->flags |= IONIC_QCQ_F_INITED;
1352
1353         return 0;
1354 }
1355
1356 int
1357 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1358 {
1359         struct ionic_queue *q = &qcq->q;
1360         struct ionic_lif *lif = qcq->lif;
1361         struct ionic_cq *cq = &qcq->cq;
1362         struct ionic_admin_ctx ctx = {
1363                 .pending_work = true,
1364                 .cmd.q_init = {
1365                         .opcode = IONIC_CMD_Q_INIT,
1366                         .lif_index = lif->index,
1367                         .type = q->type,
1368                         .index = q->index,
1369                         .flags = IONIC_QINIT_F_SG,
1370                         .intr_index = cq->bound_intr->index,
1371                         .pid = q->pid,
1372                         .ring_size = rte_log2_u32(q->num_descs),
1373                         .ring_base = q->base_pa,
1374                         .cq_ring_base = cq->base_pa,
1375                         .sg_ring_base = q->sg_base_pa,
1376                 },
1377         };
1378         int err;
1379
1380         IONIC_PRINT(DEBUG, "rxq_init.pid %d", ctx.cmd.q_init.pid);
1381         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1382         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1383                 ctx.cmd.q_init.ring_base);
1384         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1385                 ctx.cmd.q_init.ring_size);
1386
1387         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1388         if (err)
1389                 return err;
1390
1391         q->hw_type = ctx.comp.q_init.hw_type;
1392         q->hw_index = ctx.comp.q_init.hw_index;
1393         q->db = ionic_db_map(lif, q);
1394
1395         qcq->flags |= IONIC_QCQ_F_INITED;
1396
1397         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1398         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1399         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1400
1401         return 0;
1402 }
1403
1404 static int
1405 ionic_station_set(struct ionic_lif *lif)
1406 {
1407         struct ionic_admin_ctx ctx = {
1408                 .pending_work = true,
1409                 .cmd.lif_getattr = {
1410                         .opcode = IONIC_CMD_LIF_GETATTR,
1411                         .index = lif->index,
1412                         .attr = IONIC_LIF_ATTR_MAC,
1413                 },
1414         };
1415         int err;
1416
1417         IONIC_PRINT_CALL();
1418
1419         err = ionic_adminq_post_wait(lif, &ctx);
1420         if (err)
1421                 return err;
1422
1423         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1424                         lif->mac_addr)) {
1425                 IONIC_PRINT(INFO, "deleting station MAC addr");
1426
1427                 ionic_lif_addr_del(lif, lif->mac_addr);
1428         }
1429
1430         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1431
1432         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1433                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1434                 return 0;
1435         }
1436
1437         IONIC_PRINT(DEBUG, "adding station MAC addr");
1438
1439         ionic_lif_addr_add(lif, lif->mac_addr);
1440
1441         return 0;
1442 }
1443
1444 static void
1445 ionic_lif_set_name(struct ionic_lif *lif)
1446 {
1447         struct ionic_admin_ctx ctx = {
1448                 .pending_work = true,
1449                 .cmd.lif_setattr = {
1450                         .opcode = IONIC_CMD_LIF_SETATTR,
1451                         .index = lif->index,
1452                         .attr = IONIC_LIF_ATTR_NAME,
1453                 },
1454         };
1455
1456         snprintf(ctx.cmd.lif_setattr.name, sizeof(ctx.cmd.lif_setattr.name),
1457                 "%d", lif->port_id);
1458
1459         ionic_adminq_post_wait(lif, &ctx);
1460 }
1461
1462 int
1463 ionic_lif_init(struct ionic_lif *lif)
1464 {
1465         struct ionic_dev *idev = &lif->adapter->idev;
1466         struct ionic_q_init_comp comp;
1467         int err;
1468
1469         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1470
1471         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
1472         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1473         ionic_dev_cmd_comp(idev, &comp);
1474         if (err)
1475                 return err;
1476
1477         lif->hw_index = comp.hw_index;
1478
1479         err = ionic_lif_adminq_init(lif);
1480         if (err)
1481                 return err;
1482
1483         err = ionic_lif_notifyq_init(lif);
1484         if (err)
1485                 goto err_out_adminq_deinit;
1486
1487         lif->features =
1488                   IONIC_ETH_HW_VLAN_TX_TAG
1489                 | IONIC_ETH_HW_VLAN_RX_STRIP
1490                 | IONIC_ETH_HW_VLAN_RX_FILTER
1491                 | IONIC_ETH_HW_RX_HASH
1492                 | IONIC_ETH_HW_TX_SG
1493                 | IONIC_ETH_HW_RX_SG
1494                 | IONIC_ETH_HW_TX_CSUM
1495                 | IONIC_ETH_HW_RX_CSUM
1496                 | IONIC_ETH_HW_TSO
1497                 | IONIC_ETH_HW_TSO_IPV6
1498                 | IONIC_ETH_HW_TSO_ECN;
1499
1500         err = ionic_lif_set_features(lif);
1501         if (err)
1502                 goto err_out_notifyq_deinit;
1503
1504         err = ionic_rx_filters_init(lif);
1505         if (err)
1506                 goto err_out_notifyq_deinit;
1507
1508         err = ionic_station_set(lif);
1509         if (err)
1510                 goto err_out_rx_filter_deinit;
1511
1512         ionic_lif_set_name(lif);
1513
1514         lif->state |= IONIC_LIF_F_INITED;
1515
1516         return 0;
1517
1518 err_out_rx_filter_deinit:
1519         ionic_rx_filters_deinit(lif);
1520
1521 err_out_notifyq_deinit:
1522         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1523
1524 err_out_adminq_deinit:
1525         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1526
1527         return err;
1528 }
1529
1530 void
1531 ionic_lif_deinit(struct ionic_lif *lif)
1532 {
1533         if (!(lif->state & IONIC_LIF_F_INITED))
1534                 return;
1535
1536         ionic_rx_filters_deinit(lif);
1537         ionic_lif_rss_teardown(lif);
1538         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1539         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1540
1541         lif->state &= ~IONIC_LIF_F_INITED;
1542 }
1543
1544 int
1545 ionic_lif_configure(struct ionic_lif *lif)
1546 {
1547         struct ionic_identity *ident = &lif->adapter->ident;
1548         uint32_t ntxqs_per_lif =
1549                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1550         uint32_t nrxqs_per_lif =
1551                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1552         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1553         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1554
1555         lif->port_id = lif->eth_dev->data->port_id;
1556
1557         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1558                 lif->port_id);
1559
1560         if (nrxqs > 0)
1561                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1562
1563         if (ntxqs > 0)
1564                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1565
1566         lif->nrxqcqs = nrxqs_per_lif;
1567         lif->ntxqcqs = ntxqs_per_lif;
1568
1569         return 0;
1570 }
1571
1572 int
1573 ionic_lif_start(struct ionic_lif *lif)
1574 {
1575         uint32_t rx_mode = 0;
1576         uint32_t i;
1577         int err;
1578
1579         IONIC_PRINT(DEBUG, "Setting RSS configuration on port %u",
1580                 lif->port_id);
1581
1582         err = ionic_lif_rss_setup(lif);
1583         if (err)
1584                 return err;
1585
1586         IONIC_PRINT(DEBUG, "Setting RX mode on port %u",
1587                 lif->port_id);
1588
1589         rx_mode |= IONIC_RX_MODE_F_UNICAST;
1590         rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1591         rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1592
1593         lif->rx_mode = 0; /* set by ionic_set_rx_mode */
1594
1595         ionic_set_rx_mode(lif, rx_mode);
1596
1597         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1598                 "on port %u",
1599                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1600
1601         for (i = 0; i < lif->nrxqcqs; i++) {
1602                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1603                 if (!rxq->deferred_start) {
1604                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1605
1606                         if (err)
1607                                 return err;
1608                 }
1609         }
1610
1611         for (i = 0; i < lif->ntxqcqs; i++) {
1612                 struct ionic_qcq *txq = lif->txqcqs[i];
1613                 if (!txq->deferred_start) {
1614                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1615
1616                         if (err)
1617                                 return err;
1618                 }
1619         }
1620
1621         ionic_link_status_check(lif);
1622
1623         /* Carrier ON here */
1624
1625         return 0;
1626 }
1627
1628 int
1629 ionic_lif_identify(struct ionic_adapter *adapter)
1630 {
1631         struct ionic_dev *idev = &adapter->idev;
1632         struct ionic_identity *ident = &adapter->ident;
1633         int err;
1634         unsigned int i;
1635         unsigned int lif_words = sizeof(ident->lif.words) /
1636                 sizeof(ident->lif.words[0]);
1637         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1638                 sizeof(idev->dev_cmd->data[0]);
1639         unsigned int nwords;
1640
1641         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1642                 IONIC_IDENTITY_VERSION_1);
1643         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1644         if (err)
1645                 return (err);
1646
1647         nwords = RTE_MIN(lif_words, cmd_words);
1648         for (i = 0; i < nwords; i++)
1649                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1650
1651         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1652                 ident->lif.capabilities);
1653
1654         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1655                 ident->lif.eth.max_ucast_filters);
1656         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1657                 ident->lif.eth.max_mcast_filters);
1658
1659         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1660                 ident->lif.eth.config.features);
1661         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1662                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1663         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1664                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1665         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1666                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1667         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1668                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1669
1670         return 0;
1671 }
1672
1673 int
1674 ionic_lifs_size(struct ionic_adapter *adapter)
1675 {
1676         struct ionic_identity *ident = &adapter->ident;
1677         uint32_t nlifs = ident->dev.nlifs;
1678         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1679
1680         adapter->max_ntxqs_per_lif =
1681                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1682         adapter->max_nrxqs_per_lif =
1683                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1684
1685         nintrs = nlifs * 1 /* notifyq */;
1686
1687         if (nintrs > dev_nintrs) {
1688                 IONIC_PRINT(ERR, "At most %d intr queues supported, minimum required is %u",
1689                         dev_nintrs, nintrs);
1690                 return -ENOSPC;
1691         }
1692
1693         adapter->nintrs = nintrs;
1694
1695         return 0;
1696 }