drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .type = q->type,
  29                         .index = q->index,
  30                         .oper = IONIC_Q_ENABLE,
  31                 },
  32         };
  33
  34         if (qcq->flags & IONIC_QCQ_F_INTR) {
  35                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  36                         IONIC_INTR_MASK_CLEAR);
  37         }
  38
  39         return ionic_adminq_post_wait(lif, &ctx);
  40 }
  41
  42 int
  43 ionic_qcq_disable(struct ionic_qcq *qcq)
  44 {
  45         struct ionic_queue *q = &qcq->q;
  46         struct ionic_lif *lif = q->lif;
  47         struct ionic_dev *idev = &lif->adapter->idev;
  48         struct ionic_admin_ctx ctx = {
  49                 .pending_work = true,
  50                 .cmd.q_control = {
  51                         .opcode = IONIC_CMD_Q_CONTROL,
  52                         .type = q->type,
  53                         .index = q->index,
  54                         .oper = IONIC_Q_DISABLE,
  55                 },
  56         };
  57
  58         if (qcq->flags & IONIC_QCQ_F_INTR) {
  59                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  60                         IONIC_INTR_MASK_SET);
  61         }
  62
  63         return ionic_adminq_post_wait(lif, &ctx);
  64 }
  65
  66 void
  67 ionic_lif_stop(struct ionic_lif *lif)
  68 {
  69         IONIC_PRINT_CALL();
  70
  71         lif->state &= ~IONIC_LIF_F_UP;
  72 }
  73
  74 void
  75 ionic_lif_reset(struct ionic_lif *lif)
  76 {
  77         struct ionic_dev *idev = &lif->adapter->idev;
  78         int err;
  79
  80         IONIC_PRINT_CALL();
  81
  82         ionic_dev_cmd_lif_reset(idev);
  83         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  84         if (err)
  85                 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name);
  86 }
  87
  88 static void
  89 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  90 {
  91         struct ionic_lif_stats *ls = &lif->info->stats;
  92         uint32_t i;
  93         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  94                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  95         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  96                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  97
  98         memset(stats, 0, sizeof(*stats));
  99
 100         if (ls == NULL) {
 101                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 102                         lif->port_id);
 103                 return;
 104         }
 105
 106         /* RX */
 107
 108         stats->ipackets = ls->rx_ucast_packets +
 109                 ls->rx_mcast_packets +
 110                 ls->rx_bcast_packets;
 111
 112         stats->ibytes = ls->rx_ucast_bytes +
 113                 ls->rx_mcast_bytes +
 114                 ls->rx_bcast_bytes;
 115
 116         for (i = 0; i < lif->nrxqcqs; i++) {
 117                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 118                 stats->imissed +=
 119                         rx_stats->no_cb_arg +
 120                         rx_stats->bad_cq_status +
 121                         rx_stats->no_room +
 122                         rx_stats->bad_len;
 123         }
 124
 125         stats->imissed +=
 126                 ls->rx_ucast_drop_packets +
 127                 ls->rx_mcast_drop_packets +
 128                 ls->rx_bcast_drop_packets;
 129
 130         stats->imissed +=
 131                 ls->rx_queue_empty +
 132                 ls->rx_dma_error +
 133                 ls->rx_queue_disabled +
 134                 ls->rx_desc_fetch_error +
 135                 ls->rx_desc_data_error;
 136
 137         for (i = 0; i < num_rx_q_counters; i++) {
 138                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 139                 stats->q_ipackets[i] = rx_stats->packets;
 140                 stats->q_ibytes[i] = rx_stats->bytes;
 141                 stats->q_errors[i] =
 142                         rx_stats->no_cb_arg +
 143                         rx_stats->bad_cq_status +
 144                         rx_stats->no_room +
 145                         rx_stats->bad_len;
 146         }
 147
 148         /* TX */
 149
 150         stats->opackets = ls->tx_ucast_packets +
 151                 ls->tx_mcast_packets +
 152                 ls->tx_bcast_packets;
 153
 154         stats->obytes = ls->tx_ucast_bytes +
 155                 ls->tx_mcast_bytes +
 156                 ls->tx_bcast_bytes;
 157
 158         for (i = 0; i < lif->ntxqcqs; i++) {
 159                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 160                 stats->oerrors += tx_stats->drop;
 161         }
 162
 163         stats->oerrors +=
 164                 ls->tx_ucast_drop_packets +
 165                 ls->tx_mcast_drop_packets +
 166                 ls->tx_bcast_drop_packets;
 167
 168         stats->oerrors +=
 169                 ls->tx_dma_error +
 170                 ls->tx_queue_disabled +
 171                 ls->tx_desc_fetch_error +
 172                 ls->tx_desc_data_error;
 173
 174         for (i = 0; i < num_tx_q_counters; i++) {
 175                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 176                 stats->q_opackets[i] = tx_stats->packets;
 177                 stats->q_obytes[i] = tx_stats->bytes;
 178         }
 179 }
 180
 181 void
 182 ionic_lif_get_stats(const struct ionic_lif *lif,
 183                 struct rte_eth_stats *stats)
 184 {
 185         ionic_lif_get_abs_stats(lif, stats);
 186
 187         stats->ipackets  -= lif->stats_base.ipackets;
 188         stats->opackets  -= lif->stats_base.opackets;
 189         stats->ibytes    -= lif->stats_base.ibytes;
 190         stats->obytes    -= lif->stats_base.obytes;
 191         stats->imissed   -= lif->stats_base.imissed;
 192         stats->ierrors   -= lif->stats_base.ierrors;
 193         stats->oerrors   -= lif->stats_base.oerrors;
 194         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 195 }
 196
 197 void
 198 ionic_lif_reset_stats(struct ionic_lif *lif)
 199 {
 200         uint32_t i;
 201
 202         for (i = 0; i < lif->nrxqcqs; i++) {
 203                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 204                         sizeof(struct ionic_rx_stats));
 205                 memset(&lif->txqcqs[i]->stats.tx, 0,
 206                         sizeof(struct ionic_tx_stats));
 207         }
 208
 209         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 210 }
 211
 212 void
 213 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 214 {
 215         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 216         uint64_t *stats64 = (uint64_t *)stats;
 217         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 218         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 219
 220         for (i = 0; i < count; i++)
 221                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 222 }
 223
 224 void
 225 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 226 {
 227         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 228         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 229         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 230
 231         for (i = 0; i < count; i++)
 232                 lif_stats64_base[i] = lif_stats64[i];
 233 }
 234
 235 static int
 236 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 237 {
 238         struct ionic_admin_ctx ctx = {
 239                 .pending_work = true,
 240                 .cmd.rx_filter_add = {
 241                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 242                         .match = IONIC_RX_FILTER_MATCH_MAC,
 243                 },
 244         };
 245         int err;
 246
 247         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 248
 249         err = ionic_adminq_post_wait(lif, &ctx);
 250         if (err)
 251                 return err;
 252
 253         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 254                 ctx.comp.rx_filter_add.filter_id);
 255
 256         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 257 }
 258
 259 static int
 260 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 261 {
 262         struct ionic_admin_ctx ctx = {
 263                 .pending_work = true,
 264                 .cmd.rx_filter_del = {
 265                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 266                 },
 267         };
 268         struct ionic_rx_filter *f;
 269         int err;
 270
 271         IONIC_PRINT_CALL();
 272
 273         rte_spinlock_lock(&lif->rx_filters.lock);
 274
 275         f = ionic_rx_filter_by_addr(lif, addr);
 276         if (!f) {
 277                 rte_spinlock_unlock(&lif->rx_filters.lock);
 278                 return -ENOENT;
 279         }
 280
 281         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 282         ionic_rx_filter_free(f);
 283
 284         rte_spinlock_unlock(&lif->rx_filters.lock);
 285
 286         err = ionic_adminq_post_wait(lif, &ctx);
 287         if (err)
 288                 return err;
 289
 290         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 291                 ctx.cmd.rx_filter_del.filter_id);
 292
 293         return 0;
 294 }
 295
 296 int
 297 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 298                 struct rte_ether_addr *mac_addr,
 299                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 300 {
 301         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 302
 303         IONIC_PRINT_CALL();
 304
 305         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 306 }
 307
 308 void
 309 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index)
 310 {
 311         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 312         struct ionic_adapter *adapter = lif->adapter;
 313         struct rte_ether_addr *mac_addr;
 314
 315         IONIC_PRINT_CALL();
 316
 317         if (index >= adapter->max_mac_addrs) {
 318                 IONIC_PRINT(WARNING,
 319                         "Index %u is above MAC filter limit %u",
 320                         index, adapter->max_mac_addrs);
 321                 return;
 322         }
 323
 324         mac_addr = &eth_dev->data->mac_addrs[index];
 325
 326         if (!rte_is_valid_assigned_ether_addr(mac_addr))
 327                 return;
 328
 329         ionic_lif_addr_del(lif, (const uint8_t *)mac_addr);
 330 }
 331
 332 int
 333 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 334 {
 335         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 336
 337         IONIC_PRINT_CALL();
 338
 339         if (mac_addr == NULL) {
 340                 IONIC_PRINT(NOTICE, "New mac is null");
 341                 return -1;
 342         }
 343
 344         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 345                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 346                         lif->mac_addr);
 347                 ionic_lif_addr_del(lif, lif->mac_addr);
 348                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 349         }
 350
 351         IONIC_PRINT(INFO, "Updating mac addr");
 352
 353         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 354
 355         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 356 }
 357
 358 static int
 359 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 360 {
 361         struct ionic_admin_ctx ctx = {
 362                 .pending_work = true,
 363                 .cmd.rx_filter_add = {
 364                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 365                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 366                         .vlan.vlan = vid,
 367                 },
 368         };
 369         int err;
 370
 371         err = ionic_adminq_post_wait(lif, &ctx);
 372         if (err)
 373                 return err;
 374
 375         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 376                 ctx.comp.rx_filter_add.filter_id);
 377
 378         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 379 }
 380
 381 static int
 382 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 383 {
 384         struct ionic_admin_ctx ctx = {
 385                 .pending_work = true,
 386                 .cmd.rx_filter_del = {
 387                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 388                 },
 389         };
 390         struct ionic_rx_filter *f;
 391         int err;
 392
 393         IONIC_PRINT_CALL();
 394
 395         rte_spinlock_lock(&lif->rx_filters.lock);
 396
 397         f = ionic_rx_filter_by_vlan(lif, vid);
 398         if (!f) {
 399                 rte_spinlock_unlock(&lif->rx_filters.lock);
 400                 return -ENOENT;
 401         }
 402
 403         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 404         ionic_rx_filter_free(f);
 405         rte_spinlock_unlock(&lif->rx_filters.lock);
 406
 407         err = ionic_adminq_post_wait(lif, &ctx);
 408         if (err)
 409                 return err;
 410
 411         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 412                 ctx.cmd.rx_filter_del.filter_id);
 413
 414         return 0;
 415 }
 416
 417 int
 418 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 419                 int on)
 420 {
 421         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 422         int err;
 423
 424         if (on)
 425                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 426         else
 427                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 428
 429         return err;
 430 }
 431
 432 static void
 433 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 434 {
 435         struct ionic_admin_ctx ctx = {
 436                 .pending_work = true,
 437                 .cmd.rx_mode_set = {
 438                         .opcode = IONIC_CMD_RX_MODE_SET,
 439                         .rx_mode = rx_mode,
 440                 },
 441         };
 442         int err;
 443
 444         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 445                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 446         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 447                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 448         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 449                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 450         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 451                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 452         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 453                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 454
 455         err = ionic_adminq_post_wait(lif, &ctx);
 456         if (err)
 457                 IONIC_PRINT(ERR, "Failure setting RX mode");
 458 }
 459
 460 static void
 461 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 462 {
 463         if (lif->rx_mode != rx_mode) {
 464                 lif->rx_mode = rx_mode;
 465                 ionic_lif_rx_mode(lif, rx_mode);
 466         }
 467 }
 468
 469 int
 470 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 471 {
 472         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 473         uint32_t rx_mode = lif->rx_mode;
 474
 475         IONIC_PRINT_CALL();
 476
 477         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 478
 479         ionic_set_rx_mode(lif, rx_mode);
 480
 481         return 0;
 482 }
 483
 484 int
 485 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 486 {
 487         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 488         uint32_t rx_mode = lif->rx_mode;
 489
 490         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 491
 492         ionic_set_rx_mode(lif, rx_mode);
 493
 494         return 0;
 495 }
 496
 497 int
 498 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 499 {
 500         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 501         uint32_t rx_mode = lif->rx_mode;
 502
 503         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 504
 505         ionic_set_rx_mode(lif, rx_mode);
 506
 507         return 0;
 508 }
 509
 510 int
 511 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 512 {
 513         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 514         uint32_t rx_mode = lif->rx_mode;
 515
 516         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 517
 518         ionic_set_rx_mode(lif, rx_mode);
 519
 520         return 0;
 521 }
 522
 523 int
 524 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 525 {
 526         struct ionic_admin_ctx ctx = {
 527                 .pending_work = true,
 528                 .cmd.lif_setattr = {
 529                         .opcode = IONIC_CMD_LIF_SETATTR,
 530                         .attr = IONIC_LIF_ATTR_MTU,
 531                         .mtu = new_mtu,
 532                 },
 533         };
 534         int err;
 535
 536         err = ionic_adminq_post_wait(lif, &ctx);
 537         if (err)
 538                 return err;
 539
 540         return 0;
 541 }
 542
 543 int
 544 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 545 {
 546         struct ionic_adapter *adapter = lif->adapter;
 547         struct ionic_dev *idev = &adapter->idev;
 548         unsigned long index;
 549
 550         /*
 551          * Note: interrupt handler is called for index = 0 only
 552          * (we use interrupts for the notifyq only anyway,
 553          * which has index = 0)
 554          */
 555
 556         for (index = 0; index < adapter->nintrs; index++)
 557                 if (!adapter->intrs[index])
 558                         break;
 559
 560         if (index == adapter->nintrs)
 561                 return -ENOSPC;
 562
 563         adapter->intrs[index] = true;
 564
 565         ionic_intr_init(idev, intr, index);
 566
 567         return 0;
 568 }
 569
 570 void
 571 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 572 {
 573         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 574                 lif->adapter->intrs[intr->index] = false;
 575 }
 576
 577 static int
 578 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 579                 uint32_t index,
 580                 const char *base, uint32_t flags,
 581                 uint32_t num_descs,
 582                 uint32_t desc_size,
 583                 uint32_t cq_desc_size,
 584                 uint32_t sg_desc_size,
 585                 struct ionic_qcq **qcq)
 586 {
 587         struct ionic_dev *idev = &lif->adapter->idev;
 588         struct ionic_qcq *new;
 589         uint32_t q_size, cq_size, sg_size, total_size;
 590         void *q_base, *cq_base, *sg_base;
 591         rte_iova_t q_base_pa = 0;
 592         rte_iova_t cq_base_pa = 0;
 593         rte_iova_t sg_base_pa = 0;
 594         uint32_t socket_id = rte_socket_id();
 595         int err;
 596
 597         *qcq = NULL;
 598
 599         q_size  = num_descs * desc_size;
 600         cq_size = num_descs * cq_desc_size;
 601         sg_size = num_descs * sg_desc_size;
 602
 603         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 604                 RTE_ALIGN(cq_size, PAGE_SIZE);
 605         /*
 606          * Note: aligning q_size/cq_size is not enough due to cq_base address
 607          * aligning as q_base could be not aligned to the page.
 608          * Adding PAGE_SIZE.
 609          */
 610         total_size += PAGE_SIZE;
 611
 612         if (flags & IONIC_QCQ_F_SG) {
 613                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 614                 total_size += PAGE_SIZE;
 615         }
 616
 617         new = rte_zmalloc("ionic", sizeof(*new), 0);
 618         if (!new) {
 619                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 620                 return -ENOMEM;
 621         }
 622
 623         new->lif = lif;
 624         new->flags = flags;
 625
 626         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 627         if (!new->q.info) {
 628                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 629                 return -ENOMEM;
 630         }
 631
 632         new->q.type = type;
 633
 634         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 635                 desc_size, sg_desc_size);
 636         if (err) {
 637                 IONIC_PRINT(ERR, "Queue initialization failed");
 638                 return err;
 639         }
 640
 641         if (flags & IONIC_QCQ_F_INTR) {
 642                 err = ionic_intr_alloc(lif, &new->intr);
 643                 if (err)
 644                         return err;
 645
 646                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 647                         IONIC_INTR_MASK_SET);
 648         } else {
 649                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 650         }
 651
 652         err = ionic_cq_init(lif, &new->cq, &new->intr,
 653                 num_descs, cq_desc_size);
 654         if (err) {
 655                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 656                 goto err_out_free_intr;
 657         }
 658
 659         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 660                 base /* name */, index /* queue_idx */,
 661                 total_size, IONIC_ALIGN, socket_id);
 662
 663         if (!new->base_z) {
 664                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 665                 err = -ENOMEM;
 666                 goto err_out_free_intr;
 667         }
 668
 669         new->base = new->base_z->addr;
 670         new->base_pa = new->base_z->iova;
 671         new->total_size = total_size;
 672
 673         q_base = new->base;
 674         q_base_pa = new->base_pa;
 675
 676         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 677         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 678
 679         if (flags & IONIC_QCQ_F_SG) {
 680                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 681                         PAGE_SIZE);
 682                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 683                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 684         }
 685
 686         IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 687                 "SG-base-PA = %#jx",
 688                 q_base_pa, cq_base_pa, sg_base_pa);
 689
 690         ionic_q_map(&new->q, q_base, q_base_pa);
 691         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 692         ionic_cq_bind(&new->cq, &new->q);
 693
 694         *qcq = new;
 695
 696         return 0;
 697
 698 err_out_free_intr:
 699         if (flags & IONIC_QCQ_F_INTR)
 700                 ionic_intr_free(lif, &new->intr);
 701
 702         return err;
 703 }
 704
 705 void
 706 ionic_qcq_free(struct ionic_qcq *qcq)
 707 {
 708         if (qcq->base_z) {
 709                 qcq->base = NULL;
 710                 qcq->base_pa = 0;
 711                 rte_memzone_free(qcq->base_z);
 712                 qcq->base_z = NULL;
 713         }
 714
 715         if (qcq->q.info) {
 716                 rte_free(qcq->q.info);
 717                 qcq->q.info = NULL;
 718         }
 719
 720         rte_free(qcq);
 721 }
 722
 723 int
 724 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 725                 struct ionic_qcq **qcq)
 726 {
 727         uint32_t flags;
 728         int err = -ENOMEM;
 729
 730         flags = IONIC_QCQ_F_SG;
 731         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 732                 nrxq_descs,
 733                 sizeof(struct ionic_rxq_desc),
 734                 sizeof(struct ionic_rxq_comp),
 735                 sizeof(struct ionic_rxq_sg_desc),
 736                 &lif->rxqcqs[index]);
 737         if (err)
 738                 return err;
 739
 740         *qcq = lif->rxqcqs[index];
 741
 742         return 0;
 743 }
 744
 745 int
 746 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 747                 struct ionic_qcq **qcq)
 748 {
 749         uint32_t flags;
 750         int err = -ENOMEM;
 751
 752         flags = IONIC_QCQ_F_SG;
 753         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 754                 ntxq_descs,
 755                 sizeof(struct ionic_txq_desc),
 756                 sizeof(struct ionic_txq_comp),
 757                 sizeof(struct ionic_txq_sg_desc),
 758                 &lif->txqcqs[index]);
 759         if (err)
 760                 return err;
 761
 762         *qcq = lif->txqcqs[index];
 763
 764         return 0;
 765 }
 766
 767 static int
 768 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 769 {
 770         uint32_t flags;
 771         int err = -ENOMEM;
 772
 773         flags = 0;
 774         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 775                 IONIC_ADMINQ_LENGTH,
 776                 sizeof(struct ionic_admin_cmd),
 777                 sizeof(struct ionic_admin_comp),
 778                 0,
 779                 &lif->adminqcq);
 780         if (err)
 781                 return err;
 782
 783         return 0;
 784 }
 785
 786 static int
 787 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 788 {
 789         uint32_t flags;
 790         int err = -ENOMEM;
 791
 792         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 793
 794         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 795                 flags,
 796                 IONIC_NOTIFYQ_LENGTH,
 797                 sizeof(struct ionic_notifyq_cmd),
 798                 sizeof(union ionic_notifyq_comp),
 799                 0,
 800                 &lif->notifyqcq);
 801         if (err)
 802                 return err;
 803
 804         return 0;
 805 }
 806
 807 static void *
 808 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 809 {
 810         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 811
 812         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 813                 return NULL;
 814
 815         return (void *)&vaddr[page_num << PAGE_SHIFT];
 816 }
 817
 818 int
 819 ionic_lif_alloc(struct ionic_lif *lif)
 820 {
 821         struct ionic_adapter *adapter = lif->adapter;
 822         uint32_t socket_id = rte_socket_id();
 823         int err;
 824
 825         /*
 826          * lif->name was zeroed on allocation.
 827          * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated.
 828          */
 829         memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1);
 830
 831         IONIC_PRINT(DEBUG, "LIF: %s", lif->name);
 832
 833         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 834
 835         rte_spinlock_init(&lif->adminq_lock);
 836         rte_spinlock_init(&lif->adminq_service_lock);
 837
 838         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, 0);
 839         if (!lif->kern_dbpage) {
 840                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 841                 return -ENOMEM;
 842         }
 843
 844         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 845                 adapter->max_ntxqs_per_lif, 0);
 846
 847         if (!lif->txqcqs) {
 848                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 849                 return -ENOMEM;
 850         }
 851
 852         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 853                 adapter->max_nrxqs_per_lif, 0);
 854
 855         if (!lif->rxqcqs) {
 856                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 857                 return -ENOMEM;
 858         }
 859
 860         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 861
 862         err = ionic_notify_qcq_alloc(lif);
 863         if (err) {
 864                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 865                 return err;
 866         }
 867
 868         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 869
 870         err = ionic_admin_qcq_alloc(lif);
 871         if (err) {
 872                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 873                 return err;
 874         }
 875
 876         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 877
 878         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 879
 880         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 881                 "lif_info", 0 /* queue_idx*/,
 882                 lif->info_sz, IONIC_ALIGN, socket_id);
 883         if (!lif->info_z) {
 884                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 885                 return -ENOMEM;
 886         }
 887
 888         lif->info = lif->info_z->addr;
 889         lif->info_pa = lif->info_z->iova;
 890
 891         return 0;
 892 }
 893
 894 void
 895 ionic_lif_free(struct ionic_lif *lif)
 896 {
 897         if (lif->notifyqcq) {
 898                 ionic_qcq_free(lif->notifyqcq);
 899                 lif->notifyqcq = NULL;
 900         }
 901
 902         if (lif->adminqcq) {
 903                 ionic_qcq_free(lif->adminqcq);
 904                 lif->adminqcq = NULL;
 905         }
 906
 907         if (lif->txqcqs) {
 908                 rte_free(lif->txqcqs);
 909                 lif->txqcqs = NULL;
 910         }
 911
 912         if (lif->rxqcqs) {
 913                 rte_free(lif->rxqcqs);
 914                 lif->rxqcqs = NULL;
 915         }
 916
 917         if (lif->info) {
 918                 rte_memzone_free(lif->info_z);
 919                 lif->info = NULL;
 920         }
 921 }
 922
 923 void
 924 ionic_lif_free_queues(struct ionic_lif *lif)
 925 {
 926         uint32_t i;
 927
 928         for (i = 0; i < lif->ntxqcqs; i++) {
 929                 ionic_dev_tx_queue_release(lif->eth_dev->data->tx_queues[i]);
 930                 lif->eth_dev->data->tx_queues[i] = NULL;
 931         }
 932         for (i = 0; i < lif->nrxqcqs; i++) {
 933                 ionic_dev_rx_queue_release(lif->eth_dev->data->rx_queues[i]);
 934                 lif->eth_dev->data->rx_queues[i] = NULL;
 935         }
 936 }
 937
 938 int
 939 ionic_lif_rss_config(struct ionic_lif *lif,
 940                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 941 {
 942         struct ionic_admin_ctx ctx = {
 943                 .pending_work = true,
 944                 .cmd.lif_setattr = {
 945                         .opcode = IONIC_CMD_LIF_SETATTR,
 946                         .attr = IONIC_LIF_ATTR_RSS,
 947                         .rss.types = types,
 948                         .rss.addr = lif->rss_ind_tbl_pa,
 949                 },
 950         };
 951         unsigned int i;
 952
 953         IONIC_PRINT_CALL();
 954
 955         lif->rss_types = types;
 956
 957         if (key)
 958                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 959
 960         if (indir)
 961                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 962                         lif->rss_ind_tbl[i] = indir[i];
 963
 964         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 965                IONIC_RSS_HASH_KEY_SIZE);
 966
 967         return ionic_adminq_post_wait(lif, &ctx);
 968 }
 969
 970 static int
 971 ionic_lif_rss_setup(struct ionic_lif *lif)
 972 {
 973         static const uint8_t toeplitz_symmetric_key[] = {
 974                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 975                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 976                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 977                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 978                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 979         };
 980         uint32_t i;
 981         uint16_t tbl_sz = lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 982
 983         IONIC_PRINT_CALL();
 984
 985         if (!lif->rss_ind_tbl_z) {
 986                 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 987                                         "rss_ind_tbl", 0 /* queue_idx */,
 988                                         sizeof(*lif->rss_ind_tbl) * tbl_sz,
 989                                         IONIC_ALIGN, rte_socket_id());
 990                 if (!lif->rss_ind_tbl_z) {
 991                         IONIC_PRINT(ERR, "OOM");
 992                         return -ENOMEM;
 993                 }
 994
 995                 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 996                 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
 997         }
 998
 999         if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) {
1000                 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs;
1001
1002                 /* Fill indirection table with 'default' values */
1003                 for (i = 0; i < tbl_sz; i++)
1004                         lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
1005         }
1006
1007         return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
1008                         toeplitz_symmetric_key, NULL);
1009 }
1010
1011 static void
1012 ionic_lif_rss_teardown(struct ionic_lif *lif)
1013 {
1014         if (!lif->rss_ind_tbl)
1015                 return;
1016
1017         if (lif->rss_ind_tbl_z) {
1018                 /* Disable RSS on the NIC */
1019                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1020
1021                 lif->rss_ind_tbl = NULL;
1022                 lif->rss_ind_tbl_pa = 0;
1023                 rte_memzone_free(lif->rss_ind_tbl_z);
1024                 lif->rss_ind_tbl_z = NULL;
1025         }
1026 }
1027
1028 static void
1029 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1030 {
1031         struct ionic_dev *idev = &lif->adapter->idev;
1032
1033         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1034                 return;
1035
1036         if (qcq->flags & IONIC_QCQ_F_INTR)
1037                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1038                         IONIC_INTR_MASK_SET);
1039
1040         qcq->flags &= ~IONIC_QCQ_F_INITED;
1041 }
1042
1043 void
1044 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1045 {
1046         ionic_lif_qcq_deinit(qcq->lif, qcq);
1047 }
1048
1049 void
1050 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1051 {
1052         ionic_lif_qcq_deinit(qcq->lif, qcq);
1053 }
1054
1055 bool
1056 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1057                 void *cb_arg __rte_unused)
1058 {
1059         struct ionic_admin_comp *cq_desc_base = cq->base;
1060         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1061
1062         if (!color_match(cq_desc->color, cq->done_color))
1063                 return false;
1064
1065         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1066
1067         return true;
1068 }
1069
1070 /* This acts like ionic_napi */
1071 int
1072 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1073                 void *cb_arg)
1074 {
1075         struct ionic_cq *cq = &qcq->cq;
1076         uint32_t work_done;
1077
1078         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1079
1080         return work_done;
1081 }
1082
1083 static void
1084 ionic_link_status_check(struct ionic_lif *lif)
1085 {
1086         struct ionic_adapter *adapter = lif->adapter;
1087         bool link_up;
1088
1089         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1090
1091         if (!lif->info)
1092                 return;
1093
1094         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1095
1096         if ((link_up  && adapter->link_up) ||
1097             (!link_up && !adapter->link_up))
1098                 return;
1099
1100         if (link_up) {
1101                 adapter->link_speed = lif->info->status.link_speed;
1102                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1103                         adapter->link_speed);
1104         } else {
1105                 IONIC_PRINT(DEBUG, "Link down");
1106         }
1107
1108         adapter->link_up = link_up;
1109         ionic_dev_link_update(lif->eth_dev, 0);
1110 }
1111
1112 static void
1113 ionic_lif_handle_fw_down(struct ionic_lif *lif)
1114 {
1115         if (lif->state & IONIC_LIF_F_FW_RESET)
1116                 return;
1117
1118         lif->state |= IONIC_LIF_F_FW_RESET;
1119
1120         if (lif->state & IONIC_LIF_F_UP) {
1121                 IONIC_PRINT(NOTICE,
1122                         "Surprise FW stop, stopping %s\n", lif->name);
1123                 ionic_lif_stop(lif);
1124         }
1125
1126         IONIC_PRINT(NOTICE, "FW down, %s stopped", lif->name);
1127 }
1128
1129 static bool
1130 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1131 {
1132         union ionic_notifyq_comp *cq_desc_base = cq->base;
1133         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1134         struct ionic_lif *lif = cb_arg;
1135
1136         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1137                 cq_desc->event.eid, cq_desc->event.ecode);
1138
1139         /* Have we run out of new completions to process? */
1140         if (!(cq_desc->event.eid > lif->last_eid))
1141                 return false;
1142
1143         lif->last_eid = cq_desc->event.eid;
1144
1145         switch (cq_desc->event.ecode) {
1146         case IONIC_EVENT_LINK_CHANGE:
1147                 IONIC_PRINT(DEBUG,
1148                         "Notifyq IONIC_EVENT_LINK_CHANGE %s "
1149                         "eid=%jd link_status=%d link_speed=%d",
1150                         lif->name,
1151                         cq_desc->event.eid,
1152                         cq_desc->link_change.link_status,
1153                         cq_desc->link_change.link_speed);
1154
1155                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1156                 break;
1157
1158         case IONIC_EVENT_RESET:
1159                 IONIC_PRINT(NOTICE,
1160                         "Notifyq IONIC_EVENT_RESET %s "
1161                         "eid=%jd, reset_code=%d state=%d",
1162                         lif->name,
1163                         cq_desc->event.eid,
1164                         cq_desc->reset.reset_code,
1165                         cq_desc->reset.state);
1166                 ionic_lif_handle_fw_down(lif);
1167                 break;
1168
1169         default:
1170                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1171                         cq_desc->event.ecode, cq_desc->event.eid);
1172                 break;
1173         }
1174
1175         return true;
1176 }
1177
1178 int
1179 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1180 {
1181         struct ionic_dev *idev = &lif->adapter->idev;
1182         struct ionic_qcq *qcq = lif->notifyqcq;
1183         uint32_t work_done;
1184
1185         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1186                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1187                 return -1;
1188         }
1189
1190         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1191                 IONIC_INTR_MASK_SET);
1192
1193         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1194
1195         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1196                 ionic_link_status_check(lif);
1197
1198         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1199                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1200
1201         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1202                 IONIC_INTR_MASK_CLEAR);
1203
1204         return 0;
1205 }
1206
1207 static int
1208 ionic_lif_adminq_init(struct ionic_lif *lif)
1209 {
1210         struct ionic_dev *idev = &lif->adapter->idev;
1211         struct ionic_qcq *qcq = lif->adminqcq;
1212         struct ionic_queue *q = &qcq->q;
1213         struct ionic_q_init_comp comp;
1214         int err;
1215
1216         ionic_dev_cmd_adminq_init(idev, qcq, qcq->intr.index);
1217         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1218         if (err)
1219                 return err;
1220
1221         ionic_dev_cmd_comp(idev, &comp);
1222
1223         q->hw_type = comp.hw_type;
1224         q->hw_index = comp.hw_index;
1225         q->db = ionic_db_map(lif, q);
1226
1227         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1228         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1229         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1230
1231         if (qcq->flags & IONIC_QCQ_F_INTR)
1232                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1233                         IONIC_INTR_MASK_CLEAR);
1234
1235         qcq->flags |= IONIC_QCQ_F_INITED;
1236
1237         return 0;
1238 }
1239
1240 static int
1241 ionic_lif_notifyq_init(struct ionic_lif *lif)
1242 {
1243         struct ionic_dev *idev = &lif->adapter->idev;
1244         struct ionic_qcq *qcq = lif->notifyqcq;
1245         struct ionic_queue *q = &qcq->q;
1246         int err;
1247
1248         struct ionic_admin_ctx ctx = {
1249                 .pending_work = true,
1250                 .cmd.q_init = {
1251                         .opcode = IONIC_CMD_Q_INIT,
1252                         .type = q->type,
1253                         .index = q->index,
1254                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1255                         .intr_index = qcq->intr.index,
1256                         .ring_size = rte_log2_u32(q->num_descs),
1257                         .ring_base = q->base_pa,
1258                 }
1259         };
1260
1261         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1262                 ctx.cmd.q_init.index);
1263         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1264                 ctx.cmd.q_init.ring_base);
1265         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1266                 ctx.cmd.q_init.ring_size);
1267         IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver);
1268
1269         err = ionic_adminq_post_wait(lif, &ctx);
1270         if (err)
1271                 return err;
1272
1273         q->hw_type = ctx.comp.q_init.hw_type;
1274         q->hw_index = ctx.comp.q_init.hw_index;
1275         q->db = NULL;
1276
1277         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1278         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1279         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1280
1281         if (qcq->flags & IONIC_QCQ_F_INTR)
1282                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1283                         IONIC_INTR_MASK_CLEAR);
1284
1285         qcq->flags |= IONIC_QCQ_F_INITED;
1286
1287         return 0;
1288 }
1289
1290 int
1291 ionic_lif_set_features(struct ionic_lif *lif)
1292 {
1293         struct ionic_admin_ctx ctx = {
1294                 .pending_work = true,
1295                 .cmd.lif_setattr = {
1296                         .opcode = IONIC_CMD_LIF_SETATTR,
1297                         .attr = IONIC_LIF_ATTR_FEATURES,
1298                         .features = lif->features,
1299                 },
1300         };
1301         int err;
1302
1303         err = ionic_adminq_post_wait(lif, &ctx);
1304         if (err)
1305                 return err;
1306
1307         lif->hw_features = (ctx.cmd.lif_setattr.features &
1308                 ctx.comp.lif_setattr.features);
1309
1310         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1311                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1312         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1313                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1314         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1315                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1316         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1317                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1318         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1319                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1320         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1321                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1322         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1323                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1324         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1325                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1326         if (lif->hw_features & IONIC_ETH_HW_TSO)
1327                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1328         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1329                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1330         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1331                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1332         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1333                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1334         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1335                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1336         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1337                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1338         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1339                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1340         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1341                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1342         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1343                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1344
1345         return 0;
1346 }
1347
1348 int
1349 ionic_lif_txq_init(struct ionic_qcq *qcq)
1350 {
1351         struct ionic_queue *q = &qcq->q;
1352         struct ionic_lif *lif = qcq->lif;
1353         struct ionic_cq *cq = &qcq->cq;
1354         struct ionic_admin_ctx ctx = {
1355                 .pending_work = true,
1356                 .cmd.q_init = {
1357                         .opcode = IONIC_CMD_Q_INIT,
1358                         .type = q->type,
1359                         .index = q->index,
1360                         .flags = IONIC_QINIT_F_SG,
1361                         .intr_index = cq->bound_intr->index,
1362                         .ring_size = rte_log2_u32(q->num_descs),
1363                         .ring_base = q->base_pa,
1364                         .cq_ring_base = cq->base_pa,
1365                         .sg_ring_base = q->sg_base_pa,
1366                 },
1367         };
1368         int err;
1369
1370         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1371         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1372                 ctx.cmd.q_init.ring_base);
1373         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1374                 ctx.cmd.q_init.ring_size);
1375         IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver);
1376
1377         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1378         if (err)
1379                 return err;
1380
1381         q->hw_type = ctx.comp.q_init.hw_type;
1382         q->hw_index = ctx.comp.q_init.hw_index;
1383         q->db = ionic_db_map(lif, q);
1384
1385         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1386         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1387         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1388
1389         qcq->flags |= IONIC_QCQ_F_INITED;
1390
1391         return 0;
1392 }
1393
1394 int
1395 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1396 {
1397         struct ionic_queue *q = &qcq->q;
1398         struct ionic_lif *lif = qcq->lif;
1399         struct ionic_cq *cq = &qcq->cq;
1400         struct ionic_admin_ctx ctx = {
1401                 .pending_work = true,
1402                 .cmd.q_init = {
1403                         .opcode = IONIC_CMD_Q_INIT,
1404                         .type = q->type,
1405                         .index = q->index,
1406                         .flags = IONIC_QINIT_F_SG,
1407                         .intr_index = cq->bound_intr->index,
1408                         .ring_size = rte_log2_u32(q->num_descs),
1409                         .ring_base = q->base_pa,
1410                         .cq_ring_base = cq->base_pa,
1411                         .sg_ring_base = q->sg_base_pa,
1412                 },
1413         };
1414         int err;
1415
1416         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1417         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1418                 ctx.cmd.q_init.ring_base);
1419         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1420                 ctx.cmd.q_init.ring_size);
1421         IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver);
1422
1423         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1424         if (err)
1425                 return err;
1426
1427         q->hw_type = ctx.comp.q_init.hw_type;
1428         q->hw_index = ctx.comp.q_init.hw_index;
1429         q->db = ionic_db_map(lif, q);
1430
1431         qcq->flags |= IONIC_QCQ_F_INITED;
1432
1433         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1434         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1435         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1436
1437         return 0;
1438 }
1439
1440 static int
1441 ionic_station_set(struct ionic_lif *lif)
1442 {
1443         struct ionic_admin_ctx ctx = {
1444                 .pending_work = true,
1445                 .cmd.lif_getattr = {
1446                         .opcode = IONIC_CMD_LIF_GETATTR,
1447                         .attr = IONIC_LIF_ATTR_MAC,
1448                 },
1449         };
1450         int err;
1451
1452         IONIC_PRINT_CALL();
1453
1454         err = ionic_adminq_post_wait(lif, &ctx);
1455         if (err)
1456                 return err;
1457
1458         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1459                         lif->mac_addr)) {
1460                 IONIC_PRINT(INFO, "deleting station MAC addr");
1461
1462                 ionic_lif_addr_del(lif, lif->mac_addr);
1463         }
1464
1465         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1466
1467         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1468                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1469                 return 0;
1470         }
1471
1472         IONIC_PRINT(DEBUG, "adding station MAC addr");
1473
1474         ionic_lif_addr_add(lif, lif->mac_addr);
1475
1476         return 0;
1477 }
1478
1479 static void
1480 ionic_lif_set_name(struct ionic_lif *lif)
1481 {
1482         struct ionic_admin_ctx ctx = {
1483                 .pending_work = true,
1484                 .cmd.lif_setattr = {
1485                         .opcode = IONIC_CMD_LIF_SETATTR,
1486                         .attr = IONIC_LIF_ATTR_NAME,
1487                 },
1488         };
1489
1490         memcpy(ctx.cmd.lif_setattr.name, lif->name,
1491                 sizeof(ctx.cmd.lif_setattr.name) - 1);
1492
1493         ionic_adminq_post_wait(lif, &ctx);
1494 }
1495
1496 int
1497 ionic_lif_init(struct ionic_lif *lif)
1498 {
1499         struct ionic_dev *idev = &lif->adapter->idev;
1500         struct ionic_q_init_comp comp;
1501         int err;
1502
1503         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1504
1505         ionic_dev_cmd_lif_init(idev, lif->info_pa);
1506         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1507         ionic_dev_cmd_comp(idev, &comp);
1508         if (err)
1509                 return err;
1510
1511         lif->hw_index = comp.hw_index;
1512
1513         err = ionic_lif_adminq_init(lif);
1514         if (err)
1515                 return err;
1516
1517         err = ionic_lif_notifyq_init(lif);
1518         if (err)
1519                 goto err_out_adminq_deinit;
1520
1521         lif->features =
1522                   IONIC_ETH_HW_VLAN_TX_TAG
1523                 | IONIC_ETH_HW_VLAN_RX_STRIP
1524                 | IONIC_ETH_HW_VLAN_RX_FILTER
1525                 | IONIC_ETH_HW_RX_HASH
1526                 | IONIC_ETH_HW_TX_SG
1527                 | IONIC_ETH_HW_RX_SG
1528                 | IONIC_ETH_HW_TX_CSUM
1529                 | IONIC_ETH_HW_RX_CSUM
1530                 | IONIC_ETH_HW_TSO
1531                 | IONIC_ETH_HW_TSO_IPV6
1532                 | IONIC_ETH_HW_TSO_ECN;
1533
1534         err = ionic_lif_set_features(lif);
1535         if (err)
1536                 goto err_out_notifyq_deinit;
1537
1538         err = ionic_rx_filters_init(lif);
1539         if (err)
1540                 goto err_out_notifyq_deinit;
1541
1542         err = ionic_station_set(lif);
1543         if (err)
1544                 goto err_out_rx_filter_deinit;
1545
1546         ionic_lif_set_name(lif);
1547
1548         lif->state |= IONIC_LIF_F_INITED;
1549
1550         return 0;
1551
1552 err_out_rx_filter_deinit:
1553         ionic_rx_filters_deinit(lif);
1554
1555 err_out_notifyq_deinit:
1556         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1557
1558 err_out_adminq_deinit:
1559         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1560
1561         return err;
1562 }
1563
1564 void
1565 ionic_lif_deinit(struct ionic_lif *lif)
1566 {
1567         if (!(lif->state & IONIC_LIF_F_INITED))
1568                 return;
1569
1570         ionic_rx_filters_deinit(lif);
1571         ionic_lif_rss_teardown(lif);
1572         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1573         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1574
1575         lif->state &= ~IONIC_LIF_F_INITED;
1576 }
1577
1578 int
1579 ionic_lif_configure(struct ionic_lif *lif)
1580 {
1581         struct ionic_identity *ident = &lif->adapter->ident;
1582         uint32_t ntxqs_per_lif =
1583                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1584         uint32_t nrxqs_per_lif =
1585                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1586         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1587         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1588
1589         lif->port_id = lif->eth_dev->data->port_id;
1590
1591         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1592                 lif->port_id);
1593
1594         if (nrxqs > 0)
1595                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1596
1597         if (ntxqs > 0)
1598                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1599
1600         lif->nrxqcqs = nrxqs_per_lif;
1601         lif->ntxqcqs = ntxqs_per_lif;
1602
1603         return 0;
1604 }
1605
1606 int
1607 ionic_lif_start(struct ionic_lif *lif)
1608 {
1609         uint32_t rx_mode;
1610         uint32_t i;
1611         int err;
1612
1613         err = ionic_lif_rss_setup(lif);
1614         if (err)
1615                 return err;
1616
1617         if (!lif->rx_mode) {
1618                 IONIC_PRINT(DEBUG, "Setting RX mode on %s",
1619                         lif->name);
1620
1621                 rx_mode  = IONIC_RX_MODE_F_UNICAST;
1622                 rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1623                 rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1624
1625                 ionic_set_rx_mode(lif, rx_mode);
1626         }
1627
1628         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1629                 "on port %u",
1630                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1631
1632         for (i = 0; i < lif->nrxqcqs; i++) {
1633                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1634                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1635                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1636
1637                         if (err)
1638                                 return err;
1639                 }
1640         }
1641
1642         for (i = 0; i < lif->ntxqcqs; i++) {
1643                 struct ionic_qcq *txq = lif->txqcqs[i];
1644                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1645                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1646
1647                         if (err)
1648                                 return err;
1649                 }
1650         }
1651
1652         /* Carrier ON here */
1653         lif->state |= IONIC_LIF_F_UP;
1654
1655         ionic_link_status_check(lif);
1656
1657         return 0;
1658 }
1659
1660 int
1661 ionic_lif_identify(struct ionic_adapter *adapter)
1662 {
1663         struct ionic_dev *idev = &adapter->idev;
1664         struct ionic_identity *ident = &adapter->ident;
1665         int err;
1666         unsigned int i;
1667         unsigned int lif_words = sizeof(ident->lif.words) /
1668                 sizeof(ident->lif.words[0]);
1669         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1670                 sizeof(idev->dev_cmd->data[0]);
1671         unsigned int nwords;
1672
1673         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1674                 IONIC_IDENTITY_VERSION_1);
1675         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1676         if (err)
1677                 return (err);
1678
1679         nwords = RTE_MIN(lif_words, cmd_words);
1680         for (i = 0; i < nwords; i++)
1681                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1682
1683         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1684                 ident->lif.capabilities);
1685
1686         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1687                 ident->lif.eth.max_ucast_filters);
1688         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1689                 ident->lif.eth.max_mcast_filters);
1690
1691         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1692                 ident->lif.eth.config.features);
1693         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1694                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1695         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1696                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1697         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1698                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1699         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1700                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1701
1702         return 0;
1703 }
1704
1705 int
1706 ionic_lifs_size(struct ionic_adapter *adapter)
1707 {
1708         struct ionic_identity *ident = &adapter->ident;
1709         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1710
1711         adapter->max_ntxqs_per_lif =
1712                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1713         adapter->max_nrxqs_per_lif =
1714                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1715
1716         nintrs = 1 /* notifyq */;
1717
1718         if (nintrs > dev_nintrs) {
1719                 IONIC_PRINT(ERR,
1720                         "At most %d intr supported, minimum req'd is %u",
1721                         dev_nintrs, nintrs);
1722                 return -ENOSPC;
1723         }
1724
1725         adapter->nintrs = nintrs;
1726
1727         return 0;
1728 }