drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_admin_ctx ctx = {
  24                 .pending_work = true,
  25                 .cmd.q_control = {
  26                         .opcode = IONIC_CMD_Q_CONTROL,
  27                         .type = q->type,
  28                         .index = q->index,
  29                         .oper = IONIC_Q_ENABLE,
  30                 },
  31         };
  32
  33         return ionic_adminq_post_wait(lif, &ctx);
  34 }
  35
  36 int
  37 ionic_qcq_disable(struct ionic_qcq *qcq)
  38 {
  39         struct ionic_queue *q = &qcq->q;
  40         struct ionic_lif *lif = q->lif;
  41         struct ionic_admin_ctx ctx = {
  42                 .pending_work = true,
  43                 .cmd.q_control = {
  44                         .opcode = IONIC_CMD_Q_CONTROL,
  45                         .type = q->type,
  46                         .index = q->index,
  47                         .oper = IONIC_Q_DISABLE,
  48                 },
  49         };
  50
  51         return ionic_adminq_post_wait(lif, &ctx);
  52 }
  53
  54 void
  55 ionic_lif_stop(struct ionic_lif *lif)
  56 {
  57         uint32_t i;
  58
  59         IONIC_PRINT_CALL();
  60
  61         lif->state &= ~IONIC_LIF_F_UP;
  62
  63         for (i = 0; i < lif->nrxqcqs; i++) {
  64                 struct ionic_qcq *rxq = lif->rxqcqs[i];
  65                 if (rxq->flags & IONIC_QCQ_F_INITED)
  66                         (void)ionic_dev_rx_queue_stop(lif->eth_dev, i);
  67         }
  68
  69         for (i = 0; i < lif->ntxqcqs; i++) {
  70                 struct ionic_qcq *txq = lif->txqcqs[i];
  71                 if (txq->flags & IONIC_QCQ_F_INITED)
  72                         (void)ionic_dev_tx_queue_stop(lif->eth_dev, i);
  73         }
  74 }
  75
  76 void
  77 ionic_lif_reset(struct ionic_lif *lif)
  78 {
  79         struct ionic_dev *idev = &lif->adapter->idev;
  80         int err;
  81
  82         IONIC_PRINT_CALL();
  83
  84         ionic_dev_cmd_lif_reset(idev);
  85         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  86         if (err)
  87                 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name);
  88 }
  89
  90 static void
  91 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  92 {
  93         struct ionic_lif_stats *ls = &lif->info->stats;
  94         uint32_t i;
  95         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  96                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  97         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  98                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  99
 100         memset(stats, 0, sizeof(*stats));
 101
 102         if (ls == NULL) {
 103                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 104                         lif->port_id);
 105                 return;
 106         }
 107
 108         /* RX */
 109
 110         stats->ipackets = ls->rx_ucast_packets +
 111                 ls->rx_mcast_packets +
 112                 ls->rx_bcast_packets;
 113
 114         stats->ibytes = ls->rx_ucast_bytes +
 115                 ls->rx_mcast_bytes +
 116                 ls->rx_bcast_bytes;
 117
 118         for (i = 0; i < lif->nrxqcqs; i++) {
 119                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 120                 stats->imissed +=
 121                         rx_stats->no_cb_arg +
 122                         rx_stats->bad_cq_status +
 123                         rx_stats->no_room +
 124                         rx_stats->bad_len;
 125         }
 126
 127         stats->imissed +=
 128                 ls->rx_ucast_drop_packets +
 129                 ls->rx_mcast_drop_packets +
 130                 ls->rx_bcast_drop_packets;
 131
 132         stats->imissed +=
 133                 ls->rx_queue_empty +
 134                 ls->rx_dma_error +
 135                 ls->rx_queue_disabled +
 136                 ls->rx_desc_fetch_error +
 137                 ls->rx_desc_data_error;
 138
 139         for (i = 0; i < num_rx_q_counters; i++) {
 140                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 141                 stats->q_ipackets[i] = rx_stats->packets;
 142                 stats->q_ibytes[i] = rx_stats->bytes;
 143                 stats->q_errors[i] =
 144                         rx_stats->no_cb_arg +
 145                         rx_stats->bad_cq_status +
 146                         rx_stats->no_room +
 147                         rx_stats->bad_len;
 148         }
 149
 150         /* TX */
 151
 152         stats->opackets = ls->tx_ucast_packets +
 153                 ls->tx_mcast_packets +
 154                 ls->tx_bcast_packets;
 155
 156         stats->obytes = ls->tx_ucast_bytes +
 157                 ls->tx_mcast_bytes +
 158                 ls->tx_bcast_bytes;
 159
 160         for (i = 0; i < lif->ntxqcqs; i++) {
 161                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 162                 stats->oerrors += tx_stats->drop;
 163         }
 164
 165         stats->oerrors +=
 166                 ls->tx_ucast_drop_packets +
 167                 ls->tx_mcast_drop_packets +
 168                 ls->tx_bcast_drop_packets;
 169
 170         stats->oerrors +=
 171                 ls->tx_dma_error +
 172                 ls->tx_queue_disabled +
 173                 ls->tx_desc_fetch_error +
 174                 ls->tx_desc_data_error;
 175
 176         for (i = 0; i < num_tx_q_counters; i++) {
 177                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 178                 stats->q_opackets[i] = tx_stats->packets;
 179                 stats->q_obytes[i] = tx_stats->bytes;
 180         }
 181 }
 182
 183 void
 184 ionic_lif_get_stats(const struct ionic_lif *lif,
 185                 struct rte_eth_stats *stats)
 186 {
 187         ionic_lif_get_abs_stats(lif, stats);
 188
 189         stats->ipackets  -= lif->stats_base.ipackets;
 190         stats->opackets  -= lif->stats_base.opackets;
 191         stats->ibytes    -= lif->stats_base.ibytes;
 192         stats->obytes    -= lif->stats_base.obytes;
 193         stats->imissed   -= lif->stats_base.imissed;
 194         stats->ierrors   -= lif->stats_base.ierrors;
 195         stats->oerrors   -= lif->stats_base.oerrors;
 196         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 197 }
 198
 199 void
 200 ionic_lif_reset_stats(struct ionic_lif *lif)
 201 {
 202         uint32_t i;
 203
 204         for (i = 0; i < lif->nrxqcqs; i++) {
 205                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 206                         sizeof(struct ionic_rx_stats));
 207                 memset(&lif->txqcqs[i]->stats.tx, 0,
 208                         sizeof(struct ionic_tx_stats));
 209         }
 210
 211         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 212 }
 213
 214 void
 215 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 216 {
 217         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 218         uint64_t *stats64 = (uint64_t *)stats;
 219         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 220         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 221
 222         for (i = 0; i < count; i++)
 223                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 224 }
 225
 226 void
 227 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 228 {
 229         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 230         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 231         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 232
 233         for (i = 0; i < count; i++)
 234                 lif_stats64_base[i] = lif_stats64[i];
 235 }
 236
 237 static int
 238 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 239 {
 240         struct ionic_admin_ctx ctx = {
 241                 .pending_work = true,
 242                 .cmd.rx_filter_add = {
 243                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 244                         .match = IONIC_RX_FILTER_MATCH_MAC,
 245                 },
 246         };
 247         int err;
 248
 249         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 250
 251         err = ionic_adminq_post_wait(lif, &ctx);
 252         if (err)
 253                 return err;
 254
 255         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 256                 ctx.comp.rx_filter_add.filter_id);
 257
 258         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 259 }
 260
 261 static int
 262 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 263 {
 264         struct ionic_admin_ctx ctx = {
 265                 .pending_work = true,
 266                 .cmd.rx_filter_del = {
 267                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 268                 },
 269         };
 270         struct ionic_rx_filter *f;
 271         int err;
 272
 273         IONIC_PRINT_CALL();
 274
 275         rte_spinlock_lock(&lif->rx_filters.lock);
 276
 277         f = ionic_rx_filter_by_addr(lif, addr);
 278         if (!f) {
 279                 rte_spinlock_unlock(&lif->rx_filters.lock);
 280                 return -ENOENT;
 281         }
 282
 283         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 284         ionic_rx_filter_free(f);
 285
 286         rte_spinlock_unlock(&lif->rx_filters.lock);
 287
 288         err = ionic_adminq_post_wait(lif, &ctx);
 289         if (err)
 290                 return err;
 291
 292         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 293                 ctx.cmd.rx_filter_del.filter_id);
 294
 295         return 0;
 296 }
 297
 298 int
 299 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 300                 struct rte_ether_addr *mac_addr,
 301                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 302 {
 303         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 304
 305         IONIC_PRINT_CALL();
 306
 307         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 308 }
 309
 310 void
 311 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index)
 312 {
 313         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 314         struct ionic_adapter *adapter = lif->adapter;
 315         struct rte_ether_addr *mac_addr;
 316
 317         IONIC_PRINT_CALL();
 318
 319         if (index >= adapter->max_mac_addrs) {
 320                 IONIC_PRINT(WARNING,
 321                         "Index %u is above MAC filter limit %u",
 322                         index, adapter->max_mac_addrs);
 323                 return;
 324         }
 325
 326         mac_addr = &eth_dev->data->mac_addrs[index];
 327
 328         if (!rte_is_valid_assigned_ether_addr(mac_addr))
 329                 return;
 330
 331         ionic_lif_addr_del(lif, (const uint8_t *)mac_addr);
 332 }
 333
 334 int
 335 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 336 {
 337         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 338
 339         IONIC_PRINT_CALL();
 340
 341         if (mac_addr == NULL) {
 342                 IONIC_PRINT(NOTICE, "New mac is null");
 343                 return -1;
 344         }
 345
 346         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 347                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 348                         lif->mac_addr);
 349                 ionic_lif_addr_del(lif, lif->mac_addr);
 350                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 351         }
 352
 353         IONIC_PRINT(INFO, "Updating mac addr");
 354
 355         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 356
 357         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 358 }
 359
 360 static int
 361 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 362 {
 363         struct ionic_admin_ctx ctx = {
 364                 .pending_work = true,
 365                 .cmd.rx_filter_add = {
 366                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 367                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 368                         .vlan.vlan = vid,
 369                 },
 370         };
 371         int err;
 372
 373         err = ionic_adminq_post_wait(lif, &ctx);
 374         if (err)
 375                 return err;
 376
 377         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 378                 ctx.comp.rx_filter_add.filter_id);
 379
 380         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 381 }
 382
 383 static int
 384 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 385 {
 386         struct ionic_admin_ctx ctx = {
 387                 .pending_work = true,
 388                 .cmd.rx_filter_del = {
 389                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 390                 },
 391         };
 392         struct ionic_rx_filter *f;
 393         int err;
 394
 395         IONIC_PRINT_CALL();
 396
 397         rte_spinlock_lock(&lif->rx_filters.lock);
 398
 399         f = ionic_rx_filter_by_vlan(lif, vid);
 400         if (!f) {
 401                 rte_spinlock_unlock(&lif->rx_filters.lock);
 402                 return -ENOENT;
 403         }
 404
 405         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 406         ionic_rx_filter_free(f);
 407         rte_spinlock_unlock(&lif->rx_filters.lock);
 408
 409         err = ionic_adminq_post_wait(lif, &ctx);
 410         if (err)
 411                 return err;
 412
 413         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 414                 ctx.cmd.rx_filter_del.filter_id);
 415
 416         return 0;
 417 }
 418
 419 int
 420 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 421                 int on)
 422 {
 423         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 424         int err;
 425
 426         if (on)
 427                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 428         else
 429                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 430
 431         return err;
 432 }
 433
 434 static void
 435 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 436 {
 437         struct ionic_admin_ctx ctx = {
 438                 .pending_work = true,
 439                 .cmd.rx_mode_set = {
 440                         .opcode = IONIC_CMD_RX_MODE_SET,
 441                         .rx_mode = rx_mode,
 442                 },
 443         };
 444         int err;
 445
 446         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 447                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 448         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 449                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 450         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 451                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 452         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 453                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 454         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 455                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 456
 457         err = ionic_adminq_post_wait(lif, &ctx);
 458         if (err)
 459                 IONIC_PRINT(ERR, "Failure setting RX mode");
 460 }
 461
 462 static void
 463 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 464 {
 465         if (lif->rx_mode != rx_mode) {
 466                 lif->rx_mode = rx_mode;
 467                 ionic_lif_rx_mode(lif, rx_mode);
 468         }
 469 }
 470
 471 int
 472 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 473 {
 474         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 475         uint32_t rx_mode = lif->rx_mode;
 476
 477         IONIC_PRINT_CALL();
 478
 479         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 480
 481         ionic_set_rx_mode(lif, rx_mode);
 482
 483         return 0;
 484 }
 485
 486 int
 487 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 488 {
 489         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 490         uint32_t rx_mode = lif->rx_mode;
 491
 492         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 493
 494         ionic_set_rx_mode(lif, rx_mode);
 495
 496         return 0;
 497 }
 498
 499 int
 500 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 501 {
 502         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 503         uint32_t rx_mode = lif->rx_mode;
 504
 505         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 506
 507         ionic_set_rx_mode(lif, rx_mode);
 508
 509         return 0;
 510 }
 511
 512 int
 513 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 514 {
 515         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 516         uint32_t rx_mode = lif->rx_mode;
 517
 518         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 519
 520         ionic_set_rx_mode(lif, rx_mode);
 521
 522         return 0;
 523 }
 524
 525 int
 526 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 527 {
 528         struct ionic_admin_ctx ctx = {
 529                 .pending_work = true,
 530                 .cmd.lif_setattr = {
 531                         .opcode = IONIC_CMD_LIF_SETATTR,
 532                         .attr = IONIC_LIF_ATTR_MTU,
 533                         .mtu = new_mtu,
 534                 },
 535         };
 536         int err;
 537
 538         err = ionic_adminq_post_wait(lif, &ctx);
 539         if (err)
 540                 return err;
 541
 542         return 0;
 543 }
 544
 545 int
 546 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 547 {
 548         struct ionic_adapter *adapter = lif->adapter;
 549         struct ionic_dev *idev = &adapter->idev;
 550         unsigned long index;
 551
 552         /*
 553          * Note: interrupt handler is called for index = 0 only
 554          * (we use interrupts for the notifyq only anyway,
 555          * which has index = 0)
 556          */
 557
 558         for (index = 0; index < adapter->nintrs; index++)
 559                 if (!adapter->intrs[index])
 560                         break;
 561
 562         if (index == adapter->nintrs)
 563                 return -ENOSPC;
 564
 565         adapter->intrs[index] = true;
 566
 567         ionic_intr_init(idev, intr, index);
 568
 569         return 0;
 570 }
 571
 572 void
 573 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 574 {
 575         if (intr->index != IONIC_INTR_NONE)
 576                 lif->adapter->intrs[intr->index] = false;
 577 }
 578
 579 static int
 580 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 581                 uint32_t index,
 582                 const char *base, uint32_t flags,
 583                 uint32_t num_descs,
 584                 uint32_t desc_size,
 585                 uint32_t cq_desc_size,
 586                 uint32_t sg_desc_size,
 587                 struct ionic_qcq **qcq)
 588 {
 589         struct ionic_dev *idev = &lif->adapter->idev;
 590         struct ionic_qcq *new;
 591         uint32_t q_size, cq_size, sg_size, total_size;
 592         void *q_base, *cq_base, *sg_base;
 593         rte_iova_t q_base_pa = 0;
 594         rte_iova_t cq_base_pa = 0;
 595         rte_iova_t sg_base_pa = 0;
 596         uint32_t socket_id = rte_socket_id();
 597         int err;
 598
 599         *qcq = NULL;
 600
 601         q_size  = num_descs * desc_size;
 602         cq_size = num_descs * cq_desc_size;
 603         sg_size = num_descs * sg_desc_size;
 604
 605         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 606                 RTE_ALIGN(cq_size, PAGE_SIZE);
 607         /*
 608          * Note: aligning q_size/cq_size is not enough due to cq_base address
 609          * aligning as q_base could be not aligned to the page.
 610          * Adding PAGE_SIZE.
 611          */
 612         total_size += PAGE_SIZE;
 613
 614         if (flags & IONIC_QCQ_F_SG) {
 615                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 616                 total_size += PAGE_SIZE;
 617         }
 618
 619         new = rte_zmalloc("ionic", sizeof(*new), 0);
 620         if (!new) {
 621                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 622                 return -ENOMEM;
 623         }
 624
 625         new->lif = lif;
 626         new->flags = flags;
 627
 628         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 629         if (!new->q.info) {
 630                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 631                 err = -ENOMEM;
 632                 goto err_out_free_qcq;
 633         }
 634
 635         new->q.type = type;
 636
 637         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 638                 desc_size, sg_desc_size);
 639         if (err) {
 640                 IONIC_PRINT(ERR, "Queue initialization failed");
 641                 goto err_out_free_info;
 642         }
 643
 644         err = ionic_cq_init(lif, &new->cq, num_descs, cq_desc_size);
 645         if (err) {
 646                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 647                 goto err_out_free_info;
 648         }
 649
 650         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 651                 base /* name */, index /* queue_idx */,
 652                 total_size, IONIC_ALIGN, socket_id);
 653
 654         if (!new->base_z) {
 655                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 656                 err = -ENOMEM;
 657                 goto err_out_free_info;
 658         }
 659
 660         new->base = new->base_z->addr;
 661         new->base_pa = new->base_z->iova;
 662         new->total_size = total_size;
 663
 664         q_base = new->base;
 665         q_base_pa = new->base_pa;
 666
 667         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 668         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 669
 670         if (flags & IONIC_QCQ_F_SG) {
 671                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 672                         PAGE_SIZE);
 673                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 674                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 675         }
 676
 677         IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 678                 "SG-base-PA = %#jx",
 679                 q_base_pa, cq_base_pa, sg_base_pa);
 680
 681         ionic_q_map(&new->q, q_base, q_base_pa);
 682         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 683         ionic_cq_bind(&new->cq, &new->q);
 684
 685         *qcq = new;
 686
 687         return 0;
 688
 689 err_out_free_info:
 690         rte_free(new->q.info);
 691 err_out_free_qcq:
 692         rte_free(new);
 693
 694         return err;
 695 }
 696
 697 void
 698 ionic_qcq_free(struct ionic_qcq *qcq)
 699 {
 700         if (qcq->base_z) {
 701                 qcq->base = NULL;
 702                 qcq->base_pa = 0;
 703                 rte_memzone_free(qcq->base_z);
 704                 qcq->base_z = NULL;
 705         }
 706
 707         if (qcq->q.info) {
 708                 rte_free(qcq->q.info);
 709                 qcq->q.info = NULL;
 710         }
 711
 712         rte_free(qcq);
 713 }
 714
 715 int
 716 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 717                 struct ionic_qcq **qcq)
 718 {
 719         uint32_t flags;
 720         int err = -ENOMEM;
 721
 722         flags = IONIC_QCQ_F_SG;
 723         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 724                 nrxq_descs,
 725                 sizeof(struct ionic_rxq_desc),
 726                 sizeof(struct ionic_rxq_comp),
 727                 sizeof(struct ionic_rxq_sg_desc),
 728                 &lif->rxqcqs[index]);
 729         if (err)
 730                 return err;
 731
 732         *qcq = lif->rxqcqs[index];
 733
 734         return 0;
 735 }
 736
 737 int
 738 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 739                 struct ionic_qcq **qcq)
 740 {
 741         uint32_t flags;
 742         int err = -ENOMEM;
 743
 744         flags = IONIC_QCQ_F_SG;
 745         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 746                 ntxq_descs,
 747                 sizeof(struct ionic_txq_desc),
 748                 sizeof(struct ionic_txq_comp),
 749                 sizeof(struct ionic_txq_sg_desc),
 750                 &lif->txqcqs[index]);
 751         if (err)
 752                 return err;
 753
 754         *qcq = lif->txqcqs[index];
 755
 756         return 0;
 757 }
 758
 759 static int
 760 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 761 {
 762         uint32_t flags;
 763         int err = -ENOMEM;
 764
 765         flags = 0;
 766         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 767                 IONIC_ADMINQ_LENGTH,
 768                 sizeof(struct ionic_admin_cmd),
 769                 sizeof(struct ionic_admin_comp),
 770                 0,
 771                 &lif->adminqcq);
 772         if (err)
 773                 return err;
 774
 775         return 0;
 776 }
 777
 778 static int
 779 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 780 {
 781         struct ionic_qcq *nqcq;
 782         struct ionic_dev *idev = &lif->adapter->idev;
 783         uint32_t flags = 0;
 784         int err = -ENOMEM;
 785
 786         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 787                 flags,
 788                 IONIC_NOTIFYQ_LENGTH,
 789                 sizeof(struct ionic_notifyq_cmd),
 790                 sizeof(union ionic_notifyq_comp),
 791                 0,
 792                 &nqcq);
 793         if (err)
 794                 return err;
 795
 796         err = ionic_intr_alloc(lif, &nqcq->intr);
 797         if (err) {
 798                 ionic_qcq_free(nqcq);
 799                 return err;
 800         }
 801
 802         ionic_intr_mask_assert(idev->intr_ctrl, nqcq->intr.index,
 803                 IONIC_INTR_MASK_SET);
 804
 805         lif->notifyqcq = nqcq;
 806
 807         return 0;
 808 }
 809
 810 static void *
 811 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 812 {
 813         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 814
 815         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 816                 return NULL;
 817
 818         return (void *)&vaddr[page_num << PAGE_SHIFT];
 819 }
 820
 821 int
 822 ionic_lif_alloc(struct ionic_lif *lif)
 823 {
 824         struct ionic_adapter *adapter = lif->adapter;
 825         uint32_t socket_id = rte_socket_id();
 826         int err;
 827
 828         /*
 829          * lif->name was zeroed on allocation.
 830          * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated.
 831          */
 832         memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1);
 833
 834         IONIC_PRINT(DEBUG, "LIF: %s", lif->name);
 835
 836         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 837
 838         rte_spinlock_init(&lif->adminq_lock);
 839         rte_spinlock_init(&lif->adminq_service_lock);
 840
 841         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, 0);
 842         if (!lif->kern_dbpage) {
 843                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 844                 return -ENOMEM;
 845         }
 846
 847         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 848                 adapter->max_ntxqs_per_lif, 0);
 849
 850         if (!lif->txqcqs) {
 851                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 852                 return -ENOMEM;
 853         }
 854
 855         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 856                 adapter->max_nrxqs_per_lif, 0);
 857
 858         if (!lif->rxqcqs) {
 859                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 860                 return -ENOMEM;
 861         }
 862
 863         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 864
 865         err = ionic_notify_qcq_alloc(lif);
 866         if (err) {
 867                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 868                 return err;
 869         }
 870
 871         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 872
 873         err = ionic_admin_qcq_alloc(lif);
 874         if (err) {
 875                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 876                 return err;
 877         }
 878
 879         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 880
 881         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 882
 883         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 884                 "lif_info", 0 /* queue_idx*/,
 885                 lif->info_sz, IONIC_ALIGN, socket_id);
 886         if (!lif->info_z) {
 887                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 888                 return -ENOMEM;
 889         }
 890
 891         lif->info = lif->info_z->addr;
 892         lif->info_pa = lif->info_z->iova;
 893
 894         return 0;
 895 }
 896
 897 void
 898 ionic_lif_free(struct ionic_lif *lif)
 899 {
 900         if (lif->notifyqcq) {
 901                 ionic_qcq_free(lif->notifyqcq);
 902                 lif->notifyqcq = NULL;
 903         }
 904
 905         if (lif->adminqcq) {
 906                 ionic_qcq_free(lif->adminqcq);
 907                 lif->adminqcq = NULL;
 908         }
 909
 910         if (lif->txqcqs) {
 911                 rte_free(lif->txqcqs);
 912                 lif->txqcqs = NULL;
 913         }
 914
 915         if (lif->rxqcqs) {
 916                 rte_free(lif->rxqcqs);
 917                 lif->rxqcqs = NULL;
 918         }
 919
 920         if (lif->info) {
 921                 rte_memzone_free(lif->info_z);
 922                 lif->info = NULL;
 923         }
 924 }
 925
 926 void
 927 ionic_lif_free_queues(struct ionic_lif *lif)
 928 {
 929         uint32_t i;
 930
 931         for (i = 0; i < lif->ntxqcqs; i++) {
 932                 ionic_dev_tx_queue_release(lif->eth_dev->data->tx_queues[i]);
 933                 lif->eth_dev->data->tx_queues[i] = NULL;
 934         }
 935         for (i = 0; i < lif->nrxqcqs; i++) {
 936                 ionic_dev_rx_queue_release(lif->eth_dev->data->rx_queues[i]);
 937                 lif->eth_dev->data->rx_queues[i] = NULL;
 938         }
 939 }
 940
 941 int
 942 ionic_lif_rss_config(struct ionic_lif *lif,
 943                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 944 {
 945         struct ionic_admin_ctx ctx = {
 946                 .pending_work = true,
 947                 .cmd.lif_setattr = {
 948                         .opcode = IONIC_CMD_LIF_SETATTR,
 949                         .attr = IONIC_LIF_ATTR_RSS,
 950                         .rss.types = types,
 951                         .rss.addr = lif->rss_ind_tbl_pa,
 952                 },
 953         };
 954         unsigned int i;
 955
 956         IONIC_PRINT_CALL();
 957
 958         lif->rss_types = types;
 959
 960         if (key)
 961                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 962
 963         if (indir)
 964                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 965                         lif->rss_ind_tbl[i] = indir[i];
 966
 967         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 968                IONIC_RSS_HASH_KEY_SIZE);
 969
 970         return ionic_adminq_post_wait(lif, &ctx);
 971 }
 972
 973 static int
 974 ionic_lif_rss_setup(struct ionic_lif *lif)
 975 {
 976         static const uint8_t toeplitz_symmetric_key[] = {
 977                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 978                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 979                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 980                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 981                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 982         };
 983         uint32_t i;
 984         uint16_t tbl_sz = lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 985
 986         IONIC_PRINT_CALL();
 987
 988         if (!lif->rss_ind_tbl_z) {
 989                 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 990                                         "rss_ind_tbl", 0 /* queue_idx */,
 991                                         sizeof(*lif->rss_ind_tbl) * tbl_sz,
 992                                         IONIC_ALIGN, rte_socket_id());
 993                 if (!lif->rss_ind_tbl_z) {
 994                         IONIC_PRINT(ERR, "OOM");
 995                         return -ENOMEM;
 996                 }
 997
 998                 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 999                 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
1000         }
1001
1002         if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) {
1003                 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs;
1004
1005                 /* Fill indirection table with 'default' values */
1006                 for (i = 0; i < tbl_sz; i++)
1007                         lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
1008         }
1009
1010         return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
1011                         toeplitz_symmetric_key, NULL);
1012 }
1013
1014 static void
1015 ionic_lif_rss_teardown(struct ionic_lif *lif)
1016 {
1017         if (!lif->rss_ind_tbl)
1018                 return;
1019
1020         if (lif->rss_ind_tbl_z) {
1021                 /* Disable RSS on the NIC */
1022                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1023
1024                 lif->rss_ind_tbl = NULL;
1025                 lif->rss_ind_tbl_pa = 0;
1026                 rte_memzone_free(lif->rss_ind_tbl_z);
1027                 lif->rss_ind_tbl_z = NULL;
1028         }
1029 }
1030
1031 static void
1032 ionic_lif_qcq_deinit(struct ionic_qcq *qcq)
1033 {
1034         qcq->flags &= ~IONIC_QCQ_F_INITED;
1035 }
1036
1037 void
1038 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1039 {
1040         ionic_lif_qcq_deinit(qcq);
1041 }
1042
1043 void
1044 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1045 {
1046         ionic_lif_qcq_deinit(qcq);
1047 }
1048
1049 static void
1050 ionic_lif_notifyq_deinit(struct ionic_lif *lif)
1051 {
1052         struct ionic_qcq *nqcq = lif->notifyqcq;
1053         struct ionic_dev *idev = &lif->adapter->idev;
1054
1055         if (!(nqcq->flags & IONIC_QCQ_F_INITED))
1056                 return;
1057
1058         ionic_intr_mask(idev->intr_ctrl, nqcq->intr.index,
1059                 IONIC_INTR_MASK_SET);
1060
1061         nqcq->flags &= ~IONIC_QCQ_F_INITED;
1062 }
1063
1064 bool
1065 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1066                 void *cb_arg __rte_unused)
1067 {
1068         struct ionic_admin_comp *cq_desc_base = cq->base;
1069         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1070
1071         if (!color_match(cq_desc->color, cq->done_color))
1072                 return false;
1073
1074         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1075
1076         return true;
1077 }
1078
1079 /* This acts like ionic_napi */
1080 int
1081 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1082                 void *cb_arg)
1083 {
1084         struct ionic_cq *cq = &qcq->cq;
1085         uint32_t work_done;
1086
1087         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1088
1089         return work_done;
1090 }
1091
1092 static void
1093 ionic_link_status_check(struct ionic_lif *lif)
1094 {
1095         struct ionic_adapter *adapter = lif->adapter;
1096         bool link_up;
1097
1098         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1099
1100         if (!lif->info)
1101                 return;
1102
1103         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1104
1105         if ((link_up  && adapter->link_up) ||
1106             (!link_up && !adapter->link_up))
1107                 return;
1108
1109         if (link_up) {
1110                 adapter->link_speed = lif->info->status.link_speed;
1111                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1112                         adapter->link_speed);
1113         } else {
1114                 IONIC_PRINT(DEBUG, "Link down");
1115         }
1116
1117         adapter->link_up = link_up;
1118         ionic_dev_link_update(lif->eth_dev, 0);
1119 }
1120
1121 static void
1122 ionic_lif_handle_fw_down(struct ionic_lif *lif)
1123 {
1124         if (lif->state & IONIC_LIF_F_FW_RESET)
1125                 return;
1126
1127         lif->state |= IONIC_LIF_F_FW_RESET;
1128
1129         if (lif->state & IONIC_LIF_F_UP) {
1130                 IONIC_PRINT(NOTICE,
1131                         "Surprise FW stop, stopping %s\n", lif->name);
1132                 ionic_lif_stop(lif);
1133         }
1134
1135         IONIC_PRINT(NOTICE, "FW down, %s stopped", lif->name);
1136 }
1137
1138 static bool
1139 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1140 {
1141         union ionic_notifyq_comp *cq_desc_base = cq->base;
1142         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1143         struct ionic_lif *lif = cb_arg;
1144
1145         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1146                 cq_desc->event.eid, cq_desc->event.ecode);
1147
1148         /* Have we run out of new completions to process? */
1149         if (!(cq_desc->event.eid > lif->last_eid))
1150                 return false;
1151
1152         lif->last_eid = cq_desc->event.eid;
1153
1154         switch (cq_desc->event.ecode) {
1155         case IONIC_EVENT_LINK_CHANGE:
1156                 IONIC_PRINT(DEBUG,
1157                         "Notifyq IONIC_EVENT_LINK_CHANGE %s "
1158                         "eid=%jd link_status=%d link_speed=%d",
1159                         lif->name,
1160                         cq_desc->event.eid,
1161                         cq_desc->link_change.link_status,
1162                         cq_desc->link_change.link_speed);
1163
1164                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1165                 break;
1166
1167         case IONIC_EVENT_RESET:
1168                 IONIC_PRINT(NOTICE,
1169                         "Notifyq IONIC_EVENT_RESET %s "
1170                         "eid=%jd, reset_code=%d state=%d",
1171                         lif->name,
1172                         cq_desc->event.eid,
1173                         cq_desc->reset.reset_code,
1174                         cq_desc->reset.state);
1175                 ionic_lif_handle_fw_down(lif);
1176                 break;
1177
1178         default:
1179                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1180                         cq_desc->event.ecode, cq_desc->event.eid);
1181                 break;
1182         }
1183
1184         return true;
1185 }
1186
1187 int
1188 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1189 {
1190         struct ionic_dev *idev = &lif->adapter->idev;
1191         struct ionic_qcq *qcq = lif->notifyqcq;
1192         uint32_t work_done;
1193
1194         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1195                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1196                 return -1;
1197         }
1198
1199         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1200                 IONIC_INTR_MASK_SET);
1201
1202         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1203
1204         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1205                 ionic_link_status_check(lif);
1206
1207         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1208                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1209
1210         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1211                 IONIC_INTR_MASK_CLEAR);
1212
1213         return 0;
1214 }
1215
1216 static int
1217 ionic_lif_adminq_init(struct ionic_lif *lif)
1218 {
1219         struct ionic_dev *idev = &lif->adapter->idev;
1220         struct ionic_qcq *qcq = lif->adminqcq;
1221         struct ionic_queue *q = &qcq->q;
1222         struct ionic_q_init_comp comp;
1223         int err;
1224
1225         ionic_dev_cmd_adminq_init(idev, qcq);
1226         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1227         if (err)
1228                 return err;
1229
1230         ionic_dev_cmd_comp(idev, &comp);
1231
1232         q->hw_type = comp.hw_type;
1233         q->hw_index = comp.hw_index;
1234         q->db = ionic_db_map(lif, q);
1235
1236         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1237         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1238         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1239
1240         qcq->flags |= IONIC_QCQ_F_INITED;
1241
1242         return 0;
1243 }
1244
1245 static int
1246 ionic_lif_notifyq_init(struct ionic_lif *lif)
1247 {
1248         struct ionic_dev *idev = &lif->adapter->idev;
1249         struct ionic_qcq *qcq = lif->notifyqcq;
1250         struct ionic_queue *q = &qcq->q;
1251         int err;
1252
1253         struct ionic_admin_ctx ctx = {
1254                 .pending_work = true,
1255                 .cmd.q_init = {
1256                         .opcode = IONIC_CMD_Q_INIT,
1257                         .type = q->type,
1258                         .index = q->index,
1259                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1260                         .intr_index = qcq->intr.index,
1261                         .ring_size = rte_log2_u32(q->num_descs),
1262                         .ring_base = q->base_pa,
1263                 }
1264         };
1265
1266         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1267                 ctx.cmd.q_init.index);
1268         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1269                 ctx.cmd.q_init.ring_base);
1270         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1271                 ctx.cmd.q_init.ring_size);
1272         IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver);
1273
1274         err = ionic_adminq_post_wait(lif, &ctx);
1275         if (err)
1276                 return err;
1277
1278         q->hw_type = ctx.comp.q_init.hw_type;
1279         q->hw_index = ctx.comp.q_init.hw_index;
1280         q->db = NULL;
1281
1282         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1283         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1284         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1285
1286         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1287                 IONIC_INTR_MASK_CLEAR);
1288
1289         qcq->flags |= IONIC_QCQ_F_INITED;
1290
1291         return 0;
1292 }
1293
1294 int
1295 ionic_lif_set_features(struct ionic_lif *lif)
1296 {
1297         struct ionic_admin_ctx ctx = {
1298                 .pending_work = true,
1299                 .cmd.lif_setattr = {
1300                         .opcode = IONIC_CMD_LIF_SETATTR,
1301                         .attr = IONIC_LIF_ATTR_FEATURES,
1302                         .features = lif->features,
1303                 },
1304         };
1305         int err;
1306
1307         err = ionic_adminq_post_wait(lif, &ctx);
1308         if (err)
1309                 return err;
1310
1311         lif->hw_features = (ctx.cmd.lif_setattr.features &
1312                 ctx.comp.lif_setattr.features);
1313
1314         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1315                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1316         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1317                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1318         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1319                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1320         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1321                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1322         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1323                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1324         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1325                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1326         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1327                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1328         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1329                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1330         if (lif->hw_features & IONIC_ETH_HW_TSO)
1331                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1332         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1333                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1334         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1335                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1336         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1337                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1338         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1339                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1340         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1341                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1342         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1343                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1344         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1345                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1346         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1347                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1348
1349         return 0;
1350 }
1351
1352 int
1353 ionic_lif_txq_init(struct ionic_qcq *qcq)
1354 {
1355         struct ionic_queue *q = &qcq->q;
1356         struct ionic_lif *lif = qcq->lif;
1357         struct ionic_cq *cq = &qcq->cq;
1358         struct ionic_admin_ctx ctx = {
1359                 .pending_work = true,
1360                 .cmd.q_init = {
1361                         .opcode = IONIC_CMD_Q_INIT,
1362                         .type = q->type,
1363                         .index = q->index,
1364                         .flags = IONIC_QINIT_F_SG | IONIC_QINIT_F_ENA,
1365                         .intr_index = IONIC_INTR_NONE,
1366                         .ring_size = rte_log2_u32(q->num_descs),
1367                         .ring_base = q->base_pa,
1368                         .cq_ring_base = cq->base_pa,
1369                         .sg_ring_base = q->sg_base_pa,
1370                 },
1371         };
1372         int err;
1373
1374         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1375         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1376                 ctx.cmd.q_init.ring_base);
1377         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1378                 ctx.cmd.q_init.ring_size);
1379         IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver);
1380
1381         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1382         if (err)
1383                 return err;
1384
1385         q->hw_type = ctx.comp.q_init.hw_type;
1386         q->hw_index = ctx.comp.q_init.hw_index;
1387         q->db = ionic_db_map(lif, q);
1388
1389         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1390         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1391         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1392
1393         qcq->flags |= IONIC_QCQ_F_INITED;
1394
1395         return 0;
1396 }
1397
1398 int
1399 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1400 {
1401         struct ionic_queue *q = &qcq->q;
1402         struct ionic_lif *lif = qcq->lif;
1403         struct ionic_cq *cq = &qcq->cq;
1404         struct ionic_admin_ctx ctx = {
1405                 .pending_work = true,
1406                 .cmd.q_init = {
1407                         .opcode = IONIC_CMD_Q_INIT,
1408                         .type = q->type,
1409                         .index = q->index,
1410                         .flags = IONIC_QINIT_F_SG | IONIC_QINIT_F_ENA,
1411                         .intr_index = IONIC_INTR_NONE,
1412                         .ring_size = rte_log2_u32(q->num_descs),
1413                         .ring_base = q->base_pa,
1414                         .cq_ring_base = cq->base_pa,
1415                         .sg_ring_base = q->sg_base_pa,
1416                 },
1417         };
1418         int err;
1419
1420         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1421         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1422                 ctx.cmd.q_init.ring_base);
1423         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1424                 ctx.cmd.q_init.ring_size);
1425         IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver);
1426
1427         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1428         if (err)
1429                 return err;
1430
1431         q->hw_type = ctx.comp.q_init.hw_type;
1432         q->hw_index = ctx.comp.q_init.hw_index;
1433         q->db = ionic_db_map(lif, q);
1434
1435         qcq->flags |= IONIC_QCQ_F_INITED;
1436
1437         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1438         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1439         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1440
1441         return 0;
1442 }
1443
1444 static int
1445 ionic_station_set(struct ionic_lif *lif)
1446 {
1447         struct ionic_admin_ctx ctx = {
1448                 .pending_work = true,
1449                 .cmd.lif_getattr = {
1450                         .opcode = IONIC_CMD_LIF_GETATTR,
1451                         .attr = IONIC_LIF_ATTR_MAC,
1452                 },
1453         };
1454         int err;
1455
1456         IONIC_PRINT_CALL();
1457
1458         err = ionic_adminq_post_wait(lif, &ctx);
1459         if (err)
1460                 return err;
1461
1462         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1463
1464         return 0;
1465 }
1466
1467 static void
1468 ionic_lif_set_name(struct ionic_lif *lif)
1469 {
1470         struct ionic_admin_ctx ctx = {
1471                 .pending_work = true,
1472                 .cmd.lif_setattr = {
1473                         .opcode = IONIC_CMD_LIF_SETATTR,
1474                         .attr = IONIC_LIF_ATTR_NAME,
1475                 },
1476         };
1477
1478         memcpy(ctx.cmd.lif_setattr.name, lif->name,
1479                 sizeof(ctx.cmd.lif_setattr.name) - 1);
1480
1481         ionic_adminq_post_wait(lif, &ctx);
1482 }
1483
1484 int
1485 ionic_lif_init(struct ionic_lif *lif)
1486 {
1487         struct ionic_dev *idev = &lif->adapter->idev;
1488         struct ionic_q_init_comp comp;
1489         int err;
1490
1491         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1492
1493         ionic_dev_cmd_lif_init(idev, lif->info_pa);
1494         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1495         ionic_dev_cmd_comp(idev, &comp);
1496         if (err)
1497                 return err;
1498
1499         lif->hw_index = comp.hw_index;
1500
1501         err = ionic_lif_adminq_init(lif);
1502         if (err)
1503                 return err;
1504
1505         err = ionic_lif_notifyq_init(lif);
1506         if (err)
1507                 goto err_out_adminq_deinit;
1508
1509         /*
1510          * Configure initial feature set
1511          * This will be updated later by the dev_configure() step
1512          */
1513         lif->features = IONIC_ETH_HW_RX_HASH | IONIC_ETH_HW_VLAN_RX_FILTER;
1514
1515         err = ionic_lif_set_features(lif);
1516         if (err)
1517                 goto err_out_notifyq_deinit;
1518
1519         err = ionic_rx_filters_init(lif);
1520         if (err)
1521                 goto err_out_notifyq_deinit;
1522
1523         err = ionic_station_set(lif);
1524         if (err)
1525                 goto err_out_rx_filter_deinit;
1526
1527         ionic_lif_set_name(lif);
1528
1529         lif->state |= IONIC_LIF_F_INITED;
1530
1531         return 0;
1532
1533 err_out_rx_filter_deinit:
1534         ionic_rx_filters_deinit(lif);
1535
1536 err_out_notifyq_deinit:
1537         ionic_lif_notifyq_deinit(lif);
1538
1539 err_out_adminq_deinit:
1540         ionic_lif_qcq_deinit(lif->adminqcq);
1541
1542         return err;
1543 }
1544
1545 void
1546 ionic_lif_deinit(struct ionic_lif *lif)
1547 {
1548         if (!(lif->state & IONIC_LIF_F_INITED))
1549                 return;
1550
1551         ionic_rx_filters_deinit(lif);
1552         ionic_lif_rss_teardown(lif);
1553         ionic_lif_notifyq_deinit(lif);
1554         ionic_lif_qcq_deinit(lif->adminqcq);
1555
1556         lif->state &= ~IONIC_LIF_F_INITED;
1557 }
1558
1559 void
1560 ionic_lif_configure_vlan_offload(struct ionic_lif *lif, int mask)
1561 {
1562         struct rte_eth_dev *eth_dev = lif->eth_dev;
1563         struct rte_eth_rxmode *rxmode = &eth_dev->data->dev_conf.rxmode;
1564
1565         /*
1566          * IONIC_ETH_HW_VLAN_RX_FILTER cannot be turned off, so
1567          * set DEV_RX_OFFLOAD_VLAN_FILTER and ignore ETH_VLAN_FILTER_MASK
1568          */
1569         rxmode->offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
1570
1571         if (mask & ETH_VLAN_STRIP_MASK) {
1572                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
1573                         lif->features |= IONIC_ETH_HW_VLAN_RX_STRIP;
1574                 else
1575                         lif->features &= ~IONIC_ETH_HW_VLAN_RX_STRIP;
1576         }
1577 }
1578
1579 void
1580 ionic_lif_configure(struct ionic_lif *lif)
1581 {
1582         struct rte_eth_rxmode *rxmode = &lif->eth_dev->data->dev_conf.rxmode;
1583         struct rte_eth_txmode *txmode = &lif->eth_dev->data->dev_conf.txmode;
1584         struct ionic_identity *ident = &lif->adapter->ident;
1585         uint32_t ntxqs_per_lif =
1586                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1587         uint32_t nrxqs_per_lif =
1588                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1589         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1590         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1591
1592         lif->port_id = lif->eth_dev->data->port_id;
1593
1594         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1595                 lif->port_id);
1596
1597         if (nrxqs > 0)
1598                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1599
1600         if (ntxqs > 0)
1601                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1602
1603         lif->nrxqcqs = nrxqs_per_lif;
1604         lif->ntxqcqs = ntxqs_per_lif;
1605
1606         /* Update the LIF configuration based on the eth_dev */
1607
1608         /*
1609          * NB: While it is true that RSS_HASH is always enabled on ionic,
1610          *     setting this flag unconditionally causes problems in DTS.
1611          * rxmode->offloads |= DEV_RX_OFFLOAD_RSS_HASH;
1612          */
1613
1614         /* RX per-port */
1615
1616         if (rxmode->offloads & DEV_RX_OFFLOAD_IPV4_CKSUM ||
1617             rxmode->offloads & DEV_RX_OFFLOAD_UDP_CKSUM ||
1618             rxmode->offloads & DEV_RX_OFFLOAD_TCP_CKSUM)
1619                 lif->features |= IONIC_ETH_HW_RX_CSUM;
1620         else
1621                 lif->features &= ~IONIC_ETH_HW_RX_CSUM;
1622
1623         if (rxmode->offloads & DEV_RX_OFFLOAD_SCATTER) {
1624                 lif->features |= IONIC_ETH_HW_RX_SG;
1625                 lif->eth_dev->data->scattered_rx = 1;
1626         } else {
1627                 lif->features &= ~IONIC_ETH_HW_RX_SG;
1628                 lif->eth_dev->data->scattered_rx = 0;
1629         }
1630
1631         /* Covers VLAN_STRIP */
1632         ionic_lif_configure_vlan_offload(lif, ETH_VLAN_STRIP_MASK);
1633
1634         /* TX per-port */
1635
1636         if (txmode->offloads & DEV_TX_OFFLOAD_IPV4_CKSUM ||
1637             txmode->offloads & DEV_TX_OFFLOAD_UDP_CKSUM ||
1638             txmode->offloads & DEV_TX_OFFLOAD_TCP_CKSUM ||
1639             txmode->offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM ||
1640             txmode->offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM)
1641                 lif->features |= IONIC_ETH_HW_TX_CSUM;
1642         else
1643                 lif->features &= ~IONIC_ETH_HW_TX_CSUM;
1644
1645         if (txmode->offloads & DEV_TX_OFFLOAD_VLAN_INSERT)
1646                 lif->features |= IONIC_ETH_HW_VLAN_TX_TAG;
1647         else
1648                 lif->features &= ~IONIC_ETH_HW_VLAN_TX_TAG;
1649
1650         if (txmode->offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
1651                 lif->features |= IONIC_ETH_HW_TX_SG;
1652         else
1653                 lif->features &= ~IONIC_ETH_HW_TX_SG;
1654
1655         if (txmode->offloads & DEV_TX_OFFLOAD_TCP_TSO) {
1656                 lif->features |= IONIC_ETH_HW_TSO;
1657                 lif->features |= IONIC_ETH_HW_TSO_IPV6;
1658                 lif->features |= IONIC_ETH_HW_TSO_ECN;
1659         } else {
1660                 lif->features &= ~IONIC_ETH_HW_TSO;
1661                 lif->features &= ~IONIC_ETH_HW_TSO_IPV6;
1662                 lif->features &= ~IONIC_ETH_HW_TSO_ECN;
1663         }
1664 }
1665
1666 int
1667 ionic_lif_start(struct ionic_lif *lif)
1668 {
1669         uint32_t rx_mode;
1670         uint32_t i;
1671         int err;
1672
1673         err = ionic_lif_rss_setup(lif);
1674         if (err)
1675                 return err;
1676
1677         if (!lif->rx_mode) {
1678                 IONIC_PRINT(DEBUG, "Setting RX mode on %s",
1679                         lif->name);
1680
1681                 rx_mode  = IONIC_RX_MODE_F_UNICAST;
1682                 rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1683                 rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1684
1685                 ionic_set_rx_mode(lif, rx_mode);
1686         }
1687
1688         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1689                 "on port %u",
1690                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1691
1692         for (i = 0; i < lif->nrxqcqs; i++) {
1693                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1694                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1695                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1696
1697                         if (err)
1698                                 return err;
1699                 }
1700         }
1701
1702         for (i = 0; i < lif->ntxqcqs; i++) {
1703                 struct ionic_qcq *txq = lif->txqcqs[i];
1704                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1705                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1706
1707                         if (err)
1708                                 return err;
1709                 }
1710         }
1711
1712         /* Carrier ON here */
1713         lif->state |= IONIC_LIF_F_UP;
1714
1715         ionic_link_status_check(lif);
1716
1717         return 0;
1718 }
1719
1720 int
1721 ionic_lif_identify(struct ionic_adapter *adapter)
1722 {
1723         struct ionic_dev *idev = &adapter->idev;
1724         struct ionic_identity *ident = &adapter->ident;
1725         int err;
1726         unsigned int i;
1727         unsigned int lif_words = sizeof(ident->lif.words) /
1728                 sizeof(ident->lif.words[0]);
1729         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1730                 sizeof(idev->dev_cmd->data[0]);
1731         unsigned int nwords;
1732
1733         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1734                 IONIC_IDENTITY_VERSION_1);
1735         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1736         if (err)
1737                 return (err);
1738
1739         nwords = RTE_MIN(lif_words, cmd_words);
1740         for (i = 0; i < nwords; i++)
1741                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1742
1743         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1744                 ident->lif.capabilities);
1745
1746         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1747                 ident->lif.eth.max_ucast_filters);
1748         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1749                 ident->lif.eth.max_mcast_filters);
1750
1751         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1752                 ident->lif.eth.config.features);
1753         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1754                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1755         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1756                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1757         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1758                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1759         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1760                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1761
1762         return 0;
1763 }
1764
1765 int
1766 ionic_lifs_size(struct ionic_adapter *adapter)
1767 {
1768         struct ionic_identity *ident = &adapter->ident;
1769         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1770
1771         adapter->max_ntxqs_per_lif =
1772                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1773         adapter->max_nrxqs_per_lif =
1774                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1775
1776         nintrs = 1 /* notifyq */;
1777
1778         if (nintrs > dev_nintrs) {
1779                 IONIC_PRINT(ERR,
1780                         "At most %d intr supported, minimum req'd is %u",
1781                         dev_nintrs, nintrs);
1782                 return -ENOSPC;
1783         }
1784
1785         adapter->nintrs = nintrs;
1786
1787         return 0;
1788 }