drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .type = q->type,
  29                         .index = q->index,
  30                         .oper = IONIC_Q_ENABLE,
  31                 },
  32         };
  33
  34         if (qcq->flags & IONIC_QCQ_F_INTR) {
  35                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  36                         IONIC_INTR_MASK_CLEAR);
  37         }
  38
  39         return ionic_adminq_post_wait(lif, &ctx);
  40 }
  41
  42 int
  43 ionic_qcq_disable(struct ionic_qcq *qcq)
  44 {
  45         struct ionic_queue *q = &qcq->q;
  46         struct ionic_lif *lif = q->lif;
  47         struct ionic_dev *idev = &lif->adapter->idev;
  48         struct ionic_admin_ctx ctx = {
  49                 .pending_work = true,
  50                 .cmd.q_control = {
  51                         .opcode = IONIC_CMD_Q_CONTROL,
  52                         .type = q->type,
  53                         .index = q->index,
  54                         .oper = IONIC_Q_DISABLE,
  55                 },
  56         };
  57
  58         if (qcq->flags & IONIC_QCQ_F_INTR) {
  59                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  60                         IONIC_INTR_MASK_SET);
  61         }
  62
  63         return ionic_adminq_post_wait(lif, &ctx);
  64 }
  65
  66 int
  67 ionic_lif_stop(struct ionic_lif *lif __rte_unused)
  68 {
  69         /* Carrier OFF here */
  70
  71         return 0;
  72 }
  73
  74 void
  75 ionic_lif_reset(struct ionic_lif *lif)
  76 {
  77         struct ionic_dev *idev = &lif->adapter->idev;
  78         int err;
  79
  80         IONIC_PRINT_CALL();
  81
  82         ionic_dev_cmd_lif_reset(idev);
  83         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  84         if (err)
  85                 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name);
  86 }
  87
  88 static void
  89 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  90 {
  91         struct ionic_lif_stats *ls = &lif->info->stats;
  92         uint32_t i;
  93         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  94                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  95         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  96                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  97
  98         memset(stats, 0, sizeof(*stats));
  99
 100         if (ls == NULL) {
 101                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 102                         lif->port_id);
 103                 return;
 104         }
 105
 106         /* RX */
 107
 108         stats->ipackets = ls->rx_ucast_packets +
 109                 ls->rx_mcast_packets +
 110                 ls->rx_bcast_packets;
 111
 112         stats->ibytes = ls->rx_ucast_bytes +
 113                 ls->rx_mcast_bytes +
 114                 ls->rx_bcast_bytes;
 115
 116         for (i = 0; i < lif->nrxqcqs; i++) {
 117                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 118                 stats->imissed +=
 119                         rx_stats->no_cb_arg +
 120                         rx_stats->bad_cq_status +
 121                         rx_stats->no_room +
 122                         rx_stats->bad_len;
 123         }
 124
 125         stats->imissed +=
 126                 ls->rx_ucast_drop_packets +
 127                 ls->rx_mcast_drop_packets +
 128                 ls->rx_bcast_drop_packets;
 129
 130         stats->imissed +=
 131                 ls->rx_queue_empty +
 132                 ls->rx_dma_error +
 133                 ls->rx_queue_disabled +
 134                 ls->rx_desc_fetch_error +
 135                 ls->rx_desc_data_error;
 136
 137         for (i = 0; i < num_rx_q_counters; i++) {
 138                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 139                 stats->q_ipackets[i] = rx_stats->packets;
 140                 stats->q_ibytes[i] = rx_stats->bytes;
 141                 stats->q_errors[i] =
 142                         rx_stats->no_cb_arg +
 143                         rx_stats->bad_cq_status +
 144                         rx_stats->no_room +
 145                         rx_stats->bad_len;
 146         }
 147
 148         /* TX */
 149
 150         stats->opackets = ls->tx_ucast_packets +
 151                 ls->tx_mcast_packets +
 152                 ls->tx_bcast_packets;
 153
 154         stats->obytes = ls->tx_ucast_bytes +
 155                 ls->tx_mcast_bytes +
 156                 ls->tx_bcast_bytes;
 157
 158         for (i = 0; i < lif->ntxqcqs; i++) {
 159                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 160                 stats->oerrors += tx_stats->drop;
 161         }
 162
 163         stats->oerrors +=
 164                 ls->tx_ucast_drop_packets +
 165                 ls->tx_mcast_drop_packets +
 166                 ls->tx_bcast_drop_packets;
 167
 168         stats->oerrors +=
 169                 ls->tx_dma_error +
 170                 ls->tx_queue_disabled +
 171                 ls->tx_desc_fetch_error +
 172                 ls->tx_desc_data_error;
 173
 174         for (i = 0; i < num_tx_q_counters; i++) {
 175                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 176                 stats->q_opackets[i] = tx_stats->packets;
 177                 stats->q_obytes[i] = tx_stats->bytes;
 178         }
 179 }
 180
 181 void
 182 ionic_lif_get_stats(const struct ionic_lif *lif,
 183                 struct rte_eth_stats *stats)
 184 {
 185         ionic_lif_get_abs_stats(lif, stats);
 186
 187         stats->ipackets  -= lif->stats_base.ipackets;
 188         stats->opackets  -= lif->stats_base.opackets;
 189         stats->ibytes    -= lif->stats_base.ibytes;
 190         stats->obytes    -= lif->stats_base.obytes;
 191         stats->imissed   -= lif->stats_base.imissed;
 192         stats->ierrors   -= lif->stats_base.ierrors;
 193         stats->oerrors   -= lif->stats_base.oerrors;
 194         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 195 }
 196
 197 void
 198 ionic_lif_reset_stats(struct ionic_lif *lif)
 199 {
 200         uint32_t i;
 201
 202         for (i = 0; i < lif->nrxqcqs; i++) {
 203                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 204                         sizeof(struct ionic_rx_stats));
 205                 memset(&lif->txqcqs[i]->stats.tx, 0,
 206                         sizeof(struct ionic_tx_stats));
 207         }
 208
 209         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 210 }
 211
 212 void
 213 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 214 {
 215         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 216         uint64_t *stats64 = (uint64_t *)stats;
 217         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 218         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 219
 220         for (i = 0; i < count; i++)
 221                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 222 }
 223
 224 void
 225 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 226 {
 227         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 228         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 229         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 230
 231         for (i = 0; i < count; i++)
 232                 lif_stats64_base[i] = lif_stats64[i];
 233 }
 234
 235 static int
 236 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 237 {
 238         struct ionic_admin_ctx ctx = {
 239                 .pending_work = true,
 240                 .cmd.rx_filter_add = {
 241                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 242                         .match = IONIC_RX_FILTER_MATCH_MAC,
 243                 },
 244         };
 245         int err;
 246
 247         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 248
 249         err = ionic_adminq_post_wait(lif, &ctx);
 250         if (err)
 251                 return err;
 252
 253         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 254                 ctx.comp.rx_filter_add.filter_id);
 255
 256         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 257 }
 258
 259 static int
 260 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 261 {
 262         struct ionic_admin_ctx ctx = {
 263                 .pending_work = true,
 264                 .cmd.rx_filter_del = {
 265                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 266                 },
 267         };
 268         struct ionic_rx_filter *f;
 269         int err;
 270
 271         IONIC_PRINT_CALL();
 272
 273         rte_spinlock_lock(&lif->rx_filters.lock);
 274
 275         f = ionic_rx_filter_by_addr(lif, addr);
 276         if (!f) {
 277                 rte_spinlock_unlock(&lif->rx_filters.lock);
 278                 return -ENOENT;
 279         }
 280
 281         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 282         ionic_rx_filter_free(f);
 283
 284         rte_spinlock_unlock(&lif->rx_filters.lock);
 285
 286         err = ionic_adminq_post_wait(lif, &ctx);
 287         if (err)
 288                 return err;
 289
 290         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 291                 ctx.cmd.rx_filter_del.filter_id);
 292
 293         return 0;
 294 }
 295
 296 int
 297 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 298                 struct rte_ether_addr *mac_addr,
 299                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 300 {
 301         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 302
 303         IONIC_PRINT_CALL();
 304
 305         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 306 }
 307
 308 void
 309 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index)
 310 {
 311         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 312         struct ionic_adapter *adapter = lif->adapter;
 313         struct rte_ether_addr *mac_addr;
 314
 315         IONIC_PRINT_CALL();
 316
 317         if (index >= adapter->max_mac_addrs) {
 318                 IONIC_PRINT(WARNING,
 319                         "Index %u is above MAC filter limit %u",
 320                         index, adapter->max_mac_addrs);
 321                 return;
 322         }
 323
 324         mac_addr = &eth_dev->data->mac_addrs[index];
 325
 326         if (!rte_is_valid_assigned_ether_addr(mac_addr))
 327                 return;
 328
 329         ionic_lif_addr_del(lif, (const uint8_t *)mac_addr);
 330 }
 331
 332 int
 333 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 334 {
 335         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 336
 337         IONIC_PRINT_CALL();
 338
 339         if (mac_addr == NULL) {
 340                 IONIC_PRINT(NOTICE, "New mac is null");
 341                 return -1;
 342         }
 343
 344         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 345                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 346                         lif->mac_addr);
 347                 ionic_lif_addr_del(lif, lif->mac_addr);
 348                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 349         }
 350
 351         IONIC_PRINT(INFO, "Updating mac addr");
 352
 353         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 354
 355         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 356 }
 357
 358 static int
 359 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 360 {
 361         struct ionic_admin_ctx ctx = {
 362                 .pending_work = true,
 363                 .cmd.rx_filter_add = {
 364                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 365                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 366                         .vlan.vlan = vid,
 367                 },
 368         };
 369         int err;
 370
 371         err = ionic_adminq_post_wait(lif, &ctx);
 372         if (err)
 373                 return err;
 374
 375         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 376                 ctx.comp.rx_filter_add.filter_id);
 377
 378         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 379 }
 380
 381 static int
 382 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 383 {
 384         struct ionic_admin_ctx ctx = {
 385                 .pending_work = true,
 386                 .cmd.rx_filter_del = {
 387                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 388                 },
 389         };
 390         struct ionic_rx_filter *f;
 391         int err;
 392
 393         IONIC_PRINT_CALL();
 394
 395         rte_spinlock_lock(&lif->rx_filters.lock);
 396
 397         f = ionic_rx_filter_by_vlan(lif, vid);
 398         if (!f) {
 399                 rte_spinlock_unlock(&lif->rx_filters.lock);
 400                 return -ENOENT;
 401         }
 402
 403         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 404         ionic_rx_filter_free(f);
 405         rte_spinlock_unlock(&lif->rx_filters.lock);
 406
 407         err = ionic_adminq_post_wait(lif, &ctx);
 408         if (err)
 409                 return err;
 410
 411         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 412                 ctx.cmd.rx_filter_del.filter_id);
 413
 414         return 0;
 415 }
 416
 417 int
 418 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 419                 int on)
 420 {
 421         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 422         int err;
 423
 424         if (on)
 425                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 426         else
 427                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 428
 429         return err;
 430 }
 431
 432 static void
 433 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 434 {
 435         struct ionic_admin_ctx ctx = {
 436                 .pending_work = true,
 437                 .cmd.rx_mode_set = {
 438                         .opcode = IONIC_CMD_RX_MODE_SET,
 439                         .rx_mode = rx_mode,
 440                 },
 441         };
 442         int err;
 443
 444         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 445                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 446         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 447                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 448         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 449                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 450         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 451                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 452         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 453                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 454
 455         err = ionic_adminq_post_wait(lif, &ctx);
 456         if (err)
 457                 IONIC_PRINT(ERR, "Failure setting RX mode");
 458 }
 459
 460 static void
 461 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 462 {
 463         if (lif->rx_mode != rx_mode) {
 464                 lif->rx_mode = rx_mode;
 465                 ionic_lif_rx_mode(lif, rx_mode);
 466         }
 467 }
 468
 469 int
 470 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 471 {
 472         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 473         uint32_t rx_mode = lif->rx_mode;
 474
 475         IONIC_PRINT_CALL();
 476
 477         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 478
 479         ionic_set_rx_mode(lif, rx_mode);
 480
 481         return 0;
 482 }
 483
 484 int
 485 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 486 {
 487         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 488         uint32_t rx_mode = lif->rx_mode;
 489
 490         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 491
 492         ionic_set_rx_mode(lif, rx_mode);
 493
 494         return 0;
 495 }
 496
 497 int
 498 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 499 {
 500         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 501         uint32_t rx_mode = lif->rx_mode;
 502
 503         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 504
 505         ionic_set_rx_mode(lif, rx_mode);
 506
 507         return 0;
 508 }
 509
 510 int
 511 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 512 {
 513         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 514         uint32_t rx_mode = lif->rx_mode;
 515
 516         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 517
 518         ionic_set_rx_mode(lif, rx_mode);
 519
 520         return 0;
 521 }
 522
 523 int
 524 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 525 {
 526         struct ionic_admin_ctx ctx = {
 527                 .pending_work = true,
 528                 .cmd.lif_setattr = {
 529                         .opcode = IONIC_CMD_LIF_SETATTR,
 530                         .attr = IONIC_LIF_ATTR_MTU,
 531                         .mtu = new_mtu,
 532                 },
 533         };
 534         int err;
 535
 536         err = ionic_adminq_post_wait(lif, &ctx);
 537         if (err)
 538                 return err;
 539
 540         return 0;
 541 }
 542
 543 int
 544 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 545 {
 546         struct ionic_adapter *adapter = lif->adapter;
 547         struct ionic_dev *idev = &adapter->idev;
 548         unsigned long index;
 549
 550         /*
 551          * Note: interrupt handler is called for index = 0 only
 552          * (we use interrupts for the notifyq only anyway,
 553          * which has index = 0)
 554          */
 555
 556         for (index = 0; index < adapter->nintrs; index++)
 557                 if (!adapter->intrs[index])
 558                         break;
 559
 560         if (index == adapter->nintrs)
 561                 return -ENOSPC;
 562
 563         adapter->intrs[index] = true;
 564
 565         ionic_intr_init(idev, intr, index);
 566
 567         return 0;
 568 }
 569
 570 void
 571 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 572 {
 573         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 574                 lif->adapter->intrs[intr->index] = false;
 575 }
 576
 577 static int
 578 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 579                 uint32_t index,
 580                 const char *base, uint32_t flags,
 581                 uint32_t num_descs,
 582                 uint32_t desc_size,
 583                 uint32_t cq_desc_size,
 584                 uint32_t sg_desc_size,
 585                 struct ionic_qcq **qcq)
 586 {
 587         struct ionic_dev *idev = &lif->adapter->idev;
 588         struct ionic_qcq *new;
 589         uint32_t q_size, cq_size, sg_size, total_size;
 590         void *q_base, *cq_base, *sg_base;
 591         rte_iova_t q_base_pa = 0;
 592         rte_iova_t cq_base_pa = 0;
 593         rte_iova_t sg_base_pa = 0;
 594         uint32_t socket_id = rte_socket_id();
 595         int err;
 596
 597         *qcq = NULL;
 598
 599         q_size  = num_descs * desc_size;
 600         cq_size = num_descs * cq_desc_size;
 601         sg_size = num_descs * sg_desc_size;
 602
 603         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 604                 RTE_ALIGN(cq_size, PAGE_SIZE);
 605         /*
 606          * Note: aligning q_size/cq_size is not enough due to cq_base address
 607          * aligning as q_base could be not aligned to the page.
 608          * Adding PAGE_SIZE.
 609          */
 610         total_size += PAGE_SIZE;
 611
 612         if (flags & IONIC_QCQ_F_SG) {
 613                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 614                 total_size += PAGE_SIZE;
 615         }
 616
 617         new = rte_zmalloc("ionic", sizeof(*new), 0);
 618         if (!new) {
 619                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 620                 return -ENOMEM;
 621         }
 622
 623         new->lif = lif;
 624         new->flags = flags;
 625
 626         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 627         if (!new->q.info) {
 628                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 629                 return -ENOMEM;
 630         }
 631
 632         new->q.type = type;
 633
 634         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 635                 desc_size, sg_desc_size);
 636         if (err) {
 637                 IONIC_PRINT(ERR, "Queue initialization failed");
 638                 return err;
 639         }
 640
 641         if (flags & IONIC_QCQ_F_INTR) {
 642                 err = ionic_intr_alloc(lif, &new->intr);
 643                 if (err)
 644                         return err;
 645
 646                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 647                         IONIC_INTR_MASK_SET);
 648         } else {
 649                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 650         }
 651
 652         err = ionic_cq_init(lif, &new->cq, &new->intr,
 653                 num_descs, cq_desc_size);
 654         if (err) {
 655                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 656                 goto err_out_free_intr;
 657         }
 658
 659         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 660                 base /* name */, index /* queue_idx */,
 661                 total_size, IONIC_ALIGN, socket_id);
 662
 663         if (!new->base_z) {
 664                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 665                 err = -ENOMEM;
 666                 goto err_out_free_intr;
 667         }
 668
 669         new->base = new->base_z->addr;
 670         new->base_pa = new->base_z->iova;
 671         new->total_size = total_size;
 672
 673         q_base = new->base;
 674         q_base_pa = new->base_pa;
 675
 676         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 677         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 678
 679         if (flags & IONIC_QCQ_F_SG) {
 680                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 681                         PAGE_SIZE);
 682                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 683                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 684         }
 685
 686         IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 687                 "SG-base-PA = %#jx",
 688                 q_base_pa, cq_base_pa, sg_base_pa);
 689
 690         ionic_q_map(&new->q, q_base, q_base_pa);
 691         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 692         ionic_cq_bind(&new->cq, &new->q);
 693
 694         *qcq = new;
 695
 696         return 0;
 697
 698 err_out_free_intr:
 699         if (flags & IONIC_QCQ_F_INTR)
 700                 ionic_intr_free(lif, &new->intr);
 701
 702         return err;
 703 }
 704
 705 void
 706 ionic_qcq_free(struct ionic_qcq *qcq)
 707 {
 708         if (qcq->base_z) {
 709                 qcq->base = NULL;
 710                 qcq->base_pa = 0;
 711                 rte_memzone_free(qcq->base_z);
 712                 qcq->base_z = NULL;
 713         }
 714
 715         if (qcq->q.info) {
 716                 rte_free(qcq->q.info);
 717                 qcq->q.info = NULL;
 718         }
 719
 720         rte_free(qcq);
 721 }
 722
 723 int
 724 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 725                 struct ionic_qcq **qcq)
 726 {
 727         uint32_t flags;
 728         int err = -ENOMEM;
 729
 730         flags = IONIC_QCQ_F_SG;
 731         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 732                 nrxq_descs,
 733                 sizeof(struct ionic_rxq_desc),
 734                 sizeof(struct ionic_rxq_comp),
 735                 sizeof(struct ionic_rxq_sg_desc),
 736                 &lif->rxqcqs[index]);
 737         if (err)
 738                 return err;
 739
 740         *qcq = lif->rxqcqs[index];
 741
 742         return 0;
 743 }
 744
 745 int
 746 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 747                 struct ionic_qcq **qcq)
 748 {
 749         uint32_t flags;
 750         int err = -ENOMEM;
 751
 752         flags = IONIC_QCQ_F_SG;
 753         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 754                 ntxq_descs,
 755                 sizeof(struct ionic_txq_desc),
 756                 sizeof(struct ionic_txq_comp),
 757                 sizeof(struct ionic_txq_sg_desc),
 758                 &lif->txqcqs[index]);
 759         if (err)
 760                 return err;
 761
 762         *qcq = lif->txqcqs[index];
 763
 764         return 0;
 765 }
 766
 767 static int
 768 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 769 {
 770         uint32_t flags;
 771         int err = -ENOMEM;
 772
 773         flags = 0;
 774         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 775                 IONIC_ADMINQ_LENGTH,
 776                 sizeof(struct ionic_admin_cmd),
 777                 sizeof(struct ionic_admin_comp),
 778                 0,
 779                 &lif->adminqcq);
 780         if (err)
 781                 return err;
 782
 783         return 0;
 784 }
 785
 786 static int
 787 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 788 {
 789         uint32_t flags;
 790         int err = -ENOMEM;
 791
 792         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 793
 794         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 795                 flags,
 796                 IONIC_NOTIFYQ_LENGTH,
 797                 sizeof(struct ionic_notifyq_cmd),
 798                 sizeof(union ionic_notifyq_comp),
 799                 0,
 800                 &lif->notifyqcq);
 801         if (err)
 802                 return err;
 803
 804         return 0;
 805 }
 806
 807 static void *
 808 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 809 {
 810         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 811
 812         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 813                 return NULL;
 814
 815         return (void *)&vaddr[page_num << PAGE_SHIFT];
 816 }
 817
 818 int
 819 ionic_lif_alloc(struct ionic_lif *lif)
 820 {
 821         struct ionic_adapter *adapter = lif->adapter;
 822         uint32_t socket_id = rte_socket_id();
 823         int err;
 824
 825         /*
 826          * lif->name was zeroed on allocation.
 827          * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated.
 828          */
 829         memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1);
 830
 831         IONIC_PRINT(DEBUG, "LIF: %s", lif->name);
 832
 833         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 834
 835         rte_spinlock_init(&lif->adminq_lock);
 836         rte_spinlock_init(&lif->adminq_service_lock);
 837
 838         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, 0);
 839         if (!lif->kern_dbpage) {
 840                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 841                 return -ENOMEM;
 842         }
 843
 844         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 845                 adapter->max_ntxqs_per_lif, 0);
 846
 847         if (!lif->txqcqs) {
 848                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 849                 return -ENOMEM;
 850         }
 851
 852         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 853                 adapter->max_nrxqs_per_lif, 0);
 854
 855         if (!lif->rxqcqs) {
 856                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 857                 return -ENOMEM;
 858         }
 859
 860         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 861
 862         err = ionic_notify_qcq_alloc(lif);
 863         if (err) {
 864                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 865                 return err;
 866         }
 867
 868         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 869
 870         err = ionic_admin_qcq_alloc(lif);
 871         if (err) {
 872                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 873                 return err;
 874         }
 875
 876         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 877
 878         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 879
 880         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 881                 "lif_info", 0 /* queue_idx*/,
 882                 lif->info_sz, IONIC_ALIGN, socket_id);
 883         if (!lif->info_z) {
 884                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 885                 return -ENOMEM;
 886         }
 887
 888         lif->info = lif->info_z->addr;
 889         lif->info_pa = lif->info_z->iova;
 890
 891         return 0;
 892 }
 893
 894 void
 895 ionic_lif_free(struct ionic_lif *lif)
 896 {
 897         if (lif->notifyqcq) {
 898                 ionic_qcq_free(lif->notifyqcq);
 899                 lif->notifyqcq = NULL;
 900         }
 901
 902         if (lif->adminqcq) {
 903                 ionic_qcq_free(lif->adminqcq);
 904                 lif->adminqcq = NULL;
 905         }
 906
 907         if (lif->txqcqs) {
 908                 rte_free(lif->txqcqs);
 909                 lif->txqcqs = NULL;
 910         }
 911
 912         if (lif->rxqcqs) {
 913                 rte_free(lif->rxqcqs);
 914                 lif->rxqcqs = NULL;
 915         }
 916
 917         if (lif->info) {
 918                 rte_memzone_free(lif->info_z);
 919                 lif->info = NULL;
 920         }
 921 }
 922
 923 int
 924 ionic_lif_rss_config(struct ionic_lif *lif,
 925                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 926 {
 927         struct ionic_admin_ctx ctx = {
 928                 .pending_work = true,
 929                 .cmd.lif_setattr = {
 930                         .opcode = IONIC_CMD_LIF_SETATTR,
 931                         .attr = IONIC_LIF_ATTR_RSS,
 932                         .rss.types = types,
 933                         .rss.addr = lif->rss_ind_tbl_pa,
 934                 },
 935         };
 936         unsigned int i;
 937
 938         IONIC_PRINT_CALL();
 939
 940         lif->rss_types = types;
 941
 942         if (key)
 943                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 944
 945         if (indir)
 946                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 947                         lif->rss_ind_tbl[i] = indir[i];
 948
 949         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 950                IONIC_RSS_HASH_KEY_SIZE);
 951
 952         return ionic_adminq_post_wait(lif, &ctx);
 953 }
 954
 955 static int
 956 ionic_lif_rss_setup(struct ionic_lif *lif)
 957 {
 958         static const uint8_t toeplitz_symmetric_key[] = {
 959                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 960                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 961                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 962                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 963                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 964         };
 965         uint32_t i;
 966         uint16_t tbl_sz = lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 967
 968         IONIC_PRINT_CALL();
 969
 970         if (!lif->rss_ind_tbl_z) {
 971                 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 972                                         "rss_ind_tbl", 0 /* queue_idx */,
 973                                         sizeof(*lif->rss_ind_tbl) * tbl_sz,
 974                                         IONIC_ALIGN, rte_socket_id());
 975                 if (!lif->rss_ind_tbl_z) {
 976                         IONIC_PRINT(ERR, "OOM");
 977                         return -ENOMEM;
 978                 }
 979
 980                 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 981                 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
 982         }
 983
 984         if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) {
 985                 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs;
 986
 987                 /* Fill indirection table with 'default' values */
 988                 for (i = 0; i < tbl_sz; i++)
 989                         lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
 990         }
 991
 992         return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
 993                         toeplitz_symmetric_key, NULL);
 994 }
 995
 996 static void
 997 ionic_lif_rss_teardown(struct ionic_lif *lif)
 998 {
 999         if (!lif->rss_ind_tbl)
1000                 return;
1001
1002         if (lif->rss_ind_tbl_z) {
1003                 /* Disable RSS on the NIC */
1004                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1005
1006                 lif->rss_ind_tbl = NULL;
1007                 lif->rss_ind_tbl_pa = 0;
1008                 rte_memzone_free(lif->rss_ind_tbl_z);
1009                 lif->rss_ind_tbl_z = NULL;
1010         }
1011 }
1012
1013 static void
1014 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1015 {
1016         struct ionic_dev *idev = &lif->adapter->idev;
1017
1018         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1019                 return;
1020
1021         if (qcq->flags & IONIC_QCQ_F_INTR)
1022                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1023                         IONIC_INTR_MASK_SET);
1024
1025         qcq->flags &= ~IONIC_QCQ_F_INITED;
1026 }
1027
1028 void
1029 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1030 {
1031         ionic_lif_qcq_deinit(qcq->lif, qcq);
1032 }
1033
1034 void
1035 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1036 {
1037         ionic_lif_qcq_deinit(qcq->lif, qcq);
1038 }
1039
1040 bool
1041 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1042                 void *cb_arg __rte_unused)
1043 {
1044         struct ionic_admin_comp *cq_desc_base = cq->base;
1045         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1046
1047         if (!color_match(cq_desc->color, cq->done_color))
1048                 return false;
1049
1050         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1051
1052         return true;
1053 }
1054
1055 /* This acts like ionic_napi */
1056 int
1057 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1058                 void *cb_arg)
1059 {
1060         struct ionic_cq *cq = &qcq->cq;
1061         uint32_t work_done;
1062
1063         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1064
1065         return work_done;
1066 }
1067
1068 static void
1069 ionic_link_status_check(struct ionic_lif *lif)
1070 {
1071         struct ionic_adapter *adapter = lif->adapter;
1072         bool link_up;
1073
1074         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1075
1076         if (!lif->info)
1077                 return;
1078
1079         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1080
1081         if ((link_up  && adapter->link_up) ||
1082             (!link_up && !adapter->link_up))
1083                 return;
1084
1085         if (link_up) {
1086                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1087                         lif->info->status.link_speed);
1088                 adapter->link_speed = lif->info->status.link_speed;
1089         } else {
1090                 IONIC_PRINT(DEBUG, "Link down");
1091         }
1092
1093         adapter->link_up = link_up;
1094 }
1095
1096 static bool
1097 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1098 {
1099         union ionic_notifyq_comp *cq_desc_base = cq->base;
1100         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1101         struct ionic_lif *lif = cb_arg;
1102
1103         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1104                 cq_desc->event.eid, cq_desc->event.ecode);
1105
1106         /* Have we run out of new completions to process? */
1107         if (!(cq_desc->event.eid > lif->last_eid))
1108                 return false;
1109
1110         lif->last_eid = cq_desc->event.eid;
1111
1112         switch (cq_desc->event.ecode) {
1113         case IONIC_EVENT_LINK_CHANGE:
1114                 IONIC_PRINT(DEBUG,
1115                         "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
1116                         cq_desc->event.eid,
1117                         cq_desc->link_change.link_status,
1118                         cq_desc->link_change.link_speed);
1119
1120                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1121
1122                 break;
1123         default:
1124                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1125                         cq_desc->event.ecode, cq_desc->event.eid);
1126                 break;
1127         }
1128
1129         return true;
1130 }
1131
1132 int
1133 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1134 {
1135         struct ionic_dev *idev = &lif->adapter->idev;
1136         struct ionic_qcq *qcq = lif->notifyqcq;
1137         uint32_t work_done;
1138
1139         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1140                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1141                 return -1;
1142         }
1143
1144         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1145                 IONIC_INTR_MASK_SET);
1146
1147         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1148
1149         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1150                 ionic_link_status_check(lif);
1151
1152         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1153                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1154
1155         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1156                 IONIC_INTR_MASK_CLEAR);
1157
1158         return 0;
1159 }
1160
1161 static int
1162 ionic_lif_adminq_init(struct ionic_lif *lif)
1163 {
1164         struct ionic_dev *idev = &lif->adapter->idev;
1165         struct ionic_qcq *qcq = lif->adminqcq;
1166         struct ionic_queue *q = &qcq->q;
1167         struct ionic_q_init_comp comp;
1168         int err;
1169
1170         ionic_dev_cmd_adminq_init(idev, qcq, qcq->intr.index);
1171         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1172         if (err)
1173                 return err;
1174
1175         ionic_dev_cmd_comp(idev, &comp);
1176
1177         q->hw_type = comp.hw_type;
1178         q->hw_index = comp.hw_index;
1179         q->db = ionic_db_map(lif, q);
1180
1181         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1182         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1183         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1184
1185         if (qcq->flags & IONIC_QCQ_F_INTR)
1186                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1187                         IONIC_INTR_MASK_CLEAR);
1188
1189         qcq->flags |= IONIC_QCQ_F_INITED;
1190
1191         return 0;
1192 }
1193
1194 static int
1195 ionic_lif_notifyq_init(struct ionic_lif *lif)
1196 {
1197         struct ionic_dev *idev = &lif->adapter->idev;
1198         struct ionic_qcq *qcq = lif->notifyqcq;
1199         struct ionic_queue *q = &qcq->q;
1200         int err;
1201
1202         struct ionic_admin_ctx ctx = {
1203                 .pending_work = true,
1204                 .cmd.q_init = {
1205                         .opcode = IONIC_CMD_Q_INIT,
1206                         .type = q->type,
1207                         .index = q->index,
1208                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1209                         .intr_index = qcq->intr.index,
1210                         .ring_size = rte_log2_u32(q->num_descs),
1211                         .ring_base = q->base_pa,
1212                 }
1213         };
1214
1215         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1216                 ctx.cmd.q_init.index);
1217         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1218                 ctx.cmd.q_init.ring_base);
1219         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1220                 ctx.cmd.q_init.ring_size);
1221         IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver);
1222
1223         err = ionic_adminq_post_wait(lif, &ctx);
1224         if (err)
1225                 return err;
1226
1227         q->hw_type = ctx.comp.q_init.hw_type;
1228         q->hw_index = ctx.comp.q_init.hw_index;
1229         q->db = NULL;
1230
1231         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1232         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1233         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1234
1235         if (qcq->flags & IONIC_QCQ_F_INTR)
1236                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1237                         IONIC_INTR_MASK_CLEAR);
1238
1239         qcq->flags |= IONIC_QCQ_F_INITED;
1240
1241         return 0;
1242 }
1243
1244 int
1245 ionic_lif_set_features(struct ionic_lif *lif)
1246 {
1247         struct ionic_admin_ctx ctx = {
1248                 .pending_work = true,
1249                 .cmd.lif_setattr = {
1250                         .opcode = IONIC_CMD_LIF_SETATTR,
1251                         .attr = IONIC_LIF_ATTR_FEATURES,
1252                         .features = lif->features,
1253                 },
1254         };
1255         int err;
1256
1257         err = ionic_adminq_post_wait(lif, &ctx);
1258         if (err)
1259                 return err;
1260
1261         lif->hw_features = (ctx.cmd.lif_setattr.features &
1262                 ctx.comp.lif_setattr.features);
1263
1264         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1265                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1266         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1267                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1268         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1269                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1270         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1271                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1272         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1273                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1274         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1275                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1276         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1277                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1278         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1279                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1280         if (lif->hw_features & IONIC_ETH_HW_TSO)
1281                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1282         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1283                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1284         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1285                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1286         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1287                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1288         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1289                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1290         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1291                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1292         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1293                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1294         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1295                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1296         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1297                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1298
1299         return 0;
1300 }
1301
1302 int
1303 ionic_lif_txq_init(struct ionic_qcq *qcq)
1304 {
1305         struct ionic_queue *q = &qcq->q;
1306         struct ionic_lif *lif = qcq->lif;
1307         struct ionic_cq *cq = &qcq->cq;
1308         struct ionic_admin_ctx ctx = {
1309                 .pending_work = true,
1310                 .cmd.q_init = {
1311                         .opcode = IONIC_CMD_Q_INIT,
1312                         .type = q->type,
1313                         .index = q->index,
1314                         .flags = IONIC_QINIT_F_SG,
1315                         .intr_index = cq->bound_intr->index,
1316                         .ring_size = rte_log2_u32(q->num_descs),
1317                         .ring_base = q->base_pa,
1318                         .cq_ring_base = cq->base_pa,
1319                         .sg_ring_base = q->sg_base_pa,
1320                 },
1321         };
1322         int err;
1323
1324         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1325         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1326                 ctx.cmd.q_init.ring_base);
1327         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1328                 ctx.cmd.q_init.ring_size);
1329         IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver);
1330
1331         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1332         if (err)
1333                 return err;
1334
1335         q->hw_type = ctx.comp.q_init.hw_type;
1336         q->hw_index = ctx.comp.q_init.hw_index;
1337         q->db = ionic_db_map(lif, q);
1338
1339         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1340         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1341         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1342
1343         qcq->flags |= IONIC_QCQ_F_INITED;
1344
1345         return 0;
1346 }
1347
1348 int
1349 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1350 {
1351         struct ionic_queue *q = &qcq->q;
1352         struct ionic_lif *lif = qcq->lif;
1353         struct ionic_cq *cq = &qcq->cq;
1354         struct ionic_admin_ctx ctx = {
1355                 .pending_work = true,
1356                 .cmd.q_init = {
1357                         .opcode = IONIC_CMD_Q_INIT,
1358                         .type = q->type,
1359                         .index = q->index,
1360                         .flags = IONIC_QINIT_F_SG,
1361                         .intr_index = cq->bound_intr->index,
1362                         .ring_size = rte_log2_u32(q->num_descs),
1363                         .ring_base = q->base_pa,
1364                         .cq_ring_base = cq->base_pa,
1365                         .sg_ring_base = q->sg_base_pa,
1366                 },
1367         };
1368         int err;
1369
1370         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1371         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1372                 ctx.cmd.q_init.ring_base);
1373         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1374                 ctx.cmd.q_init.ring_size);
1375         IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver);
1376
1377         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1378         if (err)
1379                 return err;
1380
1381         q->hw_type = ctx.comp.q_init.hw_type;
1382         q->hw_index = ctx.comp.q_init.hw_index;
1383         q->db = ionic_db_map(lif, q);
1384
1385         qcq->flags |= IONIC_QCQ_F_INITED;
1386
1387         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1388         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1389         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1390
1391         return 0;
1392 }
1393
1394 static int
1395 ionic_station_set(struct ionic_lif *lif)
1396 {
1397         struct ionic_admin_ctx ctx = {
1398                 .pending_work = true,
1399                 .cmd.lif_getattr = {
1400                         .opcode = IONIC_CMD_LIF_GETATTR,
1401                         .attr = IONIC_LIF_ATTR_MAC,
1402                 },
1403         };
1404         int err;
1405
1406         IONIC_PRINT_CALL();
1407
1408         err = ionic_adminq_post_wait(lif, &ctx);
1409         if (err)
1410                 return err;
1411
1412         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1413                         lif->mac_addr)) {
1414                 IONIC_PRINT(INFO, "deleting station MAC addr");
1415
1416                 ionic_lif_addr_del(lif, lif->mac_addr);
1417         }
1418
1419         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1420
1421         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1422                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1423                 return 0;
1424         }
1425
1426         IONIC_PRINT(DEBUG, "adding station MAC addr");
1427
1428         ionic_lif_addr_add(lif, lif->mac_addr);
1429
1430         return 0;
1431 }
1432
1433 static void
1434 ionic_lif_set_name(struct ionic_lif *lif)
1435 {
1436         struct ionic_admin_ctx ctx = {
1437                 .pending_work = true,
1438                 .cmd.lif_setattr = {
1439                         .opcode = IONIC_CMD_LIF_SETATTR,
1440                         .attr = IONIC_LIF_ATTR_NAME,
1441                 },
1442         };
1443
1444         memcpy(ctx.cmd.lif_setattr.name, lif->name,
1445                 sizeof(ctx.cmd.lif_setattr.name) - 1);
1446
1447         ionic_adminq_post_wait(lif, &ctx);
1448 }
1449
1450 int
1451 ionic_lif_init(struct ionic_lif *lif)
1452 {
1453         struct ionic_dev *idev = &lif->adapter->idev;
1454         struct ionic_q_init_comp comp;
1455         int err;
1456
1457         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1458
1459         ionic_dev_cmd_lif_init(idev, lif->info_pa);
1460         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1461         ionic_dev_cmd_comp(idev, &comp);
1462         if (err)
1463                 return err;
1464
1465         lif->hw_index = comp.hw_index;
1466
1467         err = ionic_lif_adminq_init(lif);
1468         if (err)
1469                 return err;
1470
1471         err = ionic_lif_notifyq_init(lif);
1472         if (err)
1473                 goto err_out_adminq_deinit;
1474
1475         lif->features =
1476                   IONIC_ETH_HW_VLAN_TX_TAG
1477                 | IONIC_ETH_HW_VLAN_RX_STRIP
1478                 | IONIC_ETH_HW_VLAN_RX_FILTER
1479                 | IONIC_ETH_HW_RX_HASH
1480                 | IONIC_ETH_HW_TX_SG
1481                 | IONIC_ETH_HW_RX_SG
1482                 | IONIC_ETH_HW_TX_CSUM
1483                 | IONIC_ETH_HW_RX_CSUM
1484                 | IONIC_ETH_HW_TSO
1485                 | IONIC_ETH_HW_TSO_IPV6
1486                 | IONIC_ETH_HW_TSO_ECN;
1487
1488         err = ionic_lif_set_features(lif);
1489         if (err)
1490                 goto err_out_notifyq_deinit;
1491
1492         err = ionic_rx_filters_init(lif);
1493         if (err)
1494                 goto err_out_notifyq_deinit;
1495
1496         err = ionic_station_set(lif);
1497         if (err)
1498                 goto err_out_rx_filter_deinit;
1499
1500         ionic_lif_set_name(lif);
1501
1502         lif->state |= IONIC_LIF_F_INITED;
1503
1504         return 0;
1505
1506 err_out_rx_filter_deinit:
1507         ionic_rx_filters_deinit(lif);
1508
1509 err_out_notifyq_deinit:
1510         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1511
1512 err_out_adminq_deinit:
1513         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1514
1515         return err;
1516 }
1517
1518 void
1519 ionic_lif_deinit(struct ionic_lif *lif)
1520 {
1521         if (!(lif->state & IONIC_LIF_F_INITED))
1522                 return;
1523
1524         ionic_rx_filters_deinit(lif);
1525         ionic_lif_rss_teardown(lif);
1526         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1527         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1528
1529         lif->state &= ~IONIC_LIF_F_INITED;
1530 }
1531
1532 int
1533 ionic_lif_configure(struct ionic_lif *lif)
1534 {
1535         struct ionic_identity *ident = &lif->adapter->ident;
1536         uint32_t ntxqs_per_lif =
1537                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1538         uint32_t nrxqs_per_lif =
1539                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1540         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1541         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1542
1543         lif->port_id = lif->eth_dev->data->port_id;
1544
1545         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1546                 lif->port_id);
1547
1548         if (nrxqs > 0)
1549                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1550
1551         if (ntxqs > 0)
1552                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1553
1554         lif->nrxqcqs = nrxqs_per_lif;
1555         lif->ntxqcqs = ntxqs_per_lif;
1556
1557         return 0;
1558 }
1559
1560 int
1561 ionic_lif_start(struct ionic_lif *lif)
1562 {
1563         uint32_t rx_mode;
1564         uint32_t i;
1565         int err;
1566
1567         err = ionic_lif_rss_setup(lif);
1568         if (err)
1569                 return err;
1570
1571         if (!lif->rx_mode) {
1572                 IONIC_PRINT(DEBUG, "Setting RX mode on %s",
1573                         lif->name);
1574
1575                 rx_mode  = IONIC_RX_MODE_F_UNICAST;
1576                 rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1577                 rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1578
1579                 ionic_set_rx_mode(lif, rx_mode);
1580         }
1581
1582         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1583                 "on port %u",
1584                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1585
1586         for (i = 0; i < lif->nrxqcqs; i++) {
1587                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1588                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1589                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1590
1591                         if (err)
1592                                 return err;
1593                 }
1594         }
1595
1596         for (i = 0; i < lif->ntxqcqs; i++) {
1597                 struct ionic_qcq *txq = lif->txqcqs[i];
1598                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1599                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1600
1601                         if (err)
1602                                 return err;
1603                 }
1604         }
1605
1606         ionic_link_status_check(lif);
1607
1608         /* Carrier ON here */
1609
1610         return 0;
1611 }
1612
1613 int
1614 ionic_lif_identify(struct ionic_adapter *adapter)
1615 {
1616         struct ionic_dev *idev = &adapter->idev;
1617         struct ionic_identity *ident = &adapter->ident;
1618         int err;
1619         unsigned int i;
1620         unsigned int lif_words = sizeof(ident->lif.words) /
1621                 sizeof(ident->lif.words[0]);
1622         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1623                 sizeof(idev->dev_cmd->data[0]);
1624         unsigned int nwords;
1625
1626         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1627                 IONIC_IDENTITY_VERSION_1);
1628         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1629         if (err)
1630                 return (err);
1631
1632         nwords = RTE_MIN(lif_words, cmd_words);
1633         for (i = 0; i < nwords; i++)
1634                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1635
1636         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1637                 ident->lif.capabilities);
1638
1639         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1640                 ident->lif.eth.max_ucast_filters);
1641         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1642                 ident->lif.eth.max_mcast_filters);
1643
1644         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1645                 ident->lif.eth.config.features);
1646         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1647                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1648         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1649                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1650         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1651                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1652         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1653                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1654
1655         return 0;
1656 }
1657
1658 int
1659 ionic_lifs_size(struct ionic_adapter *adapter)
1660 {
1661         struct ionic_identity *ident = &adapter->ident;
1662         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1663
1664         adapter->max_ntxqs_per_lif =
1665                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1666         adapter->max_nrxqs_per_lif =
1667                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1668
1669         nintrs = 1 /* notifyq */;
1670
1671         if (nintrs > dev_nintrs) {
1672                 IONIC_PRINT(ERR,
1673                         "At most %d intr supported, minimum req'd is %u",
1674                         dev_nintrs, nintrs);
1675                 return -ENOSPC;
1676         }
1677
1678         adapter->nintrs = nintrs;
1679
1680         return 0;
1681 }