drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .type = q->type,
  29                         .index = q->index,
  30                         .oper = IONIC_Q_ENABLE,
  31                 },
  32         };
  33
  34         if (qcq->flags & IONIC_QCQ_F_INTR) {
  35                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  36                         IONIC_INTR_MASK_CLEAR);
  37         }
  38
  39         return ionic_adminq_post_wait(lif, &ctx);
  40 }
  41
  42 int
  43 ionic_qcq_disable(struct ionic_qcq *qcq)
  44 {
  45         struct ionic_queue *q = &qcq->q;
  46         struct ionic_lif *lif = q->lif;
  47         struct ionic_dev *idev = &lif->adapter->idev;
  48         struct ionic_admin_ctx ctx = {
  49                 .pending_work = true,
  50                 .cmd.q_control = {
  51                         .opcode = IONIC_CMD_Q_CONTROL,
  52                         .type = q->type,
  53                         .index = q->index,
  54                         .oper = IONIC_Q_DISABLE,
  55                 },
  56         };
  57
  58         if (qcq->flags & IONIC_QCQ_F_INTR) {
  59                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  60                         IONIC_INTR_MASK_SET);
  61         }
  62
  63         return ionic_adminq_post_wait(lif, &ctx);
  64 }
  65
  66 int
  67 ionic_lif_stop(struct ionic_lif *lif __rte_unused)
  68 {
  69         /* Carrier OFF here */
  70
  71         return 0;
  72 }
  73
  74 void
  75 ionic_lif_reset(struct ionic_lif *lif)
  76 {
  77         struct ionic_dev *idev = &lif->adapter->idev;
  78         int err;
  79
  80         IONIC_PRINT_CALL();
  81
  82         ionic_dev_cmd_lif_reset(idev);
  83         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  84         if (err)
  85                 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name);
  86 }
  87
  88 static void
  89 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  90 {
  91         struct ionic_lif_stats *ls = &lif->info->stats;
  92         uint32_t i;
  93         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  94                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  95         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  96                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  97
  98         memset(stats, 0, sizeof(*stats));
  99
 100         if (ls == NULL) {
 101                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 102                         lif->port_id);
 103                 return;
 104         }
 105
 106         /* RX */
 107
 108         stats->ipackets = ls->rx_ucast_packets +
 109                 ls->rx_mcast_packets +
 110                 ls->rx_bcast_packets;
 111
 112         stats->ibytes = ls->rx_ucast_bytes +
 113                 ls->rx_mcast_bytes +
 114                 ls->rx_bcast_bytes;
 115
 116         for (i = 0; i < lif->nrxqcqs; i++) {
 117                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 118                 stats->imissed +=
 119                         rx_stats->no_cb_arg +
 120                         rx_stats->bad_cq_status +
 121                         rx_stats->no_room +
 122                         rx_stats->bad_len;
 123         }
 124
 125         stats->imissed +=
 126                 ls->rx_ucast_drop_packets +
 127                 ls->rx_mcast_drop_packets +
 128                 ls->rx_bcast_drop_packets;
 129
 130         stats->imissed +=
 131                 ls->rx_queue_empty +
 132                 ls->rx_dma_error +
 133                 ls->rx_queue_disabled +
 134                 ls->rx_desc_fetch_error +
 135                 ls->rx_desc_data_error;
 136
 137         for (i = 0; i < num_rx_q_counters; i++) {
 138                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 139                 stats->q_ipackets[i] = rx_stats->packets;
 140                 stats->q_ibytes[i] = rx_stats->bytes;
 141                 stats->q_errors[i] =
 142                         rx_stats->no_cb_arg +
 143                         rx_stats->bad_cq_status +
 144                         rx_stats->no_room +
 145                         rx_stats->bad_len;
 146         }
 147
 148         /* TX */
 149
 150         stats->opackets = ls->tx_ucast_packets +
 151                 ls->tx_mcast_packets +
 152                 ls->tx_bcast_packets;
 153
 154         stats->obytes = ls->tx_ucast_bytes +
 155                 ls->tx_mcast_bytes +
 156                 ls->tx_bcast_bytes;
 157
 158         for (i = 0; i < lif->ntxqcqs; i++) {
 159                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 160                 stats->oerrors += tx_stats->drop;
 161         }
 162
 163         stats->oerrors +=
 164                 ls->tx_ucast_drop_packets +
 165                 ls->tx_mcast_drop_packets +
 166                 ls->tx_bcast_drop_packets;
 167
 168         stats->oerrors +=
 169                 ls->tx_dma_error +
 170                 ls->tx_queue_disabled +
 171                 ls->tx_desc_fetch_error +
 172                 ls->tx_desc_data_error;
 173
 174         for (i = 0; i < num_tx_q_counters; i++) {
 175                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 176                 stats->q_opackets[i] = tx_stats->packets;
 177                 stats->q_obytes[i] = tx_stats->bytes;
 178         }
 179 }
 180
 181 void
 182 ionic_lif_get_stats(const struct ionic_lif *lif,
 183                 struct rte_eth_stats *stats)
 184 {
 185         ionic_lif_get_abs_stats(lif, stats);
 186
 187         stats->ipackets  -= lif->stats_base.ipackets;
 188         stats->opackets  -= lif->stats_base.opackets;
 189         stats->ibytes    -= lif->stats_base.ibytes;
 190         stats->obytes    -= lif->stats_base.obytes;
 191         stats->imissed   -= lif->stats_base.imissed;
 192         stats->ierrors   -= lif->stats_base.ierrors;
 193         stats->oerrors   -= lif->stats_base.oerrors;
 194         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 195 }
 196
 197 void
 198 ionic_lif_reset_stats(struct ionic_lif *lif)
 199 {
 200         uint32_t i;
 201
 202         for (i = 0; i < lif->nrxqcqs; i++) {
 203                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 204                         sizeof(struct ionic_rx_stats));
 205                 memset(&lif->txqcqs[i]->stats.tx, 0,
 206                         sizeof(struct ionic_tx_stats));
 207         }
 208
 209         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 210 }
 211
 212 void
 213 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 214 {
 215         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 216         uint64_t *stats64 = (uint64_t *)stats;
 217         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 218         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 219
 220         for (i = 0; i < count; i++)
 221                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 222 }
 223
 224 void
 225 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 226 {
 227         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 228         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 229         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 230
 231         for (i = 0; i < count; i++)
 232                 lif_stats64_base[i] = lif_stats64[i];
 233 }
 234
 235 static int
 236 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 237 {
 238         struct ionic_admin_ctx ctx = {
 239                 .pending_work = true,
 240                 .cmd.rx_filter_add = {
 241                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 242                         .match = IONIC_RX_FILTER_MATCH_MAC,
 243                 },
 244         };
 245         int err;
 246
 247         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 248
 249         err = ionic_adminq_post_wait(lif, &ctx);
 250         if (err)
 251                 return err;
 252
 253         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 254                 ctx.comp.rx_filter_add.filter_id);
 255
 256         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 257 }
 258
 259 static int
 260 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 261 {
 262         struct ionic_admin_ctx ctx = {
 263                 .pending_work = true,
 264                 .cmd.rx_filter_del = {
 265                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 266                 },
 267         };
 268         struct ionic_rx_filter *f;
 269         int err;
 270
 271         IONIC_PRINT_CALL();
 272
 273         rte_spinlock_lock(&lif->rx_filters.lock);
 274
 275         f = ionic_rx_filter_by_addr(lif, addr);
 276         if (!f) {
 277                 rte_spinlock_unlock(&lif->rx_filters.lock);
 278                 return -ENOENT;
 279         }
 280
 281         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 282         ionic_rx_filter_free(f);
 283
 284         rte_spinlock_unlock(&lif->rx_filters.lock);
 285
 286         err = ionic_adminq_post_wait(lif, &ctx);
 287         if (err)
 288                 return err;
 289
 290         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 291                 ctx.cmd.rx_filter_del.filter_id);
 292
 293         return 0;
 294 }
 295
 296 int
 297 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 298                 struct rte_ether_addr *mac_addr,
 299                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 300 {
 301         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 302
 303         IONIC_PRINT_CALL();
 304
 305         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 306 }
 307
 308 void
 309 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index)
 310 {
 311         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 312         struct ionic_adapter *adapter = lif->adapter;
 313         struct rte_ether_addr *mac_addr;
 314
 315         IONIC_PRINT_CALL();
 316
 317         if (index >= adapter->max_mac_addrs) {
 318                 IONIC_PRINT(WARNING,
 319                         "Index %u is above MAC filter limit %u",
 320                         index, adapter->max_mac_addrs);
 321                 return;
 322         }
 323
 324         mac_addr = &eth_dev->data->mac_addrs[index];
 325
 326         if (!rte_is_valid_assigned_ether_addr(mac_addr))
 327                 return;
 328
 329         ionic_lif_addr_del(lif, (const uint8_t *)mac_addr);
 330 }
 331
 332 int
 333 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 334 {
 335         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 336
 337         IONIC_PRINT_CALL();
 338
 339         if (mac_addr == NULL) {
 340                 IONIC_PRINT(NOTICE, "New mac is null");
 341                 return -1;
 342         }
 343
 344         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 345                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 346                         lif->mac_addr);
 347                 ionic_lif_addr_del(lif, lif->mac_addr);
 348                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 349         }
 350
 351         IONIC_PRINT(INFO, "Updating mac addr");
 352
 353         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 354
 355         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 356 }
 357
 358 static int
 359 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 360 {
 361         struct ionic_admin_ctx ctx = {
 362                 .pending_work = true,
 363                 .cmd.rx_filter_add = {
 364                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 365                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 366                         .vlan.vlan = vid,
 367                 },
 368         };
 369         int err;
 370
 371         err = ionic_adminq_post_wait(lif, &ctx);
 372         if (err)
 373                 return err;
 374
 375         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 376                 ctx.comp.rx_filter_add.filter_id);
 377
 378         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 379 }
 380
 381 static int
 382 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 383 {
 384         struct ionic_admin_ctx ctx = {
 385                 .pending_work = true,
 386                 .cmd.rx_filter_del = {
 387                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 388                 },
 389         };
 390         struct ionic_rx_filter *f;
 391         int err;
 392
 393         IONIC_PRINT_CALL();
 394
 395         rte_spinlock_lock(&lif->rx_filters.lock);
 396
 397         f = ionic_rx_filter_by_vlan(lif, vid);
 398         if (!f) {
 399                 rte_spinlock_unlock(&lif->rx_filters.lock);
 400                 return -ENOENT;
 401         }
 402
 403         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 404         ionic_rx_filter_free(f);
 405         rte_spinlock_unlock(&lif->rx_filters.lock);
 406
 407         err = ionic_adminq_post_wait(lif, &ctx);
 408         if (err)
 409                 return err;
 410
 411         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 412                 ctx.cmd.rx_filter_del.filter_id);
 413
 414         return 0;
 415 }
 416
 417 int
 418 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 419                 int on)
 420 {
 421         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 422         int err;
 423
 424         if (on)
 425                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 426         else
 427                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 428
 429         return err;
 430 }
 431
 432 static void
 433 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 434 {
 435         struct ionic_admin_ctx ctx = {
 436                 .pending_work = true,
 437                 .cmd.rx_mode_set = {
 438                         .opcode = IONIC_CMD_RX_MODE_SET,
 439                         .rx_mode = rx_mode,
 440                 },
 441         };
 442         int err;
 443
 444         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 445                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 446         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 447                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 448         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 449                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 450         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 451                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 452         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 453                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 454
 455         err = ionic_adminq_post_wait(lif, &ctx);
 456         if (err)
 457                 IONIC_PRINT(ERR, "Failure setting RX mode");
 458 }
 459
 460 static void
 461 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 462 {
 463         if (lif->rx_mode != rx_mode) {
 464                 lif->rx_mode = rx_mode;
 465                 ionic_lif_rx_mode(lif, rx_mode);
 466         }
 467 }
 468
 469 int
 470 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 471 {
 472         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 473         uint32_t rx_mode = lif->rx_mode;
 474
 475         IONIC_PRINT_CALL();
 476
 477         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 478
 479         ionic_set_rx_mode(lif, rx_mode);
 480
 481         return 0;
 482 }
 483
 484 int
 485 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 486 {
 487         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 488         uint32_t rx_mode = lif->rx_mode;
 489
 490         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 491
 492         ionic_set_rx_mode(lif, rx_mode);
 493
 494         return 0;
 495 }
 496
 497 int
 498 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 499 {
 500         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 501         uint32_t rx_mode = lif->rx_mode;
 502
 503         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 504
 505         ionic_set_rx_mode(lif, rx_mode);
 506
 507         return 0;
 508 }
 509
 510 int
 511 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 512 {
 513         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 514         uint32_t rx_mode = lif->rx_mode;
 515
 516         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 517
 518         ionic_set_rx_mode(lif, rx_mode);
 519
 520         return 0;
 521 }
 522
 523 int
 524 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 525 {
 526         struct ionic_admin_ctx ctx = {
 527                 .pending_work = true,
 528                 .cmd.lif_setattr = {
 529                         .opcode = IONIC_CMD_LIF_SETATTR,
 530                         .attr = IONIC_LIF_ATTR_MTU,
 531                         .mtu = new_mtu,
 532                 },
 533         };
 534         int err;
 535
 536         err = ionic_adminq_post_wait(lif, &ctx);
 537         if (err)
 538                 return err;
 539
 540         return 0;
 541 }
 542
 543 int
 544 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 545 {
 546         struct ionic_adapter *adapter = lif->adapter;
 547         struct ionic_dev *idev = &adapter->idev;
 548         unsigned long index;
 549
 550         /*
 551          * Note: interrupt handler is called for index = 0 only
 552          * (we use interrupts for the notifyq only anyway,
 553          * which has index = 0)
 554          */
 555
 556         for (index = 0; index < adapter->nintrs; index++)
 557                 if (!adapter->intrs[index])
 558                         break;
 559
 560         if (index == adapter->nintrs)
 561                 return -ENOSPC;
 562
 563         adapter->intrs[index] = true;
 564
 565         ionic_intr_init(idev, intr, index);
 566
 567         return 0;
 568 }
 569
 570 void
 571 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 572 {
 573         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 574                 lif->adapter->intrs[intr->index] = false;
 575 }
 576
 577 static int
 578 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 579                 uint32_t index,
 580                 const char *base, uint32_t flags,
 581                 uint32_t num_descs,
 582                 uint32_t desc_size,
 583                 uint32_t cq_desc_size,
 584                 uint32_t sg_desc_size,
 585                 struct ionic_qcq **qcq)
 586 {
 587         struct ionic_dev *idev = &lif->adapter->idev;
 588         struct ionic_qcq *new;
 589         uint32_t q_size, cq_size, sg_size, total_size;
 590         void *q_base, *cq_base, *sg_base;
 591         rte_iova_t q_base_pa = 0;
 592         rte_iova_t cq_base_pa = 0;
 593         rte_iova_t sg_base_pa = 0;
 594         uint32_t socket_id = rte_socket_id();
 595         int err;
 596
 597         *qcq = NULL;
 598
 599         q_size  = num_descs * desc_size;
 600         cq_size = num_descs * cq_desc_size;
 601         sg_size = num_descs * sg_desc_size;
 602
 603         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 604                 RTE_ALIGN(cq_size, PAGE_SIZE);
 605         /*
 606          * Note: aligning q_size/cq_size is not enough due to cq_base address
 607          * aligning as q_base could be not aligned to the page.
 608          * Adding PAGE_SIZE.
 609          */
 610         total_size += PAGE_SIZE;
 611
 612         if (flags & IONIC_QCQ_F_SG) {
 613                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 614                 total_size += PAGE_SIZE;
 615         }
 616
 617         new = rte_zmalloc("ionic", sizeof(*new), 0);
 618         if (!new) {
 619                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 620                 return -ENOMEM;
 621         }
 622
 623         new->lif = lif;
 624         new->flags = flags;
 625
 626         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 627         if (!new->q.info) {
 628                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 629                 return -ENOMEM;
 630         }
 631
 632         new->q.type = type;
 633
 634         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 635                 desc_size, sg_desc_size);
 636         if (err) {
 637                 IONIC_PRINT(ERR, "Queue initialization failed");
 638                 return err;
 639         }
 640
 641         if (flags & IONIC_QCQ_F_INTR) {
 642                 err = ionic_intr_alloc(lif, &new->intr);
 643                 if (err)
 644                         return err;
 645
 646                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 647                         IONIC_INTR_MASK_SET);
 648         } else {
 649                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 650         }
 651
 652         err = ionic_cq_init(lif, &new->cq, &new->intr,
 653                 num_descs, cq_desc_size);
 654         if (err) {
 655                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 656                 goto err_out_free_intr;
 657         }
 658
 659         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 660                 base /* name */, index /* queue_idx */,
 661                 total_size, IONIC_ALIGN, socket_id);
 662
 663         if (!new->base_z) {
 664                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 665                 err = -ENOMEM;
 666                 goto err_out_free_intr;
 667         }
 668
 669         new->base = new->base_z->addr;
 670         new->base_pa = new->base_z->iova;
 671         new->total_size = total_size;
 672
 673         q_base = new->base;
 674         q_base_pa = new->base_pa;
 675
 676         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 677         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 678
 679         if (flags & IONIC_QCQ_F_SG) {
 680                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 681                         PAGE_SIZE);
 682                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 683                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 684         }
 685
 686         IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 687                 "SG-base-PA = %#jx",
 688                 q_base_pa, cq_base_pa, sg_base_pa);
 689
 690         ionic_q_map(&new->q, q_base, q_base_pa);
 691         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 692         ionic_cq_bind(&new->cq, &new->q);
 693
 694         *qcq = new;
 695
 696         return 0;
 697
 698 err_out_free_intr:
 699         if (flags & IONIC_QCQ_F_INTR)
 700                 ionic_intr_free(lif, &new->intr);
 701
 702         return err;
 703 }
 704
 705 void
 706 ionic_qcq_free(struct ionic_qcq *qcq)
 707 {
 708         if (qcq->base_z) {
 709                 qcq->base = NULL;
 710                 qcq->base_pa = 0;
 711                 rte_memzone_free(qcq->base_z);
 712                 qcq->base_z = NULL;
 713         }
 714
 715         if (qcq->q.info) {
 716                 rte_free(qcq->q.info);
 717                 qcq->q.info = NULL;
 718         }
 719
 720         rte_free(qcq);
 721 }
 722
 723 int
 724 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 725                 struct ionic_qcq **qcq)
 726 {
 727         uint32_t flags;
 728         int err = -ENOMEM;
 729
 730         flags = IONIC_QCQ_F_SG;
 731         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 732                 nrxq_descs,
 733                 sizeof(struct ionic_rxq_desc),
 734                 sizeof(struct ionic_rxq_comp),
 735                 sizeof(struct ionic_rxq_sg_desc),
 736                 &lif->rxqcqs[index]);
 737         if (err)
 738                 return err;
 739
 740         *qcq = lif->rxqcqs[index];
 741
 742         return 0;
 743 }
 744
 745 int
 746 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 747                 struct ionic_qcq **qcq)
 748 {
 749         uint32_t flags;
 750         int err = -ENOMEM;
 751
 752         flags = IONIC_QCQ_F_SG;
 753         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 754                 ntxq_descs,
 755                 sizeof(struct ionic_txq_desc),
 756                 sizeof(struct ionic_txq_comp),
 757                 sizeof(struct ionic_txq_sg_desc),
 758                 &lif->txqcqs[index]);
 759         if (err)
 760                 return err;
 761
 762         *qcq = lif->txqcqs[index];
 763
 764         return 0;
 765 }
 766
 767 static int
 768 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 769 {
 770         uint32_t flags;
 771         int err = -ENOMEM;
 772
 773         flags = 0;
 774         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 775                 IONIC_ADMINQ_LENGTH,
 776                 sizeof(struct ionic_admin_cmd),
 777                 sizeof(struct ionic_admin_comp),
 778                 0,
 779                 &lif->adminqcq);
 780         if (err)
 781                 return err;
 782
 783         return 0;
 784 }
 785
 786 static int
 787 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 788 {
 789         uint32_t flags;
 790         int err = -ENOMEM;
 791
 792         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 793
 794         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 795                 flags,
 796                 IONIC_NOTIFYQ_LENGTH,
 797                 sizeof(struct ionic_notifyq_cmd),
 798                 sizeof(union ionic_notifyq_comp),
 799                 0,
 800                 &lif->notifyqcq);
 801         if (err)
 802                 return err;
 803
 804         return 0;
 805 }
 806
 807 static void *
 808 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 809 {
 810         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 811
 812         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 813                 return NULL;
 814
 815         return (void *)&vaddr[page_num << PAGE_SHIFT];
 816 }
 817
 818 int
 819 ionic_lif_alloc(struct ionic_lif *lif)
 820 {
 821         struct ionic_adapter *adapter = lif->adapter;
 822         uint32_t socket_id = rte_socket_id();
 823         int err;
 824
 825         /*
 826          * lif->name was zeroed on allocation.
 827          * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated.
 828          */
 829         memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1);
 830
 831         IONIC_PRINT(DEBUG, "LIF: %s", lif->name);
 832
 833         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 834
 835         rte_spinlock_init(&lif->adminq_lock);
 836         rte_spinlock_init(&lif->adminq_service_lock);
 837
 838         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, 0);
 839         if (!lif->kern_dbpage) {
 840                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 841                 return -ENOMEM;
 842         }
 843
 844         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 845                 adapter->max_ntxqs_per_lif, 0);
 846
 847         if (!lif->txqcqs) {
 848                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 849                 return -ENOMEM;
 850         }
 851
 852         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 853                 adapter->max_nrxqs_per_lif, 0);
 854
 855         if (!lif->rxqcqs) {
 856                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 857                 return -ENOMEM;
 858         }
 859
 860         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 861
 862         err = ionic_notify_qcq_alloc(lif);
 863         if (err) {
 864                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 865                 return err;
 866         }
 867
 868         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 869
 870         err = ionic_admin_qcq_alloc(lif);
 871         if (err) {
 872                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 873                 return err;
 874         }
 875
 876         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 877
 878         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 879
 880         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 881                 "lif_info", 0 /* queue_idx*/,
 882                 lif->info_sz, IONIC_ALIGN, socket_id);
 883         if (!lif->info_z) {
 884                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 885                 return -ENOMEM;
 886         }
 887
 888         lif->info = lif->info_z->addr;
 889         lif->info_pa = lif->info_z->iova;
 890
 891         return 0;
 892 }
 893
 894 void
 895 ionic_lif_free(struct ionic_lif *lif)
 896 {
 897         if (lif->notifyqcq) {
 898                 ionic_qcq_free(lif->notifyqcq);
 899                 lif->notifyqcq = NULL;
 900         }
 901
 902         if (lif->adminqcq) {
 903                 ionic_qcq_free(lif->adminqcq);
 904                 lif->adminqcq = NULL;
 905         }
 906
 907         if (lif->txqcqs) {
 908                 rte_free(lif->txqcqs);
 909                 lif->txqcqs = NULL;
 910         }
 911
 912         if (lif->rxqcqs) {
 913                 rte_free(lif->rxqcqs);
 914                 lif->rxqcqs = NULL;
 915         }
 916
 917         if (lif->info) {
 918                 rte_memzone_free(lif->info_z);
 919                 lif->info = NULL;
 920         }
 921 }
 922
 923 void
 924 ionic_lif_free_queues(struct ionic_lif *lif)
 925 {
 926         uint32_t i;
 927
 928         for (i = 0; i < lif->ntxqcqs; i++) {
 929                 ionic_dev_tx_queue_release(lif->eth_dev->data->tx_queues[i]);
 930                 lif->eth_dev->data->tx_queues[i] = NULL;
 931         }
 932         for (i = 0; i < lif->nrxqcqs; i++) {
 933                 ionic_dev_rx_queue_release(lif->eth_dev->data->rx_queues[i]);
 934                 lif->eth_dev->data->rx_queues[i] = NULL;
 935         }
 936 }
 937
 938 int
 939 ionic_lif_rss_config(struct ionic_lif *lif,
 940                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 941 {
 942         struct ionic_admin_ctx ctx = {
 943                 .pending_work = true,
 944                 .cmd.lif_setattr = {
 945                         .opcode = IONIC_CMD_LIF_SETATTR,
 946                         .attr = IONIC_LIF_ATTR_RSS,
 947                         .rss.types = types,
 948                         .rss.addr = lif->rss_ind_tbl_pa,
 949                 },
 950         };
 951         unsigned int i;
 952
 953         IONIC_PRINT_CALL();
 954
 955         lif->rss_types = types;
 956
 957         if (key)
 958                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 959
 960         if (indir)
 961                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 962                         lif->rss_ind_tbl[i] = indir[i];
 963
 964         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 965                IONIC_RSS_HASH_KEY_SIZE);
 966
 967         return ionic_adminq_post_wait(lif, &ctx);
 968 }
 969
 970 static int
 971 ionic_lif_rss_setup(struct ionic_lif *lif)
 972 {
 973         static const uint8_t toeplitz_symmetric_key[] = {
 974                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 975                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 976                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 977                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 978                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 979         };
 980         uint32_t i;
 981         uint16_t tbl_sz = lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 982
 983         IONIC_PRINT_CALL();
 984
 985         if (!lif->rss_ind_tbl_z) {
 986                 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 987                                         "rss_ind_tbl", 0 /* queue_idx */,
 988                                         sizeof(*lif->rss_ind_tbl) * tbl_sz,
 989                                         IONIC_ALIGN, rte_socket_id());
 990                 if (!lif->rss_ind_tbl_z) {
 991                         IONIC_PRINT(ERR, "OOM");
 992                         return -ENOMEM;
 993                 }
 994
 995                 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 996                 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
 997         }
 998
 999         if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) {
1000                 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs;
1001
1002                 /* Fill indirection table with 'default' values */
1003                 for (i = 0; i < tbl_sz; i++)
1004                         lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
1005         }
1006
1007         return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
1008                         toeplitz_symmetric_key, NULL);
1009 }
1010
1011 static void
1012 ionic_lif_rss_teardown(struct ionic_lif *lif)
1013 {
1014         if (!lif->rss_ind_tbl)
1015                 return;
1016
1017         if (lif->rss_ind_tbl_z) {
1018                 /* Disable RSS on the NIC */
1019                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1020
1021                 lif->rss_ind_tbl = NULL;
1022                 lif->rss_ind_tbl_pa = 0;
1023                 rte_memzone_free(lif->rss_ind_tbl_z);
1024                 lif->rss_ind_tbl_z = NULL;
1025         }
1026 }
1027
1028 static void
1029 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1030 {
1031         struct ionic_dev *idev = &lif->adapter->idev;
1032
1033         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1034                 return;
1035
1036         if (qcq->flags & IONIC_QCQ_F_INTR)
1037                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1038                         IONIC_INTR_MASK_SET);
1039
1040         qcq->flags &= ~IONIC_QCQ_F_INITED;
1041 }
1042
1043 void
1044 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1045 {
1046         ionic_lif_qcq_deinit(qcq->lif, qcq);
1047 }
1048
1049 void
1050 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1051 {
1052         ionic_lif_qcq_deinit(qcq->lif, qcq);
1053 }
1054
1055 bool
1056 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1057                 void *cb_arg __rte_unused)
1058 {
1059         struct ionic_admin_comp *cq_desc_base = cq->base;
1060         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1061
1062         if (!color_match(cq_desc->color, cq->done_color))
1063                 return false;
1064
1065         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1066
1067         return true;
1068 }
1069
1070 /* This acts like ionic_napi */
1071 int
1072 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1073                 void *cb_arg)
1074 {
1075         struct ionic_cq *cq = &qcq->cq;
1076         uint32_t work_done;
1077
1078         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1079
1080         return work_done;
1081 }
1082
1083 static void
1084 ionic_link_status_check(struct ionic_lif *lif)
1085 {
1086         struct ionic_adapter *adapter = lif->adapter;
1087         bool link_up;
1088
1089         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1090
1091         if (!lif->info)
1092                 return;
1093
1094         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1095
1096         if ((link_up  && adapter->link_up) ||
1097             (!link_up && !adapter->link_up))
1098                 return;
1099
1100         if (link_up) {
1101                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1102                         lif->info->status.link_speed);
1103                 adapter->link_speed = lif->info->status.link_speed;
1104         } else {
1105                 IONIC_PRINT(DEBUG, "Link down");
1106         }
1107
1108         adapter->link_up = link_up;
1109 }
1110
1111 static bool
1112 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1113 {
1114         union ionic_notifyq_comp *cq_desc_base = cq->base;
1115         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1116         struct ionic_lif *lif = cb_arg;
1117
1118         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1119                 cq_desc->event.eid, cq_desc->event.ecode);
1120
1121         /* Have we run out of new completions to process? */
1122         if (!(cq_desc->event.eid > lif->last_eid))
1123                 return false;
1124
1125         lif->last_eid = cq_desc->event.eid;
1126
1127         switch (cq_desc->event.ecode) {
1128         case IONIC_EVENT_LINK_CHANGE:
1129                 IONIC_PRINT(DEBUG,
1130                         "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
1131                         cq_desc->event.eid,
1132                         cq_desc->link_change.link_status,
1133                         cq_desc->link_change.link_speed);
1134
1135                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1136
1137                 break;
1138         default:
1139                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1140                         cq_desc->event.ecode, cq_desc->event.eid);
1141                 break;
1142         }
1143
1144         return true;
1145 }
1146
1147 int
1148 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1149 {
1150         struct ionic_dev *idev = &lif->adapter->idev;
1151         struct ionic_qcq *qcq = lif->notifyqcq;
1152         uint32_t work_done;
1153
1154         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1155                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1156                 return -1;
1157         }
1158
1159         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1160                 IONIC_INTR_MASK_SET);
1161
1162         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1163
1164         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1165                 ionic_link_status_check(lif);
1166
1167         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1168                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1169
1170         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1171                 IONIC_INTR_MASK_CLEAR);
1172
1173         return 0;
1174 }
1175
1176 static int
1177 ionic_lif_adminq_init(struct ionic_lif *lif)
1178 {
1179         struct ionic_dev *idev = &lif->adapter->idev;
1180         struct ionic_qcq *qcq = lif->adminqcq;
1181         struct ionic_queue *q = &qcq->q;
1182         struct ionic_q_init_comp comp;
1183         int err;
1184
1185         ionic_dev_cmd_adminq_init(idev, qcq, qcq->intr.index);
1186         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1187         if (err)
1188                 return err;
1189
1190         ionic_dev_cmd_comp(idev, &comp);
1191
1192         q->hw_type = comp.hw_type;
1193         q->hw_index = comp.hw_index;
1194         q->db = ionic_db_map(lif, q);
1195
1196         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1197         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1198         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1199
1200         if (qcq->flags & IONIC_QCQ_F_INTR)
1201                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1202                         IONIC_INTR_MASK_CLEAR);
1203
1204         qcq->flags |= IONIC_QCQ_F_INITED;
1205
1206         return 0;
1207 }
1208
1209 static int
1210 ionic_lif_notifyq_init(struct ionic_lif *lif)
1211 {
1212         struct ionic_dev *idev = &lif->adapter->idev;
1213         struct ionic_qcq *qcq = lif->notifyqcq;
1214         struct ionic_queue *q = &qcq->q;
1215         int err;
1216
1217         struct ionic_admin_ctx ctx = {
1218                 .pending_work = true,
1219                 .cmd.q_init = {
1220                         .opcode = IONIC_CMD_Q_INIT,
1221                         .type = q->type,
1222                         .index = q->index,
1223                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1224                         .intr_index = qcq->intr.index,
1225                         .ring_size = rte_log2_u32(q->num_descs),
1226                         .ring_base = q->base_pa,
1227                 }
1228         };
1229
1230         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1231                 ctx.cmd.q_init.index);
1232         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1233                 ctx.cmd.q_init.ring_base);
1234         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1235                 ctx.cmd.q_init.ring_size);
1236         IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver);
1237
1238         err = ionic_adminq_post_wait(lif, &ctx);
1239         if (err)
1240                 return err;
1241
1242         q->hw_type = ctx.comp.q_init.hw_type;
1243         q->hw_index = ctx.comp.q_init.hw_index;
1244         q->db = NULL;
1245
1246         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1247         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1248         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1249
1250         if (qcq->flags & IONIC_QCQ_F_INTR)
1251                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1252                         IONIC_INTR_MASK_CLEAR);
1253
1254         qcq->flags |= IONIC_QCQ_F_INITED;
1255
1256         return 0;
1257 }
1258
1259 int
1260 ionic_lif_set_features(struct ionic_lif *lif)
1261 {
1262         struct ionic_admin_ctx ctx = {
1263                 .pending_work = true,
1264                 .cmd.lif_setattr = {
1265                         .opcode = IONIC_CMD_LIF_SETATTR,
1266                         .attr = IONIC_LIF_ATTR_FEATURES,
1267                         .features = lif->features,
1268                 },
1269         };
1270         int err;
1271
1272         err = ionic_adminq_post_wait(lif, &ctx);
1273         if (err)
1274                 return err;
1275
1276         lif->hw_features = (ctx.cmd.lif_setattr.features &
1277                 ctx.comp.lif_setattr.features);
1278
1279         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1280                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1281         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1282                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1283         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1284                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1285         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1286                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1287         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1288                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1289         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1290                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1291         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1292                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1293         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1294                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1295         if (lif->hw_features & IONIC_ETH_HW_TSO)
1296                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1297         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1298                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1299         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1300                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1301         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1302                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1303         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1304                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1305         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1306                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1307         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1308                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1309         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1310                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1311         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1312                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1313
1314         return 0;
1315 }
1316
1317 int
1318 ionic_lif_txq_init(struct ionic_qcq *qcq)
1319 {
1320         struct ionic_queue *q = &qcq->q;
1321         struct ionic_lif *lif = qcq->lif;
1322         struct ionic_cq *cq = &qcq->cq;
1323         struct ionic_admin_ctx ctx = {
1324                 .pending_work = true,
1325                 .cmd.q_init = {
1326                         .opcode = IONIC_CMD_Q_INIT,
1327                         .type = q->type,
1328                         .index = q->index,
1329                         .flags = IONIC_QINIT_F_SG,
1330                         .intr_index = cq->bound_intr->index,
1331                         .ring_size = rte_log2_u32(q->num_descs),
1332                         .ring_base = q->base_pa,
1333                         .cq_ring_base = cq->base_pa,
1334                         .sg_ring_base = q->sg_base_pa,
1335                 },
1336         };
1337         int err;
1338
1339         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1340         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1341                 ctx.cmd.q_init.ring_base);
1342         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1343                 ctx.cmd.q_init.ring_size);
1344         IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver);
1345
1346         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1347         if (err)
1348                 return err;
1349
1350         q->hw_type = ctx.comp.q_init.hw_type;
1351         q->hw_index = ctx.comp.q_init.hw_index;
1352         q->db = ionic_db_map(lif, q);
1353
1354         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1355         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1356         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1357
1358         qcq->flags |= IONIC_QCQ_F_INITED;
1359
1360         return 0;
1361 }
1362
1363 int
1364 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1365 {
1366         struct ionic_queue *q = &qcq->q;
1367         struct ionic_lif *lif = qcq->lif;
1368         struct ionic_cq *cq = &qcq->cq;
1369         struct ionic_admin_ctx ctx = {
1370                 .pending_work = true,
1371                 .cmd.q_init = {
1372                         .opcode = IONIC_CMD_Q_INIT,
1373                         .type = q->type,
1374                         .index = q->index,
1375                         .flags = IONIC_QINIT_F_SG,
1376                         .intr_index = cq->bound_intr->index,
1377                         .ring_size = rte_log2_u32(q->num_descs),
1378                         .ring_base = q->base_pa,
1379                         .cq_ring_base = cq->base_pa,
1380                         .sg_ring_base = q->sg_base_pa,
1381                 },
1382         };
1383         int err;
1384
1385         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1386         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1387                 ctx.cmd.q_init.ring_base);
1388         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1389                 ctx.cmd.q_init.ring_size);
1390         IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver);
1391
1392         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1393         if (err)
1394                 return err;
1395
1396         q->hw_type = ctx.comp.q_init.hw_type;
1397         q->hw_index = ctx.comp.q_init.hw_index;
1398         q->db = ionic_db_map(lif, q);
1399
1400         qcq->flags |= IONIC_QCQ_F_INITED;
1401
1402         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1403         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1404         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1405
1406         return 0;
1407 }
1408
1409 static int
1410 ionic_station_set(struct ionic_lif *lif)
1411 {
1412         struct ionic_admin_ctx ctx = {
1413                 .pending_work = true,
1414                 .cmd.lif_getattr = {
1415                         .opcode = IONIC_CMD_LIF_GETATTR,
1416                         .attr = IONIC_LIF_ATTR_MAC,
1417                 },
1418         };
1419         int err;
1420
1421         IONIC_PRINT_CALL();
1422
1423         err = ionic_adminq_post_wait(lif, &ctx);
1424         if (err)
1425                 return err;
1426
1427         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1428                         lif->mac_addr)) {
1429                 IONIC_PRINT(INFO, "deleting station MAC addr");
1430
1431                 ionic_lif_addr_del(lif, lif->mac_addr);
1432         }
1433
1434         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1435
1436         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1437                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1438                 return 0;
1439         }
1440
1441         IONIC_PRINT(DEBUG, "adding station MAC addr");
1442
1443         ionic_lif_addr_add(lif, lif->mac_addr);
1444
1445         return 0;
1446 }
1447
1448 static void
1449 ionic_lif_set_name(struct ionic_lif *lif)
1450 {
1451         struct ionic_admin_ctx ctx = {
1452                 .pending_work = true,
1453                 .cmd.lif_setattr = {
1454                         .opcode = IONIC_CMD_LIF_SETATTR,
1455                         .attr = IONIC_LIF_ATTR_NAME,
1456                 },
1457         };
1458
1459         memcpy(ctx.cmd.lif_setattr.name, lif->name,
1460                 sizeof(ctx.cmd.lif_setattr.name) - 1);
1461
1462         ionic_adminq_post_wait(lif, &ctx);
1463 }
1464
1465 int
1466 ionic_lif_init(struct ionic_lif *lif)
1467 {
1468         struct ionic_dev *idev = &lif->adapter->idev;
1469         struct ionic_q_init_comp comp;
1470         int err;
1471
1472         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1473
1474         ionic_dev_cmd_lif_init(idev, lif->info_pa);
1475         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1476         ionic_dev_cmd_comp(idev, &comp);
1477         if (err)
1478                 return err;
1479
1480         lif->hw_index = comp.hw_index;
1481
1482         err = ionic_lif_adminq_init(lif);
1483         if (err)
1484                 return err;
1485
1486         err = ionic_lif_notifyq_init(lif);
1487         if (err)
1488                 goto err_out_adminq_deinit;
1489
1490         lif->features =
1491                   IONIC_ETH_HW_VLAN_TX_TAG
1492                 | IONIC_ETH_HW_VLAN_RX_STRIP
1493                 | IONIC_ETH_HW_VLAN_RX_FILTER
1494                 | IONIC_ETH_HW_RX_HASH
1495                 | IONIC_ETH_HW_TX_SG
1496                 | IONIC_ETH_HW_RX_SG
1497                 | IONIC_ETH_HW_TX_CSUM
1498                 | IONIC_ETH_HW_RX_CSUM
1499                 | IONIC_ETH_HW_TSO
1500                 | IONIC_ETH_HW_TSO_IPV6
1501                 | IONIC_ETH_HW_TSO_ECN;
1502
1503         err = ionic_lif_set_features(lif);
1504         if (err)
1505                 goto err_out_notifyq_deinit;
1506
1507         err = ionic_rx_filters_init(lif);
1508         if (err)
1509                 goto err_out_notifyq_deinit;
1510
1511         err = ionic_station_set(lif);
1512         if (err)
1513                 goto err_out_rx_filter_deinit;
1514
1515         ionic_lif_set_name(lif);
1516
1517         lif->state |= IONIC_LIF_F_INITED;
1518
1519         return 0;
1520
1521 err_out_rx_filter_deinit:
1522         ionic_rx_filters_deinit(lif);
1523
1524 err_out_notifyq_deinit:
1525         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1526
1527 err_out_adminq_deinit:
1528         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1529
1530         return err;
1531 }
1532
1533 void
1534 ionic_lif_deinit(struct ionic_lif *lif)
1535 {
1536         if (!(lif->state & IONIC_LIF_F_INITED))
1537                 return;
1538
1539         ionic_rx_filters_deinit(lif);
1540         ionic_lif_rss_teardown(lif);
1541         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1542         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1543
1544         lif->state &= ~IONIC_LIF_F_INITED;
1545 }
1546
1547 int
1548 ionic_lif_configure(struct ionic_lif *lif)
1549 {
1550         struct ionic_identity *ident = &lif->adapter->ident;
1551         uint32_t ntxqs_per_lif =
1552                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1553         uint32_t nrxqs_per_lif =
1554                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1555         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1556         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1557
1558         lif->port_id = lif->eth_dev->data->port_id;
1559
1560         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1561                 lif->port_id);
1562
1563         if (nrxqs > 0)
1564                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1565
1566         if (ntxqs > 0)
1567                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1568
1569         lif->nrxqcqs = nrxqs_per_lif;
1570         lif->ntxqcqs = ntxqs_per_lif;
1571
1572         return 0;
1573 }
1574
1575 int
1576 ionic_lif_start(struct ionic_lif *lif)
1577 {
1578         uint32_t rx_mode;
1579         uint32_t i;
1580         int err;
1581
1582         err = ionic_lif_rss_setup(lif);
1583         if (err)
1584                 return err;
1585
1586         if (!lif->rx_mode) {
1587                 IONIC_PRINT(DEBUG, "Setting RX mode on %s",
1588                         lif->name);
1589
1590                 rx_mode  = IONIC_RX_MODE_F_UNICAST;
1591                 rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1592                 rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1593
1594                 ionic_set_rx_mode(lif, rx_mode);
1595         }
1596
1597         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1598                 "on port %u",
1599                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1600
1601         for (i = 0; i < lif->nrxqcqs; i++) {
1602                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1603                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1604                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1605
1606                         if (err)
1607                                 return err;
1608                 }
1609         }
1610
1611         for (i = 0; i < lif->ntxqcqs; i++) {
1612                 struct ionic_qcq *txq = lif->txqcqs[i];
1613                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1614                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1615
1616                         if (err)
1617                                 return err;
1618                 }
1619         }
1620
1621         ionic_link_status_check(lif);
1622
1623         /* Carrier ON here */
1624
1625         return 0;
1626 }
1627
1628 int
1629 ionic_lif_identify(struct ionic_adapter *adapter)
1630 {
1631         struct ionic_dev *idev = &adapter->idev;
1632         struct ionic_identity *ident = &adapter->ident;
1633         int err;
1634         unsigned int i;
1635         unsigned int lif_words = sizeof(ident->lif.words) /
1636                 sizeof(ident->lif.words[0]);
1637         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1638                 sizeof(idev->dev_cmd->data[0]);
1639         unsigned int nwords;
1640
1641         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1642                 IONIC_IDENTITY_VERSION_1);
1643         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1644         if (err)
1645                 return (err);
1646
1647         nwords = RTE_MIN(lif_words, cmd_words);
1648         for (i = 0; i < nwords; i++)
1649                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1650
1651         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1652                 ident->lif.capabilities);
1653
1654         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1655                 ident->lif.eth.max_ucast_filters);
1656         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1657                 ident->lif.eth.max_mcast_filters);
1658
1659         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1660                 ident->lif.eth.config.features);
1661         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1662                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1663         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1664                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1665         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1666                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1667         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1668                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1669
1670         return 0;
1671 }
1672
1673 int
1674 ionic_lifs_size(struct ionic_adapter *adapter)
1675 {
1676         struct ionic_identity *ident = &adapter->ident;
1677         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1678
1679         adapter->max_ntxqs_per_lif =
1680                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1681         adapter->max_nrxqs_per_lif =
1682                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1683
1684         nintrs = 1 /* notifyq */;
1685
1686         if (nintrs > dev_nintrs) {
1687                 IONIC_PRINT(ERR,
1688                         "At most %d intr supported, minimum req'd is %u",
1689                         dev_nintrs, nintrs);
1690                 return -ENOSPC;
1691         }
1692
1693         adapter->nintrs = nintrs;
1694
1695         return 0;
1696 }