drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .type = q->type,
  29                         .index = q->index,
  30                         .oper = IONIC_Q_ENABLE,
  31                 },
  32         };
  33
  34         if (qcq->flags & IONIC_QCQ_F_INTR) {
  35                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  36                         IONIC_INTR_MASK_CLEAR);
  37         }
  38
  39         return ionic_adminq_post_wait(lif, &ctx);
  40 }
  41
  42 int
  43 ionic_qcq_disable(struct ionic_qcq *qcq)
  44 {
  45         struct ionic_queue *q = &qcq->q;
  46         struct ionic_lif *lif = q->lif;
  47         struct ionic_dev *idev = &lif->adapter->idev;
  48         struct ionic_admin_ctx ctx = {
  49                 .pending_work = true,
  50                 .cmd.q_control = {
  51                         .opcode = IONIC_CMD_Q_CONTROL,
  52                         .type = q->type,
  53                         .index = q->index,
  54                         .oper = IONIC_Q_DISABLE,
  55                 },
  56         };
  57
  58         if (qcq->flags & IONIC_QCQ_F_INTR) {
  59                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  60                         IONIC_INTR_MASK_SET);
  61         }
  62
  63         return ionic_adminq_post_wait(lif, &ctx);
  64 }
  65
  66 void
  67 ionic_lif_stop(struct ionic_lif *lif)
  68 {
  69         uint32_t i;
  70
  71         IONIC_PRINT_CALL();
  72
  73         lif->state &= ~IONIC_LIF_F_UP;
  74
  75         for (i = 0; i < lif->nrxqcqs; i++) {
  76                 struct ionic_qcq *rxq = lif->rxqcqs[i];
  77                 if (rxq->flags & IONIC_QCQ_F_INITED)
  78                         (void)ionic_dev_rx_queue_stop(lif->eth_dev, i);
  79         }
  80
  81         for (i = 0; i < lif->ntxqcqs; i++) {
  82                 struct ionic_qcq *txq = lif->txqcqs[i];
  83                 if (txq->flags & IONIC_QCQ_F_INITED)
  84                         (void)ionic_dev_tx_queue_stop(lif->eth_dev, i);
  85         }
  86 }
  87
  88 void
  89 ionic_lif_reset(struct ionic_lif *lif)
  90 {
  91         struct ionic_dev *idev = &lif->adapter->idev;
  92         int err;
  93
  94         IONIC_PRINT_CALL();
  95
  96         ionic_dev_cmd_lif_reset(idev);
  97         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  98         if (err)
  99                 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name);
 100 }
 101
 102 static void
 103 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
 104 {
 105         struct ionic_lif_stats *ls = &lif->info->stats;
 106         uint32_t i;
 107         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
 108                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
 109         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
 110                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
 111
 112         memset(stats, 0, sizeof(*stats));
 113
 114         if (ls == NULL) {
 115                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 116                         lif->port_id);
 117                 return;
 118         }
 119
 120         /* RX */
 121
 122         stats->ipackets = ls->rx_ucast_packets +
 123                 ls->rx_mcast_packets +
 124                 ls->rx_bcast_packets;
 125
 126         stats->ibytes = ls->rx_ucast_bytes +
 127                 ls->rx_mcast_bytes +
 128                 ls->rx_bcast_bytes;
 129
 130         for (i = 0; i < lif->nrxqcqs; i++) {
 131                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 132                 stats->imissed +=
 133                         rx_stats->no_cb_arg +
 134                         rx_stats->bad_cq_status +
 135                         rx_stats->no_room +
 136                         rx_stats->bad_len;
 137         }
 138
 139         stats->imissed +=
 140                 ls->rx_ucast_drop_packets +
 141                 ls->rx_mcast_drop_packets +
 142                 ls->rx_bcast_drop_packets;
 143
 144         stats->imissed +=
 145                 ls->rx_queue_empty +
 146                 ls->rx_dma_error +
 147                 ls->rx_queue_disabled +
 148                 ls->rx_desc_fetch_error +
 149                 ls->rx_desc_data_error;
 150
 151         for (i = 0; i < num_rx_q_counters; i++) {
 152                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 153                 stats->q_ipackets[i] = rx_stats->packets;
 154                 stats->q_ibytes[i] = rx_stats->bytes;
 155                 stats->q_errors[i] =
 156                         rx_stats->no_cb_arg +
 157                         rx_stats->bad_cq_status +
 158                         rx_stats->no_room +
 159                         rx_stats->bad_len;
 160         }
 161
 162         /* TX */
 163
 164         stats->opackets = ls->tx_ucast_packets +
 165                 ls->tx_mcast_packets +
 166                 ls->tx_bcast_packets;
 167
 168         stats->obytes = ls->tx_ucast_bytes +
 169                 ls->tx_mcast_bytes +
 170                 ls->tx_bcast_bytes;
 171
 172         for (i = 0; i < lif->ntxqcqs; i++) {
 173                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 174                 stats->oerrors += tx_stats->drop;
 175         }
 176
 177         stats->oerrors +=
 178                 ls->tx_ucast_drop_packets +
 179                 ls->tx_mcast_drop_packets +
 180                 ls->tx_bcast_drop_packets;
 181
 182         stats->oerrors +=
 183                 ls->tx_dma_error +
 184                 ls->tx_queue_disabled +
 185                 ls->tx_desc_fetch_error +
 186                 ls->tx_desc_data_error;
 187
 188         for (i = 0; i < num_tx_q_counters; i++) {
 189                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 190                 stats->q_opackets[i] = tx_stats->packets;
 191                 stats->q_obytes[i] = tx_stats->bytes;
 192         }
 193 }
 194
 195 void
 196 ionic_lif_get_stats(const struct ionic_lif *lif,
 197                 struct rte_eth_stats *stats)
 198 {
 199         ionic_lif_get_abs_stats(lif, stats);
 200
 201         stats->ipackets  -= lif->stats_base.ipackets;
 202         stats->opackets  -= lif->stats_base.opackets;
 203         stats->ibytes    -= lif->stats_base.ibytes;
 204         stats->obytes    -= lif->stats_base.obytes;
 205         stats->imissed   -= lif->stats_base.imissed;
 206         stats->ierrors   -= lif->stats_base.ierrors;
 207         stats->oerrors   -= lif->stats_base.oerrors;
 208         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 209 }
 210
 211 void
 212 ionic_lif_reset_stats(struct ionic_lif *lif)
 213 {
 214         uint32_t i;
 215
 216         for (i = 0; i < lif->nrxqcqs; i++) {
 217                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 218                         sizeof(struct ionic_rx_stats));
 219                 memset(&lif->txqcqs[i]->stats.tx, 0,
 220                         sizeof(struct ionic_tx_stats));
 221         }
 222
 223         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 224 }
 225
 226 void
 227 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 228 {
 229         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 230         uint64_t *stats64 = (uint64_t *)stats;
 231         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 232         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 233
 234         for (i = 0; i < count; i++)
 235                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 236 }
 237
 238 void
 239 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 240 {
 241         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 242         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 243         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 244
 245         for (i = 0; i < count; i++)
 246                 lif_stats64_base[i] = lif_stats64[i];
 247 }
 248
 249 static int
 250 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 251 {
 252         struct ionic_admin_ctx ctx = {
 253                 .pending_work = true,
 254                 .cmd.rx_filter_add = {
 255                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 256                         .match = IONIC_RX_FILTER_MATCH_MAC,
 257                 },
 258         };
 259         int err;
 260
 261         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 262
 263         err = ionic_adminq_post_wait(lif, &ctx);
 264         if (err)
 265                 return err;
 266
 267         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 268                 ctx.comp.rx_filter_add.filter_id);
 269
 270         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 271 }
 272
 273 static int
 274 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 275 {
 276         struct ionic_admin_ctx ctx = {
 277                 .pending_work = true,
 278                 .cmd.rx_filter_del = {
 279                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 280                 },
 281         };
 282         struct ionic_rx_filter *f;
 283         int err;
 284
 285         IONIC_PRINT_CALL();
 286
 287         rte_spinlock_lock(&lif->rx_filters.lock);
 288
 289         f = ionic_rx_filter_by_addr(lif, addr);
 290         if (!f) {
 291                 rte_spinlock_unlock(&lif->rx_filters.lock);
 292                 return -ENOENT;
 293         }
 294
 295         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 296         ionic_rx_filter_free(f);
 297
 298         rte_spinlock_unlock(&lif->rx_filters.lock);
 299
 300         err = ionic_adminq_post_wait(lif, &ctx);
 301         if (err)
 302                 return err;
 303
 304         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 305                 ctx.cmd.rx_filter_del.filter_id);
 306
 307         return 0;
 308 }
 309
 310 int
 311 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 312                 struct rte_ether_addr *mac_addr,
 313                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 314 {
 315         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 316
 317         IONIC_PRINT_CALL();
 318
 319         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 320 }
 321
 322 void
 323 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index)
 324 {
 325         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 326         struct ionic_adapter *adapter = lif->adapter;
 327         struct rte_ether_addr *mac_addr;
 328
 329         IONIC_PRINT_CALL();
 330
 331         if (index >= adapter->max_mac_addrs) {
 332                 IONIC_PRINT(WARNING,
 333                         "Index %u is above MAC filter limit %u",
 334                         index, adapter->max_mac_addrs);
 335                 return;
 336         }
 337
 338         mac_addr = &eth_dev->data->mac_addrs[index];
 339
 340         if (!rte_is_valid_assigned_ether_addr(mac_addr))
 341                 return;
 342
 343         ionic_lif_addr_del(lif, (const uint8_t *)mac_addr);
 344 }
 345
 346 int
 347 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 348 {
 349         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 350
 351         IONIC_PRINT_CALL();
 352
 353         if (mac_addr == NULL) {
 354                 IONIC_PRINT(NOTICE, "New mac is null");
 355                 return -1;
 356         }
 357
 358         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 359                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 360                         lif->mac_addr);
 361                 ionic_lif_addr_del(lif, lif->mac_addr);
 362                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 363         }
 364
 365         IONIC_PRINT(INFO, "Updating mac addr");
 366
 367         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 368
 369         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 370 }
 371
 372 static int
 373 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 374 {
 375         struct ionic_admin_ctx ctx = {
 376                 .pending_work = true,
 377                 .cmd.rx_filter_add = {
 378                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 379                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 380                         .vlan.vlan = vid,
 381                 },
 382         };
 383         int err;
 384
 385         err = ionic_adminq_post_wait(lif, &ctx);
 386         if (err)
 387                 return err;
 388
 389         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 390                 ctx.comp.rx_filter_add.filter_id);
 391
 392         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 393 }
 394
 395 static int
 396 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 397 {
 398         struct ionic_admin_ctx ctx = {
 399                 .pending_work = true,
 400                 .cmd.rx_filter_del = {
 401                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 402                 },
 403         };
 404         struct ionic_rx_filter *f;
 405         int err;
 406
 407         IONIC_PRINT_CALL();
 408
 409         rte_spinlock_lock(&lif->rx_filters.lock);
 410
 411         f = ionic_rx_filter_by_vlan(lif, vid);
 412         if (!f) {
 413                 rte_spinlock_unlock(&lif->rx_filters.lock);
 414                 return -ENOENT;
 415         }
 416
 417         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 418         ionic_rx_filter_free(f);
 419         rte_spinlock_unlock(&lif->rx_filters.lock);
 420
 421         err = ionic_adminq_post_wait(lif, &ctx);
 422         if (err)
 423                 return err;
 424
 425         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 426                 ctx.cmd.rx_filter_del.filter_id);
 427
 428         return 0;
 429 }
 430
 431 int
 432 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 433                 int on)
 434 {
 435         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 436         int err;
 437
 438         if (on)
 439                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 440         else
 441                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 442
 443         return err;
 444 }
 445
 446 static void
 447 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 448 {
 449         struct ionic_admin_ctx ctx = {
 450                 .pending_work = true,
 451                 .cmd.rx_mode_set = {
 452                         .opcode = IONIC_CMD_RX_MODE_SET,
 453                         .rx_mode = rx_mode,
 454                 },
 455         };
 456         int err;
 457
 458         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 459                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 460         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 461                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 462         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 463                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 464         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 465                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 466         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 467                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 468
 469         err = ionic_adminq_post_wait(lif, &ctx);
 470         if (err)
 471                 IONIC_PRINT(ERR, "Failure setting RX mode");
 472 }
 473
 474 static void
 475 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 476 {
 477         if (lif->rx_mode != rx_mode) {
 478                 lif->rx_mode = rx_mode;
 479                 ionic_lif_rx_mode(lif, rx_mode);
 480         }
 481 }
 482
 483 int
 484 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 485 {
 486         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 487         uint32_t rx_mode = lif->rx_mode;
 488
 489         IONIC_PRINT_CALL();
 490
 491         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 492
 493         ionic_set_rx_mode(lif, rx_mode);
 494
 495         return 0;
 496 }
 497
 498 int
 499 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 500 {
 501         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 502         uint32_t rx_mode = lif->rx_mode;
 503
 504         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 505
 506         ionic_set_rx_mode(lif, rx_mode);
 507
 508         return 0;
 509 }
 510
 511 int
 512 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 513 {
 514         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 515         uint32_t rx_mode = lif->rx_mode;
 516
 517         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 518
 519         ionic_set_rx_mode(lif, rx_mode);
 520
 521         return 0;
 522 }
 523
 524 int
 525 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 526 {
 527         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 528         uint32_t rx_mode = lif->rx_mode;
 529
 530         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 531
 532         ionic_set_rx_mode(lif, rx_mode);
 533
 534         return 0;
 535 }
 536
 537 int
 538 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 539 {
 540         struct ionic_admin_ctx ctx = {
 541                 .pending_work = true,
 542                 .cmd.lif_setattr = {
 543                         .opcode = IONIC_CMD_LIF_SETATTR,
 544                         .attr = IONIC_LIF_ATTR_MTU,
 545                         .mtu = new_mtu,
 546                 },
 547         };
 548         int err;
 549
 550         err = ionic_adminq_post_wait(lif, &ctx);
 551         if (err)
 552                 return err;
 553
 554         return 0;
 555 }
 556
 557 int
 558 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 559 {
 560         struct ionic_adapter *adapter = lif->adapter;
 561         struct ionic_dev *idev = &adapter->idev;
 562         unsigned long index;
 563
 564         /*
 565          * Note: interrupt handler is called for index = 0 only
 566          * (we use interrupts for the notifyq only anyway,
 567          * which has index = 0)
 568          */
 569
 570         for (index = 0; index < adapter->nintrs; index++)
 571                 if (!adapter->intrs[index])
 572                         break;
 573
 574         if (index == adapter->nintrs)
 575                 return -ENOSPC;
 576
 577         adapter->intrs[index] = true;
 578
 579         ionic_intr_init(idev, intr, index);
 580
 581         return 0;
 582 }
 583
 584 void
 585 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 586 {
 587         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 588                 lif->adapter->intrs[intr->index] = false;
 589 }
 590
 591 static int
 592 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 593                 uint32_t index,
 594                 const char *base, uint32_t flags,
 595                 uint32_t num_descs,
 596                 uint32_t desc_size,
 597                 uint32_t cq_desc_size,
 598                 uint32_t sg_desc_size,
 599                 struct ionic_qcq **qcq)
 600 {
 601         struct ionic_dev *idev = &lif->adapter->idev;
 602         struct ionic_qcq *new;
 603         uint32_t q_size, cq_size, sg_size, total_size;
 604         void *q_base, *cq_base, *sg_base;
 605         rte_iova_t q_base_pa = 0;
 606         rte_iova_t cq_base_pa = 0;
 607         rte_iova_t sg_base_pa = 0;
 608         uint32_t socket_id = rte_socket_id();
 609         int err;
 610
 611         *qcq = NULL;
 612
 613         q_size  = num_descs * desc_size;
 614         cq_size = num_descs * cq_desc_size;
 615         sg_size = num_descs * sg_desc_size;
 616
 617         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 618                 RTE_ALIGN(cq_size, PAGE_SIZE);
 619         /*
 620          * Note: aligning q_size/cq_size is not enough due to cq_base address
 621          * aligning as q_base could be not aligned to the page.
 622          * Adding PAGE_SIZE.
 623          */
 624         total_size += PAGE_SIZE;
 625
 626         if (flags & IONIC_QCQ_F_SG) {
 627                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 628                 total_size += PAGE_SIZE;
 629         }
 630
 631         new = rte_zmalloc("ionic", sizeof(*new), 0);
 632         if (!new) {
 633                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 634                 return -ENOMEM;
 635         }
 636
 637         new->lif = lif;
 638         new->flags = flags;
 639
 640         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 641         if (!new->q.info) {
 642                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 643                 return -ENOMEM;
 644         }
 645
 646         new->q.type = type;
 647
 648         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 649                 desc_size, sg_desc_size);
 650         if (err) {
 651                 IONIC_PRINT(ERR, "Queue initialization failed");
 652                 return err;
 653         }
 654
 655         if (flags & IONIC_QCQ_F_INTR) {
 656                 err = ionic_intr_alloc(lif, &new->intr);
 657                 if (err)
 658                         return err;
 659
 660                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 661                         IONIC_INTR_MASK_SET);
 662         } else {
 663                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 664         }
 665
 666         err = ionic_cq_init(lif, &new->cq, &new->intr,
 667                 num_descs, cq_desc_size);
 668         if (err) {
 669                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 670                 goto err_out_free_intr;
 671         }
 672
 673         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 674                 base /* name */, index /* queue_idx */,
 675                 total_size, IONIC_ALIGN, socket_id);
 676
 677         if (!new->base_z) {
 678                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 679                 err = -ENOMEM;
 680                 goto err_out_free_intr;
 681         }
 682
 683         new->base = new->base_z->addr;
 684         new->base_pa = new->base_z->iova;
 685         new->total_size = total_size;
 686
 687         q_base = new->base;
 688         q_base_pa = new->base_pa;
 689
 690         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 691         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 692
 693         if (flags & IONIC_QCQ_F_SG) {
 694                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 695                         PAGE_SIZE);
 696                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 697                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 698         }
 699
 700         IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 701                 "SG-base-PA = %#jx",
 702                 q_base_pa, cq_base_pa, sg_base_pa);
 703
 704         ionic_q_map(&new->q, q_base, q_base_pa);
 705         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 706         ionic_cq_bind(&new->cq, &new->q);
 707
 708         *qcq = new;
 709
 710         return 0;
 711
 712 err_out_free_intr:
 713         if (flags & IONIC_QCQ_F_INTR)
 714                 ionic_intr_free(lif, &new->intr);
 715
 716         return err;
 717 }
 718
 719 void
 720 ionic_qcq_free(struct ionic_qcq *qcq)
 721 {
 722         if (qcq->base_z) {
 723                 qcq->base = NULL;
 724                 qcq->base_pa = 0;
 725                 rte_memzone_free(qcq->base_z);
 726                 qcq->base_z = NULL;
 727         }
 728
 729         if (qcq->q.info) {
 730                 rte_free(qcq->q.info);
 731                 qcq->q.info = NULL;
 732         }
 733
 734         rte_free(qcq);
 735 }
 736
 737 int
 738 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 739                 struct ionic_qcq **qcq)
 740 {
 741         uint32_t flags;
 742         int err = -ENOMEM;
 743
 744         flags = IONIC_QCQ_F_SG;
 745         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 746                 nrxq_descs,
 747                 sizeof(struct ionic_rxq_desc),
 748                 sizeof(struct ionic_rxq_comp),
 749                 sizeof(struct ionic_rxq_sg_desc),
 750                 &lif->rxqcqs[index]);
 751         if (err)
 752                 return err;
 753
 754         *qcq = lif->rxqcqs[index];
 755
 756         return 0;
 757 }
 758
 759 int
 760 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 761                 struct ionic_qcq **qcq)
 762 {
 763         uint32_t flags;
 764         int err = -ENOMEM;
 765
 766         flags = IONIC_QCQ_F_SG;
 767         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 768                 ntxq_descs,
 769                 sizeof(struct ionic_txq_desc),
 770                 sizeof(struct ionic_txq_comp),
 771                 sizeof(struct ionic_txq_sg_desc),
 772                 &lif->txqcqs[index]);
 773         if (err)
 774                 return err;
 775
 776         *qcq = lif->txqcqs[index];
 777
 778         return 0;
 779 }
 780
 781 static int
 782 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 783 {
 784         uint32_t flags;
 785         int err = -ENOMEM;
 786
 787         flags = 0;
 788         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 789                 IONIC_ADMINQ_LENGTH,
 790                 sizeof(struct ionic_admin_cmd),
 791                 sizeof(struct ionic_admin_comp),
 792                 0,
 793                 &lif->adminqcq);
 794         if (err)
 795                 return err;
 796
 797         return 0;
 798 }
 799
 800 static int
 801 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 802 {
 803         uint32_t flags;
 804         int err = -ENOMEM;
 805
 806         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 807
 808         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 809                 flags,
 810                 IONIC_NOTIFYQ_LENGTH,
 811                 sizeof(struct ionic_notifyq_cmd),
 812                 sizeof(union ionic_notifyq_comp),
 813                 0,
 814                 &lif->notifyqcq);
 815         if (err)
 816                 return err;
 817
 818         return 0;
 819 }
 820
 821 static void *
 822 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 823 {
 824         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 825
 826         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 827                 return NULL;
 828
 829         return (void *)&vaddr[page_num << PAGE_SHIFT];
 830 }
 831
 832 int
 833 ionic_lif_alloc(struct ionic_lif *lif)
 834 {
 835         struct ionic_adapter *adapter = lif->adapter;
 836         uint32_t socket_id = rte_socket_id();
 837         int err;
 838
 839         /*
 840          * lif->name was zeroed on allocation.
 841          * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated.
 842          */
 843         memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1);
 844
 845         IONIC_PRINT(DEBUG, "LIF: %s", lif->name);
 846
 847         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 848
 849         rte_spinlock_init(&lif->adminq_lock);
 850         rte_spinlock_init(&lif->adminq_service_lock);
 851
 852         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, 0);
 853         if (!lif->kern_dbpage) {
 854                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 855                 return -ENOMEM;
 856         }
 857
 858         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 859                 adapter->max_ntxqs_per_lif, 0);
 860
 861         if (!lif->txqcqs) {
 862                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 863                 return -ENOMEM;
 864         }
 865
 866         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 867                 adapter->max_nrxqs_per_lif, 0);
 868
 869         if (!lif->rxqcqs) {
 870                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 871                 return -ENOMEM;
 872         }
 873
 874         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 875
 876         err = ionic_notify_qcq_alloc(lif);
 877         if (err) {
 878                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 879                 return err;
 880         }
 881
 882         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 883
 884         err = ionic_admin_qcq_alloc(lif);
 885         if (err) {
 886                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 887                 return err;
 888         }
 889
 890         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 891
 892         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 893
 894         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 895                 "lif_info", 0 /* queue_idx*/,
 896                 lif->info_sz, IONIC_ALIGN, socket_id);
 897         if (!lif->info_z) {
 898                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 899                 return -ENOMEM;
 900         }
 901
 902         lif->info = lif->info_z->addr;
 903         lif->info_pa = lif->info_z->iova;
 904
 905         return 0;
 906 }
 907
 908 void
 909 ionic_lif_free(struct ionic_lif *lif)
 910 {
 911         if (lif->notifyqcq) {
 912                 ionic_qcq_free(lif->notifyqcq);
 913                 lif->notifyqcq = NULL;
 914         }
 915
 916         if (lif->adminqcq) {
 917                 ionic_qcq_free(lif->adminqcq);
 918                 lif->adminqcq = NULL;
 919         }
 920
 921         if (lif->txqcqs) {
 922                 rte_free(lif->txqcqs);
 923                 lif->txqcqs = NULL;
 924         }
 925
 926         if (lif->rxqcqs) {
 927                 rte_free(lif->rxqcqs);
 928                 lif->rxqcqs = NULL;
 929         }
 930
 931         if (lif->info) {
 932                 rte_memzone_free(lif->info_z);
 933                 lif->info = NULL;
 934         }
 935 }
 936
 937 void
 938 ionic_lif_free_queues(struct ionic_lif *lif)
 939 {
 940         uint32_t i;
 941
 942         for (i = 0; i < lif->ntxqcqs; i++) {
 943                 ionic_dev_tx_queue_release(lif->eth_dev->data->tx_queues[i]);
 944                 lif->eth_dev->data->tx_queues[i] = NULL;
 945         }
 946         for (i = 0; i < lif->nrxqcqs; i++) {
 947                 ionic_dev_rx_queue_release(lif->eth_dev->data->rx_queues[i]);
 948                 lif->eth_dev->data->rx_queues[i] = NULL;
 949         }
 950 }
 951
 952 int
 953 ionic_lif_rss_config(struct ionic_lif *lif,
 954                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 955 {
 956         struct ionic_admin_ctx ctx = {
 957                 .pending_work = true,
 958                 .cmd.lif_setattr = {
 959                         .opcode = IONIC_CMD_LIF_SETATTR,
 960                         .attr = IONIC_LIF_ATTR_RSS,
 961                         .rss.types = types,
 962                         .rss.addr = lif->rss_ind_tbl_pa,
 963                 },
 964         };
 965         unsigned int i;
 966
 967         IONIC_PRINT_CALL();
 968
 969         lif->rss_types = types;
 970
 971         if (key)
 972                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 973
 974         if (indir)
 975                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 976                         lif->rss_ind_tbl[i] = indir[i];
 977
 978         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 979                IONIC_RSS_HASH_KEY_SIZE);
 980
 981         return ionic_adminq_post_wait(lif, &ctx);
 982 }
 983
 984 static int
 985 ionic_lif_rss_setup(struct ionic_lif *lif)
 986 {
 987         static const uint8_t toeplitz_symmetric_key[] = {
 988                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 989                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 990                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 991                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 992                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 993         };
 994         uint32_t i;
 995         uint16_t tbl_sz = lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 996
 997         IONIC_PRINT_CALL();
 998
 999         if (!lif->rss_ind_tbl_z) {
1000                 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
1001                                         "rss_ind_tbl", 0 /* queue_idx */,
1002                                         sizeof(*lif->rss_ind_tbl) * tbl_sz,
1003                                         IONIC_ALIGN, rte_socket_id());
1004                 if (!lif->rss_ind_tbl_z) {
1005                         IONIC_PRINT(ERR, "OOM");
1006                         return -ENOMEM;
1007                 }
1008
1009                 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
1010                 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
1011         }
1012
1013         if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) {
1014                 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs;
1015
1016                 /* Fill indirection table with 'default' values */
1017                 for (i = 0; i < tbl_sz; i++)
1018                         lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
1019         }
1020
1021         return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
1022                         toeplitz_symmetric_key, NULL);
1023 }
1024
1025 static void
1026 ionic_lif_rss_teardown(struct ionic_lif *lif)
1027 {
1028         if (!lif->rss_ind_tbl)
1029                 return;
1030
1031         if (lif->rss_ind_tbl_z) {
1032                 /* Disable RSS on the NIC */
1033                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1034
1035                 lif->rss_ind_tbl = NULL;
1036                 lif->rss_ind_tbl_pa = 0;
1037                 rte_memzone_free(lif->rss_ind_tbl_z);
1038                 lif->rss_ind_tbl_z = NULL;
1039         }
1040 }
1041
1042 static void
1043 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1044 {
1045         struct ionic_dev *idev = &lif->adapter->idev;
1046
1047         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1048                 return;
1049
1050         if (qcq->flags & IONIC_QCQ_F_INTR)
1051                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1052                         IONIC_INTR_MASK_SET);
1053
1054         qcq->flags &= ~IONIC_QCQ_F_INITED;
1055 }
1056
1057 void
1058 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1059 {
1060         ionic_lif_qcq_deinit(qcq->lif, qcq);
1061 }
1062
1063 void
1064 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1065 {
1066         ionic_lif_qcq_deinit(qcq->lif, qcq);
1067 }
1068
1069 bool
1070 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1071                 void *cb_arg __rte_unused)
1072 {
1073         struct ionic_admin_comp *cq_desc_base = cq->base;
1074         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1075
1076         if (!color_match(cq_desc->color, cq->done_color))
1077                 return false;
1078
1079         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1080
1081         return true;
1082 }
1083
1084 /* This acts like ionic_napi */
1085 int
1086 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1087                 void *cb_arg)
1088 {
1089         struct ionic_cq *cq = &qcq->cq;
1090         uint32_t work_done;
1091
1092         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1093
1094         return work_done;
1095 }
1096
1097 static void
1098 ionic_link_status_check(struct ionic_lif *lif)
1099 {
1100         struct ionic_adapter *adapter = lif->adapter;
1101         bool link_up;
1102
1103         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1104
1105         if (!lif->info)
1106                 return;
1107
1108         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1109
1110         if ((link_up  && adapter->link_up) ||
1111             (!link_up && !adapter->link_up))
1112                 return;
1113
1114         if (link_up) {
1115                 adapter->link_speed = lif->info->status.link_speed;
1116                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1117                         adapter->link_speed);
1118         } else {
1119                 IONIC_PRINT(DEBUG, "Link down");
1120         }
1121
1122         adapter->link_up = link_up;
1123         ionic_dev_link_update(lif->eth_dev, 0);
1124 }
1125
1126 static void
1127 ionic_lif_handle_fw_down(struct ionic_lif *lif)
1128 {
1129         if (lif->state & IONIC_LIF_F_FW_RESET)
1130                 return;
1131
1132         lif->state |= IONIC_LIF_F_FW_RESET;
1133
1134         if (lif->state & IONIC_LIF_F_UP) {
1135                 IONIC_PRINT(NOTICE,
1136                         "Surprise FW stop, stopping %s\n", lif->name);
1137                 ionic_lif_stop(lif);
1138         }
1139
1140         IONIC_PRINT(NOTICE, "FW down, %s stopped", lif->name);
1141 }
1142
1143 static bool
1144 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1145 {
1146         union ionic_notifyq_comp *cq_desc_base = cq->base;
1147         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1148         struct ionic_lif *lif = cb_arg;
1149
1150         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1151                 cq_desc->event.eid, cq_desc->event.ecode);
1152
1153         /* Have we run out of new completions to process? */
1154         if (!(cq_desc->event.eid > lif->last_eid))
1155                 return false;
1156
1157         lif->last_eid = cq_desc->event.eid;
1158
1159         switch (cq_desc->event.ecode) {
1160         case IONIC_EVENT_LINK_CHANGE:
1161                 IONIC_PRINT(DEBUG,
1162                         "Notifyq IONIC_EVENT_LINK_CHANGE %s "
1163                         "eid=%jd link_status=%d link_speed=%d",
1164                         lif->name,
1165                         cq_desc->event.eid,
1166                         cq_desc->link_change.link_status,
1167                         cq_desc->link_change.link_speed);
1168
1169                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1170                 break;
1171
1172         case IONIC_EVENT_RESET:
1173                 IONIC_PRINT(NOTICE,
1174                         "Notifyq IONIC_EVENT_RESET %s "
1175                         "eid=%jd, reset_code=%d state=%d",
1176                         lif->name,
1177                         cq_desc->event.eid,
1178                         cq_desc->reset.reset_code,
1179                         cq_desc->reset.state);
1180                 ionic_lif_handle_fw_down(lif);
1181                 break;
1182
1183         default:
1184                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1185                         cq_desc->event.ecode, cq_desc->event.eid);
1186                 break;
1187         }
1188
1189         return true;
1190 }
1191
1192 int
1193 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1194 {
1195         struct ionic_dev *idev = &lif->adapter->idev;
1196         struct ionic_qcq *qcq = lif->notifyqcq;
1197         uint32_t work_done;
1198
1199         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1200                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1201                 return -1;
1202         }
1203
1204         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1205                 IONIC_INTR_MASK_SET);
1206
1207         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1208
1209         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1210                 ionic_link_status_check(lif);
1211
1212         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1213                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1214
1215         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1216                 IONIC_INTR_MASK_CLEAR);
1217
1218         return 0;
1219 }
1220
1221 static int
1222 ionic_lif_adminq_init(struct ionic_lif *lif)
1223 {
1224         struct ionic_dev *idev = &lif->adapter->idev;
1225         struct ionic_qcq *qcq = lif->adminqcq;
1226         struct ionic_queue *q = &qcq->q;
1227         struct ionic_q_init_comp comp;
1228         int err;
1229
1230         ionic_dev_cmd_adminq_init(idev, qcq, qcq->intr.index);
1231         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1232         if (err)
1233                 return err;
1234
1235         ionic_dev_cmd_comp(idev, &comp);
1236
1237         q->hw_type = comp.hw_type;
1238         q->hw_index = comp.hw_index;
1239         q->db = ionic_db_map(lif, q);
1240
1241         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1242         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1243         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1244
1245         if (qcq->flags & IONIC_QCQ_F_INTR)
1246                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1247                         IONIC_INTR_MASK_CLEAR);
1248
1249         qcq->flags |= IONIC_QCQ_F_INITED;
1250
1251         return 0;
1252 }
1253
1254 static int
1255 ionic_lif_notifyq_init(struct ionic_lif *lif)
1256 {
1257         struct ionic_dev *idev = &lif->adapter->idev;
1258         struct ionic_qcq *qcq = lif->notifyqcq;
1259         struct ionic_queue *q = &qcq->q;
1260         int err;
1261
1262         struct ionic_admin_ctx ctx = {
1263                 .pending_work = true,
1264                 .cmd.q_init = {
1265                         .opcode = IONIC_CMD_Q_INIT,
1266                         .type = q->type,
1267                         .index = q->index,
1268                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1269                         .intr_index = qcq->intr.index,
1270                         .ring_size = rte_log2_u32(q->num_descs),
1271                         .ring_base = q->base_pa,
1272                 }
1273         };
1274
1275         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1276                 ctx.cmd.q_init.index);
1277         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1278                 ctx.cmd.q_init.ring_base);
1279         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1280                 ctx.cmd.q_init.ring_size);
1281         IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver);
1282
1283         err = ionic_adminq_post_wait(lif, &ctx);
1284         if (err)
1285                 return err;
1286
1287         q->hw_type = ctx.comp.q_init.hw_type;
1288         q->hw_index = ctx.comp.q_init.hw_index;
1289         q->db = NULL;
1290
1291         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1292         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1293         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1294
1295         if (qcq->flags & IONIC_QCQ_F_INTR)
1296                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1297                         IONIC_INTR_MASK_CLEAR);
1298
1299         qcq->flags |= IONIC_QCQ_F_INITED;
1300
1301         return 0;
1302 }
1303
1304 int
1305 ionic_lif_set_features(struct ionic_lif *lif)
1306 {
1307         struct ionic_admin_ctx ctx = {
1308                 .pending_work = true,
1309                 .cmd.lif_setattr = {
1310                         .opcode = IONIC_CMD_LIF_SETATTR,
1311                         .attr = IONIC_LIF_ATTR_FEATURES,
1312                         .features = lif->features,
1313                 },
1314         };
1315         int err;
1316
1317         err = ionic_adminq_post_wait(lif, &ctx);
1318         if (err)
1319                 return err;
1320
1321         lif->hw_features = (ctx.cmd.lif_setattr.features &
1322                 ctx.comp.lif_setattr.features);
1323
1324         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1325                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1326         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1327                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1328         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1329                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1330         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1331                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1332         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1333                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1334         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1335                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1336         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1337                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1338         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1339                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1340         if (lif->hw_features & IONIC_ETH_HW_TSO)
1341                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1342         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1343                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1344         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1345                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1346         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1347                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1348         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1349                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1350         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1351                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1352         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1353                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1354         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1355                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1356         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1357                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1358
1359         return 0;
1360 }
1361
1362 int
1363 ionic_lif_txq_init(struct ionic_qcq *qcq)
1364 {
1365         struct ionic_queue *q = &qcq->q;
1366         struct ionic_lif *lif = qcq->lif;
1367         struct ionic_cq *cq = &qcq->cq;
1368         struct ionic_admin_ctx ctx = {
1369                 .pending_work = true,
1370                 .cmd.q_init = {
1371                         .opcode = IONIC_CMD_Q_INIT,
1372                         .type = q->type,
1373                         .index = q->index,
1374                         .flags = IONIC_QINIT_F_SG,
1375                         .intr_index = cq->bound_intr->index,
1376                         .ring_size = rte_log2_u32(q->num_descs),
1377                         .ring_base = q->base_pa,
1378                         .cq_ring_base = cq->base_pa,
1379                         .sg_ring_base = q->sg_base_pa,
1380                 },
1381         };
1382         int err;
1383
1384         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1385         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1386                 ctx.cmd.q_init.ring_base);
1387         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1388                 ctx.cmd.q_init.ring_size);
1389         IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver);
1390
1391         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1392         if (err)
1393                 return err;
1394
1395         q->hw_type = ctx.comp.q_init.hw_type;
1396         q->hw_index = ctx.comp.q_init.hw_index;
1397         q->db = ionic_db_map(lif, q);
1398
1399         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1400         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1401         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1402
1403         qcq->flags |= IONIC_QCQ_F_INITED;
1404
1405         return 0;
1406 }
1407
1408 int
1409 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1410 {
1411         struct ionic_queue *q = &qcq->q;
1412         struct ionic_lif *lif = qcq->lif;
1413         struct ionic_cq *cq = &qcq->cq;
1414         struct ionic_admin_ctx ctx = {
1415                 .pending_work = true,
1416                 .cmd.q_init = {
1417                         .opcode = IONIC_CMD_Q_INIT,
1418                         .type = q->type,
1419                         .index = q->index,
1420                         .flags = IONIC_QINIT_F_SG,
1421                         .intr_index = cq->bound_intr->index,
1422                         .ring_size = rte_log2_u32(q->num_descs),
1423                         .ring_base = q->base_pa,
1424                         .cq_ring_base = cq->base_pa,
1425                         .sg_ring_base = q->sg_base_pa,
1426                 },
1427         };
1428         int err;
1429
1430         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1431         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1432                 ctx.cmd.q_init.ring_base);
1433         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1434                 ctx.cmd.q_init.ring_size);
1435         IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver);
1436
1437         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1438         if (err)
1439                 return err;
1440
1441         q->hw_type = ctx.comp.q_init.hw_type;
1442         q->hw_index = ctx.comp.q_init.hw_index;
1443         q->db = ionic_db_map(lif, q);
1444
1445         qcq->flags |= IONIC_QCQ_F_INITED;
1446
1447         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1448         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1449         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1450
1451         return 0;
1452 }
1453
1454 static int
1455 ionic_station_set(struct ionic_lif *lif)
1456 {
1457         struct ionic_admin_ctx ctx = {
1458                 .pending_work = true,
1459                 .cmd.lif_getattr = {
1460                         .opcode = IONIC_CMD_LIF_GETATTR,
1461                         .attr = IONIC_LIF_ATTR_MAC,
1462                 },
1463         };
1464         int err;
1465
1466         IONIC_PRINT_CALL();
1467
1468         err = ionic_adminq_post_wait(lif, &ctx);
1469         if (err)
1470                 return err;
1471
1472         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1473                         lif->mac_addr)) {
1474                 IONIC_PRINT(INFO, "deleting station MAC addr");
1475
1476                 ionic_lif_addr_del(lif, lif->mac_addr);
1477         }
1478
1479         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1480
1481         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1482                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1483                 return 0;
1484         }
1485
1486         IONIC_PRINT(DEBUG, "adding station MAC addr");
1487
1488         ionic_lif_addr_add(lif, lif->mac_addr);
1489
1490         return 0;
1491 }
1492
1493 static void
1494 ionic_lif_set_name(struct ionic_lif *lif)
1495 {
1496         struct ionic_admin_ctx ctx = {
1497                 .pending_work = true,
1498                 .cmd.lif_setattr = {
1499                         .opcode = IONIC_CMD_LIF_SETATTR,
1500                         .attr = IONIC_LIF_ATTR_NAME,
1501                 },
1502         };
1503
1504         memcpy(ctx.cmd.lif_setattr.name, lif->name,
1505                 sizeof(ctx.cmd.lif_setattr.name) - 1);
1506
1507         ionic_adminq_post_wait(lif, &ctx);
1508 }
1509
1510 int
1511 ionic_lif_init(struct ionic_lif *lif)
1512 {
1513         struct ionic_dev *idev = &lif->adapter->idev;
1514         struct ionic_q_init_comp comp;
1515         int err;
1516
1517         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1518
1519         ionic_dev_cmd_lif_init(idev, lif->info_pa);
1520         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1521         ionic_dev_cmd_comp(idev, &comp);
1522         if (err)
1523                 return err;
1524
1525         lif->hw_index = comp.hw_index;
1526
1527         err = ionic_lif_adminq_init(lif);
1528         if (err)
1529                 return err;
1530
1531         err = ionic_lif_notifyq_init(lif);
1532         if (err)
1533                 goto err_out_adminq_deinit;
1534
1535         lif->features =
1536                   IONIC_ETH_HW_VLAN_TX_TAG
1537                 | IONIC_ETH_HW_VLAN_RX_STRIP
1538                 | IONIC_ETH_HW_VLAN_RX_FILTER
1539                 | IONIC_ETH_HW_RX_HASH
1540                 | IONIC_ETH_HW_TX_SG
1541                 | IONIC_ETH_HW_RX_SG
1542                 | IONIC_ETH_HW_TX_CSUM
1543                 | IONIC_ETH_HW_RX_CSUM
1544                 | IONIC_ETH_HW_TSO
1545                 | IONIC_ETH_HW_TSO_IPV6
1546                 | IONIC_ETH_HW_TSO_ECN;
1547
1548         err = ionic_lif_set_features(lif);
1549         if (err)
1550                 goto err_out_notifyq_deinit;
1551
1552         err = ionic_rx_filters_init(lif);
1553         if (err)
1554                 goto err_out_notifyq_deinit;
1555
1556         err = ionic_station_set(lif);
1557         if (err)
1558                 goto err_out_rx_filter_deinit;
1559
1560         ionic_lif_set_name(lif);
1561
1562         lif->state |= IONIC_LIF_F_INITED;
1563
1564         return 0;
1565
1566 err_out_rx_filter_deinit:
1567         ionic_rx_filters_deinit(lif);
1568
1569 err_out_notifyq_deinit:
1570         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1571
1572 err_out_adminq_deinit:
1573         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1574
1575         return err;
1576 }
1577
1578 void
1579 ionic_lif_deinit(struct ionic_lif *lif)
1580 {
1581         if (!(lif->state & IONIC_LIF_F_INITED))
1582                 return;
1583
1584         ionic_rx_filters_deinit(lif);
1585         ionic_lif_rss_teardown(lif);
1586         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1587         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1588
1589         lif->state &= ~IONIC_LIF_F_INITED;
1590 }
1591
1592 int
1593 ionic_lif_configure(struct ionic_lif *lif)
1594 {
1595         struct ionic_identity *ident = &lif->adapter->ident;
1596         uint32_t ntxqs_per_lif =
1597                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1598         uint32_t nrxqs_per_lif =
1599                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1600         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1601         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1602
1603         lif->port_id = lif->eth_dev->data->port_id;
1604
1605         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1606                 lif->port_id);
1607
1608         if (nrxqs > 0)
1609                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1610
1611         if (ntxqs > 0)
1612                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1613
1614         lif->nrxqcqs = nrxqs_per_lif;
1615         lif->ntxqcqs = ntxqs_per_lif;
1616
1617         return 0;
1618 }
1619
1620 int
1621 ionic_lif_start(struct ionic_lif *lif)
1622 {
1623         uint32_t rx_mode;
1624         uint32_t i;
1625         int err;
1626
1627         err = ionic_lif_rss_setup(lif);
1628         if (err)
1629                 return err;
1630
1631         if (!lif->rx_mode) {
1632                 IONIC_PRINT(DEBUG, "Setting RX mode on %s",
1633                         lif->name);
1634
1635                 rx_mode  = IONIC_RX_MODE_F_UNICAST;
1636                 rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1637                 rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1638
1639                 ionic_set_rx_mode(lif, rx_mode);
1640         }
1641
1642         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1643                 "on port %u",
1644                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1645
1646         for (i = 0; i < lif->nrxqcqs; i++) {
1647                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1648                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1649                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1650
1651                         if (err)
1652                                 return err;
1653                 }
1654         }
1655
1656         for (i = 0; i < lif->ntxqcqs; i++) {
1657                 struct ionic_qcq *txq = lif->txqcqs[i];
1658                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1659                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1660
1661                         if (err)
1662                                 return err;
1663                 }
1664         }
1665
1666         /* Carrier ON here */
1667         lif->state |= IONIC_LIF_F_UP;
1668
1669         ionic_link_status_check(lif);
1670
1671         return 0;
1672 }
1673
1674 int
1675 ionic_lif_identify(struct ionic_adapter *adapter)
1676 {
1677         struct ionic_dev *idev = &adapter->idev;
1678         struct ionic_identity *ident = &adapter->ident;
1679         int err;
1680         unsigned int i;
1681         unsigned int lif_words = sizeof(ident->lif.words) /
1682                 sizeof(ident->lif.words[0]);
1683         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1684                 sizeof(idev->dev_cmd->data[0]);
1685         unsigned int nwords;
1686
1687         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1688                 IONIC_IDENTITY_VERSION_1);
1689         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1690         if (err)
1691                 return (err);
1692
1693         nwords = RTE_MIN(lif_words, cmd_words);
1694         for (i = 0; i < nwords; i++)
1695                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1696
1697         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1698                 ident->lif.capabilities);
1699
1700         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1701                 ident->lif.eth.max_ucast_filters);
1702         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1703                 ident->lif.eth.max_mcast_filters);
1704
1705         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1706                 ident->lif.eth.config.features);
1707         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1708                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1709         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1710                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1711         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1712                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1713         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1714                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1715
1716         return 0;
1717 }
1718
1719 int
1720 ionic_lifs_size(struct ionic_adapter *adapter)
1721 {
1722         struct ionic_identity *ident = &adapter->ident;
1723         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1724
1725         adapter->max_ntxqs_per_lif =
1726                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1727         adapter->max_nrxqs_per_lif =
1728                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1729
1730         nintrs = 1 /* notifyq */;
1731
1732         if (nintrs > dev_nintrs) {
1733                 IONIC_PRINT(ERR,
1734                         "At most %d intr supported, minimum req'd is %u",
1735                         dev_nintrs, nintrs);
1736                 return -ENOSPC;
1737         }
1738
1739         adapter->nintrs = nintrs;
1740
1741         return 0;
1742 }