drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .lif_index = lif->index,
  29                         .type = q->type,
  30                         .index = q->index,
  31                         .oper = IONIC_Q_ENABLE,
  32                 },
  33         };
  34
  35         if (qcq->flags & IONIC_QCQ_F_INTR) {
  36                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  37                         IONIC_INTR_MASK_CLEAR);
  38         }
  39
  40         return ionic_adminq_post_wait(lif, &ctx);
  41 }
  42
  43 int
  44 ionic_qcq_disable(struct ionic_qcq *qcq)
  45 {
  46         struct ionic_queue *q = &qcq->q;
  47         struct ionic_lif *lif = q->lif;
  48         struct ionic_dev *idev = &lif->adapter->idev;
  49         struct ionic_admin_ctx ctx = {
  50                 .pending_work = true,
  51                 .cmd.q_control = {
  52                         .opcode = IONIC_CMD_Q_CONTROL,
  53                         .lif_index = lif->index,
  54                         .type = q->type,
  55                         .index = q->index,
  56                         .oper = IONIC_Q_DISABLE,
  57                 },
  58         };
  59
  60         if (qcq->flags & IONIC_QCQ_F_INTR) {
  61                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  62                         IONIC_INTR_MASK_SET);
  63         }
  64
  65         return ionic_adminq_post_wait(lif, &ctx);
  66 }
  67
  68 int
  69 ionic_lif_stop(struct ionic_lif *lif __rte_unused)
  70 {
  71         /* Carrier OFF here */
  72
  73         return 0;
  74 }
  75
  76 void
  77 ionic_lif_reset(struct ionic_lif *lif)
  78 {
  79         struct ionic_dev *idev = &lif->adapter->idev;
  80
  81         IONIC_PRINT_CALL();
  82
  83         ionic_dev_cmd_lif_reset(idev, lif->index);
  84         ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  85 }
  86
  87 static void
  88 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  89 {
  90         struct ionic_lif_stats *ls = &lif->info->stats;
  91         uint32_t i;
  92         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  93                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  94         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  95                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  96
  97         memset(stats, 0, sizeof(*stats));
  98
  99         if (ls == NULL) {
 100                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 101                         lif->port_id);
 102                 return;
 103         }
 104
 105         /* RX */
 106
 107         stats->ipackets = ls->rx_ucast_packets +
 108                 ls->rx_mcast_packets +
 109                 ls->rx_bcast_packets;
 110
 111         stats->ibytes = ls->rx_ucast_bytes +
 112                 ls->rx_mcast_bytes +
 113                 ls->rx_bcast_bytes;
 114
 115         for (i = 0; i < lif->nrxqcqs; i++) {
 116                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 117                 stats->imissed +=
 118                         rx_stats->no_cb_arg +
 119                         rx_stats->bad_cq_status +
 120                         rx_stats->no_room +
 121                         rx_stats->bad_len;
 122         }
 123
 124         stats->imissed +=
 125                 ls->rx_ucast_drop_packets +
 126                 ls->rx_mcast_drop_packets +
 127                 ls->rx_bcast_drop_packets;
 128
 129         stats->imissed +=
 130                 ls->rx_queue_empty +
 131                 ls->rx_dma_error +
 132                 ls->rx_queue_disabled +
 133                 ls->rx_desc_fetch_error +
 134                 ls->rx_desc_data_error;
 135
 136         for (i = 0; i < num_rx_q_counters; i++) {
 137                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 138                 stats->q_ipackets[i] = rx_stats->packets;
 139                 stats->q_ibytes[i] = rx_stats->bytes;
 140                 stats->q_errors[i] =
 141                         rx_stats->no_cb_arg +
 142                         rx_stats->bad_cq_status +
 143                         rx_stats->no_room +
 144                         rx_stats->bad_len;
 145         }
 146
 147         /* TX */
 148
 149         stats->opackets = ls->tx_ucast_packets +
 150                 ls->tx_mcast_packets +
 151                 ls->tx_bcast_packets;
 152
 153         stats->obytes = ls->tx_ucast_bytes +
 154                 ls->tx_mcast_bytes +
 155                 ls->tx_bcast_bytes;
 156
 157         for (i = 0; i < lif->ntxqcqs; i++) {
 158                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 159                 stats->oerrors += tx_stats->drop;
 160         }
 161
 162         stats->oerrors +=
 163                 ls->tx_ucast_drop_packets +
 164                 ls->tx_mcast_drop_packets +
 165                 ls->tx_bcast_drop_packets;
 166
 167         stats->oerrors +=
 168                 ls->tx_dma_error +
 169                 ls->tx_queue_disabled +
 170                 ls->tx_desc_fetch_error +
 171                 ls->tx_desc_data_error;
 172
 173         for (i = 0; i < num_tx_q_counters; i++) {
 174                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 175                 stats->q_opackets[i] = tx_stats->packets;
 176                 stats->q_obytes[i] = tx_stats->bytes;
 177         }
 178 }
 179
 180 void
 181 ionic_lif_get_stats(const struct ionic_lif *lif,
 182                 struct rte_eth_stats *stats)
 183 {
 184         ionic_lif_get_abs_stats(lif, stats);
 185
 186         stats->ipackets  -= lif->stats_base.ipackets;
 187         stats->opackets  -= lif->stats_base.opackets;
 188         stats->ibytes    -= lif->stats_base.ibytes;
 189         stats->obytes    -= lif->stats_base.obytes;
 190         stats->imissed   -= lif->stats_base.imissed;
 191         stats->ierrors   -= lif->stats_base.ierrors;
 192         stats->oerrors   -= lif->stats_base.oerrors;
 193         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 194 }
 195
 196 void
 197 ionic_lif_reset_stats(struct ionic_lif *lif)
 198 {
 199         uint32_t i;
 200
 201         for (i = 0; i < lif->nrxqcqs; i++) {
 202                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 203                         sizeof(struct ionic_rx_stats));
 204                 memset(&lif->txqcqs[i]->stats.tx, 0,
 205                         sizeof(struct ionic_tx_stats));
 206         }
 207
 208         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 209 }
 210
 211 void
 212 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 213 {
 214         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 215         uint64_t *stats64 = (uint64_t *)stats;
 216         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 217         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 218
 219         for (i = 0; i < count; i++)
 220                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 221 }
 222
 223 void
 224 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 225 {
 226         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 227         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 228         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 229
 230         for (i = 0; i < count; i++)
 231                 lif_stats64_base[i] = lif_stats64[i];
 232 }
 233
 234 static int
 235 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 236 {
 237         struct ionic_admin_ctx ctx = {
 238                 .pending_work = true,
 239                 .cmd.rx_filter_add = {
 240                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 241                         .match = IONIC_RX_FILTER_MATCH_MAC,
 242                 },
 243         };
 244         int err;
 245
 246         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 247
 248         err = ionic_adminq_post_wait(lif, &ctx);
 249         if (err)
 250                 return err;
 251
 252         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 253                 ctx.comp.rx_filter_add.filter_id);
 254
 255         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 256 }
 257
 258 static int
 259 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 260 {
 261         struct ionic_admin_ctx ctx = {
 262                 .pending_work = true,
 263                 .cmd.rx_filter_del = {
 264                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 265                 },
 266         };
 267         struct ionic_rx_filter *f;
 268         int err;
 269
 270         IONIC_PRINT_CALL();
 271
 272         rte_spinlock_lock(&lif->rx_filters.lock);
 273
 274         f = ionic_rx_filter_by_addr(lif, addr);
 275         if (!f) {
 276                 rte_spinlock_unlock(&lif->rx_filters.lock);
 277                 return -ENOENT;
 278         }
 279
 280         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 281         ionic_rx_filter_free(f);
 282
 283         rte_spinlock_unlock(&lif->rx_filters.lock);
 284
 285         err = ionic_adminq_post_wait(lif, &ctx);
 286         if (err)
 287                 return err;
 288
 289         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 290                 ctx.cmd.rx_filter_del.filter_id);
 291
 292         return 0;
 293 }
 294
 295 int
 296 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 297                 struct rte_ether_addr *mac_addr,
 298                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 299 {
 300         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 301
 302         IONIC_PRINT_CALL();
 303
 304         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 305 }
 306
 307 void
 308 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index __rte_unused)
 309 {
 310         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 311         struct ionic_adapter *adapter = lif->adapter;
 312
 313         IONIC_PRINT_CALL();
 314
 315         if (index >= adapter->max_mac_addrs) {
 316                 IONIC_PRINT(WARNING,
 317                         "Index %u is above MAC filter limit %u",
 318                         index, adapter->max_mac_addrs);
 319                 return;
 320         }
 321
 322         if (!rte_is_valid_assigned_ether_addr(&eth_dev->data->mac_addrs[index]))
 323                 return;
 324
 325         ionic_lif_addr_del(lif, (const uint8_t *)
 326                 &eth_dev->data->mac_addrs[index]);
 327 }
 328
 329 int
 330 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 331 {
 332         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 333
 334         IONIC_PRINT_CALL();
 335
 336         if (mac_addr == NULL) {
 337                 IONIC_PRINT(NOTICE, "New mac is null");
 338                 return -1;
 339         }
 340
 341         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 342                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 343                         lif->mac_addr);
 344                 ionic_lif_addr_del(lif, lif->mac_addr);
 345                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 346         }
 347
 348         IONIC_PRINT(INFO, "Updating mac addr");
 349
 350         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 351
 352         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 353 }
 354
 355 static int
 356 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 357 {
 358         struct ionic_admin_ctx ctx = {
 359                 .pending_work = true,
 360                 .cmd.rx_filter_add = {
 361                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 362                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 363                         .vlan.vlan = vid,
 364                 },
 365         };
 366         int err;
 367
 368         err = ionic_adminq_post_wait(lif, &ctx);
 369         if (err)
 370                 return err;
 371
 372         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 373                 ctx.comp.rx_filter_add.filter_id);
 374
 375         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 376 }
 377
 378 static int
 379 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 380 {
 381         struct ionic_admin_ctx ctx = {
 382                 .pending_work = true,
 383                 .cmd.rx_filter_del = {
 384                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 385                 },
 386         };
 387         struct ionic_rx_filter *f;
 388         int err;
 389
 390         IONIC_PRINT_CALL();
 391
 392         rte_spinlock_lock(&lif->rx_filters.lock);
 393
 394         f = ionic_rx_filter_by_vlan(lif, vid);
 395         if (!f) {
 396                 rte_spinlock_unlock(&lif->rx_filters.lock);
 397                 return -ENOENT;
 398         }
 399
 400         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 401         ionic_rx_filter_free(f);
 402         rte_spinlock_unlock(&lif->rx_filters.lock);
 403
 404         err = ionic_adminq_post_wait(lif, &ctx);
 405         if (err)
 406                 return err;
 407
 408         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 409                 ctx.cmd.rx_filter_del.filter_id);
 410
 411         return 0;
 412 }
 413
 414 int
 415 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 416                 int on)
 417 {
 418         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 419         int err;
 420
 421         if (on)
 422                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 423         else
 424                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 425
 426         return err;
 427 }
 428
 429 static void
 430 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 431 {
 432         struct ionic_admin_ctx ctx = {
 433                 .pending_work = true,
 434                 .cmd.rx_mode_set = {
 435                         .opcode = IONIC_CMD_RX_MODE_SET,
 436                         .lif_index = lif->index,
 437                         .rx_mode = rx_mode,
 438                 },
 439         };
 440         int err;
 441
 442         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 443                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 444         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 445                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 446         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 447                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 448         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 449                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 450         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 451                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 452
 453         err = ionic_adminq_post_wait(lif, &ctx);
 454         if (err)
 455                 IONIC_PRINT(ERR, "Failure setting RX mode");
 456 }
 457
 458 static void
 459 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 460 {
 461         if (lif->rx_mode != rx_mode) {
 462                 lif->rx_mode = rx_mode;
 463                 ionic_lif_rx_mode(lif, rx_mode);
 464         }
 465 }
 466
 467 int
 468 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 469 {
 470         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 471         uint32_t rx_mode = lif->rx_mode;
 472
 473         IONIC_PRINT_CALL();
 474
 475         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 476
 477         ionic_set_rx_mode(lif, rx_mode);
 478
 479         return 0;
 480 }
 481
 482 int
 483 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 484 {
 485         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 486         uint32_t rx_mode = lif->rx_mode;
 487
 488         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 489
 490         ionic_set_rx_mode(lif, rx_mode);
 491
 492         return 0;
 493 }
 494
 495 int
 496 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 497 {
 498         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 499         uint32_t rx_mode = lif->rx_mode;
 500
 501         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 502
 503         ionic_set_rx_mode(lif, rx_mode);
 504
 505         return 0;
 506 }
 507
 508 int
 509 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 510 {
 511         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 512         uint32_t rx_mode = lif->rx_mode;
 513
 514         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 515
 516         ionic_set_rx_mode(lif, rx_mode);
 517
 518         return 0;
 519 }
 520
 521 int
 522 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 523 {
 524         struct ionic_admin_ctx ctx = {
 525                 .pending_work = true,
 526                 .cmd.lif_setattr = {
 527                         .opcode = IONIC_CMD_LIF_SETATTR,
 528                         .index = lif->index,
 529                         .attr = IONIC_LIF_ATTR_MTU,
 530                         .mtu = new_mtu,
 531                 },
 532         };
 533         int err;
 534
 535         err = ionic_adminq_post_wait(lif, &ctx);
 536         if (err)
 537                 return err;
 538
 539         return 0;
 540 }
 541
 542 int
 543 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 544 {
 545         struct ionic_adapter *adapter = lif->adapter;
 546         struct ionic_dev *idev = &adapter->idev;
 547         unsigned long index;
 548
 549         /*
 550          * Note: interrupt handler is called for index = 0 only
 551          * (we use interrupts for the notifyq only anyway,
 552          * which hash index = 0)
 553          */
 554
 555         for (index = 0; index < adapter->nintrs; index++)
 556                 if (!adapter->intrs[index])
 557                         break;
 558
 559         if (index == adapter->nintrs)
 560                 return -ENOSPC;
 561
 562         adapter->intrs[index] = true;
 563
 564         ionic_intr_init(idev, intr, index);
 565
 566         return 0;
 567 }
 568
 569 void
 570 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 571 {
 572         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 573                 lif->adapter->intrs[intr->index] = false;
 574 }
 575
 576 static int
 577 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 578                 uint32_t index,
 579                 const char *base, uint32_t flags,
 580                 uint32_t num_descs,
 581                 uint32_t desc_size,
 582                 uint32_t cq_desc_size,
 583                 uint32_t sg_desc_size,
 584                 struct ionic_qcq **qcq)
 585 {
 586         struct ionic_dev *idev = &lif->adapter->idev;
 587         struct ionic_qcq *new;
 588         uint32_t q_size, cq_size, sg_size, total_size;
 589         void *q_base, *cq_base, *sg_base;
 590         rte_iova_t q_base_pa = 0;
 591         rte_iova_t cq_base_pa = 0;
 592         rte_iova_t sg_base_pa = 0;
 593         uint32_t socket_id = rte_socket_id();
 594         int err;
 595
 596         *qcq = NULL;
 597
 598         q_size  = num_descs * desc_size;
 599         cq_size = num_descs * cq_desc_size;
 600         sg_size = num_descs * sg_desc_size;
 601
 602         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 603                 RTE_ALIGN(cq_size, PAGE_SIZE);
 604         /*
 605          * Note: aligning q_size/cq_size is not enough due to cq_base address
 606          * aligning as q_base could be not aligned to the page.
 607          * Adding PAGE_SIZE.
 608          */
 609         total_size += PAGE_SIZE;
 610
 611         if (flags & IONIC_QCQ_F_SG) {
 612                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 613                 total_size += PAGE_SIZE;
 614         }
 615
 616         new = rte_zmalloc("ionic", sizeof(*new), 0);
 617         if (!new) {
 618                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 619                 return -ENOMEM;
 620         }
 621
 622         new->lif = lif;
 623         new->flags = flags;
 624
 625         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 626         if (!new->q.info) {
 627                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 628                 return -ENOMEM;
 629         }
 630
 631         new->q.type = type;
 632
 633         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 634                 desc_size, sg_desc_size);
 635         if (err) {
 636                 IONIC_PRINT(ERR, "Queue initialization failed");
 637                 return err;
 638         }
 639
 640         if (flags & IONIC_QCQ_F_INTR) {
 641                 err = ionic_intr_alloc(lif, &new->intr);
 642                 if (err)
 643                         return err;
 644
 645                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 646                         IONIC_INTR_MASK_SET);
 647         } else {
 648                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 649         }
 650
 651         err = ionic_cq_init(lif, &new->cq, &new->intr,
 652                 num_descs, cq_desc_size);
 653         if (err) {
 654                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 655                 goto err_out_free_intr;
 656         }
 657
 658         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 659                 base /* name */, index /* queue_idx */,
 660                 total_size, IONIC_ALIGN, socket_id);
 661
 662         if (!new->base_z) {
 663                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 664                 err = -ENOMEM;
 665                 goto err_out_free_intr;
 666         }
 667
 668         new->base = new->base_z->addr;
 669         new->base_pa = new->base_z->iova;
 670         new->total_size = total_size;
 671
 672         q_base = new->base;
 673         q_base_pa = new->base_pa;
 674
 675         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 676         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 677
 678         if (flags & IONIC_QCQ_F_SG) {
 679                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 680                         PAGE_SIZE);
 681                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 682                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 683         }
 684
 685         IONIC_PRINT(DEBUG, "Q-Base-PA = %ju CQ-Base-PA = %ju "
 686                 "SG-base-PA = %ju",
 687                 q_base_pa, cq_base_pa, sg_base_pa);
 688
 689         ionic_q_map(&new->q, q_base, q_base_pa);
 690         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 691         ionic_cq_bind(&new->cq, &new->q);
 692
 693         *qcq = new;
 694
 695         return 0;
 696
 697 err_out_free_intr:
 698         if (flags & IONIC_QCQ_F_INTR)
 699                 ionic_intr_free(lif, &new->intr);
 700
 701         return err;
 702 }
 703
 704 void
 705 ionic_qcq_free(struct ionic_qcq *qcq)
 706 {
 707         if (qcq->base_z) {
 708                 qcq->base = NULL;
 709                 qcq->base_pa = 0;
 710                 rte_memzone_free(qcq->base_z);
 711                 qcq->base_z = NULL;
 712         }
 713
 714         if (qcq->q.info) {
 715                 rte_free(qcq->q.info);
 716                 qcq->q.info = NULL;
 717         }
 718
 719         rte_free(qcq);
 720 }
 721
 722 int
 723 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 724                 struct ionic_qcq **qcq)
 725 {
 726         uint32_t flags;
 727         int err = -ENOMEM;
 728
 729         flags = IONIC_QCQ_F_SG;
 730         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 731                 nrxq_descs,
 732                 sizeof(struct ionic_rxq_desc),
 733                 sizeof(struct ionic_rxq_comp),
 734                 sizeof(struct ionic_rxq_sg_desc),
 735                 &lif->rxqcqs[index]);
 736         if (err)
 737                 return err;
 738
 739         *qcq = lif->rxqcqs[index];
 740
 741         return 0;
 742 }
 743
 744 int
 745 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 746                 struct ionic_qcq **qcq)
 747 {
 748         uint32_t flags;
 749         int err = -ENOMEM;
 750
 751         flags = IONIC_QCQ_F_SG;
 752         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 753                 ntxq_descs,
 754                 sizeof(struct ionic_txq_desc),
 755                 sizeof(struct ionic_txq_comp),
 756                 sizeof(struct ionic_txq_sg_desc),
 757                 &lif->txqcqs[index]);
 758         if (err)
 759                 return err;
 760
 761         *qcq = lif->txqcqs[index];
 762
 763         return 0;
 764 }
 765
 766 static int
 767 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 768 {
 769         uint32_t flags;
 770         int err = -ENOMEM;
 771
 772         flags = 0;
 773         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 774                 IONIC_ADMINQ_LENGTH,
 775                 sizeof(struct ionic_admin_cmd),
 776                 sizeof(struct ionic_admin_comp),
 777                 0,
 778                 &lif->adminqcq);
 779         if (err)
 780                 return err;
 781
 782         return 0;
 783 }
 784
 785 static int
 786 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 787 {
 788         uint32_t flags;
 789         int err = -ENOMEM;
 790
 791         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 792
 793         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 794                 flags,
 795                 IONIC_NOTIFYQ_LENGTH,
 796                 sizeof(struct ionic_notifyq_cmd),
 797                 sizeof(union ionic_notifyq_comp),
 798                 0,
 799                 &lif->notifyqcq);
 800         if (err)
 801                 return err;
 802
 803         return 0;
 804 }
 805
 806 static void *
 807 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 808 {
 809         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 810
 811         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 812                 return NULL;
 813
 814         return (void *)&vaddr[page_num << PAGE_SHIFT];
 815 }
 816
 817 int
 818 ionic_lif_alloc(struct ionic_lif *lif)
 819 {
 820         struct ionic_adapter *adapter = lif->adapter;
 821         uint32_t socket_id = rte_socket_id();
 822         int dbpage_num;
 823         int err;
 824
 825         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
 826
 827         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 828
 829         rte_spinlock_init(&lif->adminq_lock);
 830         rte_spinlock_init(&lif->adminq_service_lock);
 831
 832         dbpage_num = ionic_db_page_num(lif, 0);
 833
 834         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, dbpage_num);
 835         if (!lif->kern_dbpage) {
 836                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 837                 return -ENOMEM;
 838         }
 839
 840         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 841                 adapter->max_ntxqs_per_lif, 0);
 842
 843         if (!lif->txqcqs) {
 844                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 845                 return -ENOMEM;
 846         }
 847
 848         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 849                 adapter->max_nrxqs_per_lif, 0);
 850
 851         if (!lif->rxqcqs) {
 852                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 853                 return -ENOMEM;
 854         }
 855
 856         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 857
 858         err = ionic_notify_qcq_alloc(lif);
 859         if (err) {
 860                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 861                 return err;
 862         }
 863
 864         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 865
 866         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 867
 868         err = ionic_admin_qcq_alloc(lif);
 869         if (err) {
 870                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 871                 return err;
 872         }
 873
 874         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 875
 876         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 877
 878         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 879                 "lif_info", 0 /* queue_idx*/,
 880                 lif->info_sz, IONIC_ALIGN, socket_id);
 881         if (!lif->info_z) {
 882                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 883                 return -ENOMEM;
 884         }
 885
 886         lif->info = lif->info_z->addr;
 887         lif->info_pa = lif->info_z->iova;
 888
 889         return 0;
 890 }
 891
 892 void
 893 ionic_lif_free(struct ionic_lif *lif)
 894 {
 895         if (lif->notifyqcq) {
 896                 ionic_qcq_free(lif->notifyqcq);
 897                 lif->notifyqcq = NULL;
 898         }
 899
 900         if (lif->adminqcq) {
 901                 ionic_qcq_free(lif->adminqcq);
 902                 lif->adminqcq = NULL;
 903         }
 904
 905         if (lif->txqcqs) {
 906                 rte_free(lif->txqcqs);
 907                 lif->txqcqs = NULL;
 908         }
 909
 910         if (lif->rxqcqs) {
 911                 rte_free(lif->rxqcqs);
 912                 lif->rxqcqs = NULL;
 913         }
 914
 915         if (lif->info) {
 916                 rte_memzone_free(lif->info_z);
 917                 lif->info = NULL;
 918         }
 919 }
 920
 921 int
 922 ionic_lif_rss_config(struct ionic_lif *lif,
 923                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 924 {
 925         struct ionic_admin_ctx ctx = {
 926                 .pending_work = true,
 927                 .cmd.lif_setattr = {
 928                         .opcode = IONIC_CMD_LIF_SETATTR,
 929                         .attr = IONIC_LIF_ATTR_RSS,
 930                         .rss.types = types,
 931                         .rss.addr = lif->rss_ind_tbl_pa,
 932                 },
 933         };
 934         unsigned int i;
 935
 936         IONIC_PRINT_CALL();
 937
 938         lif->rss_types = types;
 939
 940         if (key)
 941                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 942
 943         if (indir)
 944                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 945                         lif->rss_ind_tbl[i] = indir[i];
 946
 947         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 948                IONIC_RSS_HASH_KEY_SIZE);
 949
 950         return ionic_adminq_post_wait(lif, &ctx);
 951 }
 952
 953 static int
 954 ionic_lif_rss_setup(struct ionic_lif *lif)
 955 {
 956         size_t tbl_size = sizeof(*lif->rss_ind_tbl) *
 957                 lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 958         static const uint8_t toeplitz_symmetric_key[] = {
 959                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 960                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 961                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 962                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 963                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 964         };
 965         uint32_t socket_id = rte_socket_id();
 966         uint32_t i;
 967         int err;
 968
 969         IONIC_PRINT_CALL();
 970
 971         lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 972                 "rss_ind_tbl",
 973                 0 /* queue_idx*/, tbl_size, IONIC_ALIGN, socket_id);
 974
 975         if (!lif->rss_ind_tbl_z) {
 976                 IONIC_PRINT(ERR, "OOM");
 977                 return -ENOMEM;
 978         }
 979
 980         lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 981         lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
 982
 983         /* Fill indirection table with 'default' values */
 984         for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 985                 lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
 986
 987         err = ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
 988                 toeplitz_symmetric_key, NULL);
 989         if (err)
 990                 return err;
 991
 992         return 0;
 993 }
 994
 995 static void
 996 ionic_lif_rss_teardown(struct ionic_lif *lif)
 997 {
 998         if (!lif->rss_ind_tbl)
 999                 return;
1000
1001         if (lif->rss_ind_tbl_z) {
1002                 /* Disable RSS on the NIC */
1003                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1004
1005                 lif->rss_ind_tbl = NULL;
1006                 lif->rss_ind_tbl_pa = 0;
1007                 rte_memzone_free(lif->rss_ind_tbl_z);
1008                 lif->rss_ind_tbl_z = NULL;
1009         }
1010 }
1011
1012 static void
1013 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1014 {
1015         struct ionic_dev *idev = &lif->adapter->idev;
1016
1017         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1018                 return;
1019
1020         if (qcq->flags & IONIC_QCQ_F_INTR)
1021                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1022                         IONIC_INTR_MASK_SET);
1023
1024         qcq->flags &= ~IONIC_QCQ_F_INITED;
1025 }
1026
1027 void
1028 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1029 {
1030         ionic_lif_qcq_deinit(qcq->lif, qcq);
1031 }
1032
1033 void
1034 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1035 {
1036         ionic_lif_qcq_deinit(qcq->lif, qcq);
1037 }
1038
1039 bool
1040 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1041                 void *cb_arg __rte_unused)
1042 {
1043         struct ionic_admin_comp *cq_desc_base = cq->base;
1044         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1045
1046         if (!color_match(cq_desc->color, cq->done_color))
1047                 return false;
1048
1049         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1050
1051         return true;
1052 }
1053
1054 /* This acts like ionic_napi */
1055 int
1056 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1057                 void *cb_arg)
1058 {
1059         struct ionic_cq *cq = &qcq->cq;
1060         uint32_t work_done;
1061
1062         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1063
1064         return work_done;
1065 }
1066
1067 static void
1068 ionic_link_status_check(struct ionic_lif *lif)
1069 {
1070         struct ionic_adapter *adapter = lif->adapter;
1071         bool link_up;
1072
1073         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1074
1075         if (!lif->info)
1076                 return;
1077
1078         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1079
1080         if ((link_up  && adapter->link_up) ||
1081             (!link_up && !adapter->link_up))
1082                 return;
1083
1084         if (link_up) {
1085                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1086                         lif->info->status.link_speed);
1087                 adapter->link_speed = lif->info->status.link_speed;
1088         } else {
1089                 IONIC_PRINT(DEBUG, "Link down");
1090         }
1091
1092         adapter->link_up = link_up;
1093 }
1094
1095 static bool
1096 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1097 {
1098         union ionic_notifyq_comp *cq_desc_base = cq->base;
1099         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1100         struct ionic_lif *lif = cb_arg;
1101
1102         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1103                 cq_desc->event.eid, cq_desc->event.ecode);
1104
1105         /* Have we run out of new completions to process? */
1106         if (!(cq_desc->event.eid > lif->last_eid))
1107                 return false;
1108
1109         lif->last_eid = cq_desc->event.eid;
1110
1111         switch (cq_desc->event.ecode) {
1112         case IONIC_EVENT_LINK_CHANGE:
1113                 IONIC_PRINT(DEBUG,
1114                         "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
1115                         cq_desc->event.eid,
1116                         cq_desc->link_change.link_status,
1117                         cq_desc->link_change.link_speed);
1118
1119                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1120
1121                 break;
1122         default:
1123                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1124                         cq_desc->event.ecode, cq_desc->event.eid);
1125                 break;
1126         }
1127
1128         return true;
1129 }
1130
1131 int
1132 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1133 {
1134         struct ionic_dev *idev = &lif->adapter->idev;
1135         struct ionic_qcq *qcq = lif->notifyqcq;
1136         uint32_t work_done;
1137
1138         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1139                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1140                 return -1;
1141         }
1142
1143         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1144                 IONIC_INTR_MASK_SET);
1145
1146         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1147
1148         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1149                 ionic_link_status_check(lif);
1150
1151         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1152                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1153
1154         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1155                 IONIC_INTR_MASK_CLEAR);
1156
1157         return 0;
1158 }
1159
1160 static int
1161 ionic_lif_adminq_init(struct ionic_lif *lif)
1162 {
1163         struct ionic_dev *idev = &lif->adapter->idev;
1164         struct ionic_qcq *qcq = lif->adminqcq;
1165         struct ionic_queue *q = &qcq->q;
1166         struct ionic_q_init_comp comp;
1167         int err;
1168
1169         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
1170         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1171         if (err)
1172                 return err;
1173
1174         ionic_dev_cmd_comp(idev, &comp);
1175
1176         q->hw_type = comp.hw_type;
1177         q->hw_index = comp.hw_index;
1178         q->db = ionic_db_map(lif, q);
1179
1180         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1181         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1182         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1183
1184         if (qcq->flags & IONIC_QCQ_F_INTR)
1185                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1186                         IONIC_INTR_MASK_CLEAR);
1187
1188         qcq->flags |= IONIC_QCQ_F_INITED;
1189
1190         return 0;
1191 }
1192
1193 static int
1194 ionic_lif_notifyq_init(struct ionic_lif *lif)
1195 {
1196         struct ionic_dev *idev = &lif->adapter->idev;
1197         struct ionic_qcq *qcq = lif->notifyqcq;
1198         struct ionic_queue *q = &qcq->q;
1199         int err;
1200
1201         struct ionic_admin_ctx ctx = {
1202                 .pending_work = true,
1203                 .cmd.q_init = {
1204                         .opcode = IONIC_CMD_Q_INIT,
1205                         .lif_index = lif->index,
1206                         .type = q->type,
1207                         .index = q->index,
1208                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1209                         .intr_index = qcq->intr.index,
1210                         .ring_size = rte_log2_u32(q->num_descs),
1211                         .ring_base = q->base_pa,
1212                 }
1213         };
1214
1215         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1216                 ctx.cmd.q_init.index);
1217         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1218                 ctx.cmd.q_init.ring_base);
1219         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1220                 ctx.cmd.q_init.ring_size);
1221
1222         err = ionic_adminq_post_wait(lif, &ctx);
1223         if (err)
1224                 return err;
1225
1226         q->hw_type = ctx.comp.q_init.hw_type;
1227         q->hw_index = ctx.comp.q_init.hw_index;
1228         q->db = NULL;
1229
1230         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1231         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1232         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1233
1234         if (qcq->flags & IONIC_QCQ_F_INTR)
1235                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1236                         IONIC_INTR_MASK_CLEAR);
1237
1238         qcq->flags |= IONIC_QCQ_F_INITED;
1239
1240         return 0;
1241 }
1242
1243 int
1244 ionic_lif_set_features(struct ionic_lif *lif)
1245 {
1246         struct ionic_admin_ctx ctx = {
1247                 .pending_work = true,
1248                 .cmd.lif_setattr = {
1249                         .opcode = IONIC_CMD_LIF_SETATTR,
1250                         .index = lif->index,
1251                         .attr = IONIC_LIF_ATTR_FEATURES,
1252                         .features = lif->features,
1253                 },
1254         };
1255         int err;
1256
1257         err = ionic_adminq_post_wait(lif, &ctx);
1258         if (err)
1259                 return err;
1260
1261         lif->hw_features = (ctx.cmd.lif_setattr.features &
1262                 ctx.comp.lif_setattr.features);
1263
1264         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1265                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1266         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1267                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1268         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1269                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1270         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1271                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1272         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1273                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1274         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1275                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1276         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1277                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1278         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1279                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1280         if (lif->hw_features & IONIC_ETH_HW_TSO)
1281                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1282         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1283                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1284         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1285                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1286         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1287                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1288         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1289                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1290         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1291                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1292         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1293                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1294         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1295                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1296         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1297                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1298
1299         return 0;
1300 }
1301
1302 int
1303 ionic_lif_txq_init(struct ionic_qcq *qcq)
1304 {
1305         struct ionic_queue *q = &qcq->q;
1306         struct ionic_lif *lif = qcq->lif;
1307         struct ionic_cq *cq = &qcq->cq;
1308         struct ionic_admin_ctx ctx = {
1309                 .pending_work = true,
1310                 .cmd.q_init = {
1311                         .opcode = IONIC_CMD_Q_INIT,
1312                         .lif_index = lif->index,
1313                         .type = q->type,
1314                         .index = q->index,
1315                         .flags = IONIC_QINIT_F_SG,
1316                         .intr_index = cq->bound_intr->index,
1317                         .ring_size = rte_log2_u32(q->num_descs),
1318                         .ring_base = q->base_pa,
1319                         .cq_ring_base = cq->base_pa,
1320                         .sg_ring_base = q->sg_base_pa,
1321                 },
1322         };
1323         int err;
1324
1325         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1326         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1327                 ctx.cmd.q_init.ring_base);
1328         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1329                 ctx.cmd.q_init.ring_size);
1330
1331         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1332         if (err)
1333                 return err;
1334
1335         q->hw_type = ctx.comp.q_init.hw_type;
1336         q->hw_index = ctx.comp.q_init.hw_index;
1337         q->db = ionic_db_map(lif, q);
1338
1339         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1340         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1341         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1342
1343         qcq->flags |= IONIC_QCQ_F_INITED;
1344
1345         return 0;
1346 }
1347
1348 int
1349 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1350 {
1351         struct ionic_queue *q = &qcq->q;
1352         struct ionic_lif *lif = qcq->lif;
1353         struct ionic_cq *cq = &qcq->cq;
1354         struct ionic_admin_ctx ctx = {
1355                 .pending_work = true,
1356                 .cmd.q_init = {
1357                         .opcode = IONIC_CMD_Q_INIT,
1358                         .lif_index = lif->index,
1359                         .type = q->type,
1360                         .index = q->index,
1361                         .flags = IONIC_QINIT_F_SG,
1362                         .intr_index = cq->bound_intr->index,
1363                         .ring_size = rte_log2_u32(q->num_descs),
1364                         .ring_base = q->base_pa,
1365                         .cq_ring_base = cq->base_pa,
1366                         .sg_ring_base = q->sg_base_pa,
1367                 },
1368         };
1369         int err;
1370
1371         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1372         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1373                 ctx.cmd.q_init.ring_base);
1374         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1375                 ctx.cmd.q_init.ring_size);
1376
1377         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1378         if (err)
1379                 return err;
1380
1381         q->hw_type = ctx.comp.q_init.hw_type;
1382         q->hw_index = ctx.comp.q_init.hw_index;
1383         q->db = ionic_db_map(lif, q);
1384
1385         qcq->flags |= IONIC_QCQ_F_INITED;
1386
1387         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1388         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1389         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1390
1391         return 0;
1392 }
1393
1394 static int
1395 ionic_station_set(struct ionic_lif *lif)
1396 {
1397         struct ionic_admin_ctx ctx = {
1398                 .pending_work = true,
1399                 .cmd.lif_getattr = {
1400                         .opcode = IONIC_CMD_LIF_GETATTR,
1401                         .index = lif->index,
1402                         .attr = IONIC_LIF_ATTR_MAC,
1403                 },
1404         };
1405         int err;
1406
1407         IONIC_PRINT_CALL();
1408
1409         err = ionic_adminq_post_wait(lif, &ctx);
1410         if (err)
1411                 return err;
1412
1413         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1414                         lif->mac_addr)) {
1415                 IONIC_PRINT(INFO, "deleting station MAC addr");
1416
1417                 ionic_lif_addr_del(lif, lif->mac_addr);
1418         }
1419
1420         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1421
1422         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1423                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1424                 return 0;
1425         }
1426
1427         IONIC_PRINT(DEBUG, "adding station MAC addr");
1428
1429         ionic_lif_addr_add(lif, lif->mac_addr);
1430
1431         return 0;
1432 }
1433
1434 static void
1435 ionic_lif_set_name(struct ionic_lif *lif)
1436 {
1437         struct ionic_admin_ctx ctx = {
1438                 .pending_work = true,
1439                 .cmd.lif_setattr = {
1440                         .opcode = IONIC_CMD_LIF_SETATTR,
1441                         .index = lif->index,
1442                         .attr = IONIC_LIF_ATTR_NAME,
1443                 },
1444         };
1445
1446         snprintf(ctx.cmd.lif_setattr.name, sizeof(ctx.cmd.lif_setattr.name),
1447                 "%d", lif->port_id);
1448
1449         ionic_adminq_post_wait(lif, &ctx);
1450 }
1451
1452 int
1453 ionic_lif_init(struct ionic_lif *lif)
1454 {
1455         struct ionic_dev *idev = &lif->adapter->idev;
1456         struct ionic_q_init_comp comp;
1457         int err;
1458
1459         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1460
1461         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
1462         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1463         ionic_dev_cmd_comp(idev, &comp);
1464         if (err)
1465                 return err;
1466
1467         lif->hw_index = comp.hw_index;
1468
1469         err = ionic_lif_adminq_init(lif);
1470         if (err)
1471                 return err;
1472
1473         err = ionic_lif_notifyq_init(lif);
1474         if (err)
1475                 goto err_out_adminq_deinit;
1476
1477         lif->features =
1478                   IONIC_ETH_HW_VLAN_TX_TAG
1479                 | IONIC_ETH_HW_VLAN_RX_STRIP
1480                 | IONIC_ETH_HW_VLAN_RX_FILTER
1481                 | IONIC_ETH_HW_RX_HASH
1482                 | IONIC_ETH_HW_TX_SG
1483                 | IONIC_ETH_HW_RX_SG
1484                 | IONIC_ETH_HW_TX_CSUM
1485                 | IONIC_ETH_HW_RX_CSUM
1486                 | IONIC_ETH_HW_TSO
1487                 | IONIC_ETH_HW_TSO_IPV6
1488                 | IONIC_ETH_HW_TSO_ECN;
1489
1490         err = ionic_lif_set_features(lif);
1491         if (err)
1492                 goto err_out_notifyq_deinit;
1493
1494         err = ionic_rx_filters_init(lif);
1495         if (err)
1496                 goto err_out_notifyq_deinit;
1497
1498         err = ionic_station_set(lif);
1499         if (err)
1500                 goto err_out_rx_filter_deinit;
1501
1502         ionic_lif_set_name(lif);
1503
1504         lif->state |= IONIC_LIF_F_INITED;
1505
1506         return 0;
1507
1508 err_out_rx_filter_deinit:
1509         ionic_rx_filters_deinit(lif);
1510
1511 err_out_notifyq_deinit:
1512         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1513
1514 err_out_adminq_deinit:
1515         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1516
1517         return err;
1518 }
1519
1520 void
1521 ionic_lif_deinit(struct ionic_lif *lif)
1522 {
1523         if (!(lif->state & IONIC_LIF_F_INITED))
1524                 return;
1525
1526         ionic_rx_filters_deinit(lif);
1527         ionic_lif_rss_teardown(lif);
1528         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1529         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1530
1531         lif->state &= ~IONIC_LIF_F_INITED;
1532 }
1533
1534 int
1535 ionic_lif_configure(struct ionic_lif *lif)
1536 {
1537         struct ionic_identity *ident = &lif->adapter->ident;
1538         uint32_t ntxqs_per_lif =
1539                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1540         uint32_t nrxqs_per_lif =
1541                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1542         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1543         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1544
1545         lif->port_id = lif->eth_dev->data->port_id;
1546
1547         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1548                 lif->port_id);
1549
1550         if (nrxqs > 0)
1551                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1552
1553         if (ntxqs > 0)
1554                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1555
1556         lif->nrxqcqs = nrxqs_per_lif;
1557         lif->ntxqcqs = ntxqs_per_lif;
1558
1559         return 0;
1560 }
1561
1562 int
1563 ionic_lif_start(struct ionic_lif *lif)
1564 {
1565         uint32_t rx_mode = 0;
1566         uint32_t i;
1567         int err;
1568
1569         IONIC_PRINT(DEBUG, "Setting RSS configuration on port %u",
1570                 lif->port_id);
1571
1572         err = ionic_lif_rss_setup(lif);
1573         if (err)
1574                 return err;
1575
1576         IONIC_PRINT(DEBUG, "Setting RX mode on port %u",
1577                 lif->port_id);
1578
1579         rx_mode |= IONIC_RX_MODE_F_UNICAST;
1580         rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1581         rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1582
1583         lif->rx_mode = 0; /* set by ionic_set_rx_mode */
1584
1585         ionic_set_rx_mode(lif, rx_mode);
1586
1587         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1588                 "on port %u",
1589                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1590
1591         for (i = 0; i < lif->nrxqcqs; i++) {
1592                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1593                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1594                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1595
1596                         if (err)
1597                                 return err;
1598                 }
1599         }
1600
1601         for (i = 0; i < lif->ntxqcqs; i++) {
1602                 struct ionic_qcq *txq = lif->txqcqs[i];
1603                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1604                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1605
1606                         if (err)
1607                                 return err;
1608                 }
1609         }
1610
1611         ionic_link_status_check(lif);
1612
1613         /* Carrier ON here */
1614
1615         return 0;
1616 }
1617
1618 int
1619 ionic_lif_identify(struct ionic_adapter *adapter)
1620 {
1621         struct ionic_dev *idev = &adapter->idev;
1622         struct ionic_identity *ident = &adapter->ident;
1623         int err;
1624         unsigned int i;
1625         unsigned int lif_words = sizeof(ident->lif.words) /
1626                 sizeof(ident->lif.words[0]);
1627         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1628                 sizeof(idev->dev_cmd->data[0]);
1629         unsigned int nwords;
1630
1631         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1632                 IONIC_IDENTITY_VERSION_1);
1633         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1634         if (err)
1635                 return (err);
1636
1637         nwords = RTE_MIN(lif_words, cmd_words);
1638         for (i = 0; i < nwords; i++)
1639                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1640
1641         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1642                 ident->lif.capabilities);
1643
1644         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1645                 ident->lif.eth.max_ucast_filters);
1646         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1647                 ident->lif.eth.max_mcast_filters);
1648
1649         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1650                 ident->lif.eth.config.features);
1651         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1652                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1653         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1654                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1655         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1656                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1657         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1658                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1659
1660         return 0;
1661 }
1662
1663 int
1664 ionic_lifs_size(struct ionic_adapter *adapter)
1665 {
1666         struct ionic_identity *ident = &adapter->ident;
1667         uint32_t nlifs = ident->dev.nlifs;
1668         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1669
1670         adapter->max_ntxqs_per_lif =
1671                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1672         adapter->max_nrxqs_per_lif =
1673                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1674
1675         nintrs = nlifs * 1 /* notifyq */;
1676
1677         if (nintrs > dev_nintrs) {
1678                 IONIC_PRINT(ERR, "At most %d intr queues supported, minimum required is %u",
1679                         dev_nintrs, nintrs);
1680                 return -ENOSPC;
1681         }
1682
1683         adapter->nintrs = nintrs;
1684
1685         return 0;
1686 }