drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .lif_index = lif->index,
  29                         .type = q->type,
  30                         .index = q->index,
  31                         .oper = IONIC_Q_ENABLE,
  32                 },
  33         };
  34
  35         if (qcq->flags & IONIC_QCQ_F_INTR) {
  36                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  37                         IONIC_INTR_MASK_CLEAR);
  38         }
  39
  40         return ionic_adminq_post_wait(lif, &ctx);
  41 }
  42
  43 int
  44 ionic_qcq_disable(struct ionic_qcq *qcq)
  45 {
  46         struct ionic_queue *q = &qcq->q;
  47         struct ionic_lif *lif = q->lif;
  48         struct ionic_dev *idev = &lif->adapter->idev;
  49         struct ionic_admin_ctx ctx = {
  50                 .pending_work = true,
  51                 .cmd.q_control = {
  52                         .opcode = IONIC_CMD_Q_CONTROL,
  53                         .lif_index = lif->index,
  54                         .type = q->type,
  55                         .index = q->index,
  56                         .oper = IONIC_Q_DISABLE,
  57                 },
  58         };
  59
  60         if (qcq->flags & IONIC_QCQ_F_INTR) {
  61                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  62                         IONIC_INTR_MASK_SET);
  63         }
  64
  65         return ionic_adminq_post_wait(lif, &ctx);
  66 }
  67
  68 int
  69 ionic_lif_stop(struct ionic_lif *lif __rte_unused)
  70 {
  71         /* Carrier OFF here */
  72
  73         return 0;
  74 }
  75
  76 void
  77 ionic_lif_reset(struct ionic_lif *lif)
  78 {
  79         struct ionic_dev *idev = &lif->adapter->idev;
  80         int err;
  81
  82         IONIC_PRINT_CALL();
  83
  84         ionic_dev_cmd_lif_reset(idev, lif->index);
  85         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  86         if (err)
  87                 IONIC_PRINT(WARNING, "Failed to reset lif");
  88 }
  89
  90 static void
  91 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  92 {
  93         struct ionic_lif_stats *ls = &lif->info->stats;
  94         uint32_t i;
  95         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  96                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  97         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  98                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  99
 100         memset(stats, 0, sizeof(*stats));
 101
 102         if (ls == NULL) {
 103                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 104                         lif->port_id);
 105                 return;
 106         }
 107
 108         /* RX */
 109
 110         stats->ipackets = ls->rx_ucast_packets +
 111                 ls->rx_mcast_packets +
 112                 ls->rx_bcast_packets;
 113
 114         stats->ibytes = ls->rx_ucast_bytes +
 115                 ls->rx_mcast_bytes +
 116                 ls->rx_bcast_bytes;
 117
 118         for (i = 0; i < lif->nrxqcqs; i++) {
 119                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 120                 stats->imissed +=
 121                         rx_stats->no_cb_arg +
 122                         rx_stats->bad_cq_status +
 123                         rx_stats->no_room +
 124                         rx_stats->bad_len;
 125         }
 126
 127         stats->imissed +=
 128                 ls->rx_ucast_drop_packets +
 129                 ls->rx_mcast_drop_packets +
 130                 ls->rx_bcast_drop_packets;
 131
 132         stats->imissed +=
 133                 ls->rx_queue_empty +
 134                 ls->rx_dma_error +
 135                 ls->rx_queue_disabled +
 136                 ls->rx_desc_fetch_error +
 137                 ls->rx_desc_data_error;
 138
 139         for (i = 0; i < num_rx_q_counters; i++) {
 140                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 141                 stats->q_ipackets[i] = rx_stats->packets;
 142                 stats->q_ibytes[i] = rx_stats->bytes;
 143                 stats->q_errors[i] =
 144                         rx_stats->no_cb_arg +
 145                         rx_stats->bad_cq_status +
 146                         rx_stats->no_room +
 147                         rx_stats->bad_len;
 148         }
 149
 150         /* TX */
 151
 152         stats->opackets = ls->tx_ucast_packets +
 153                 ls->tx_mcast_packets +
 154                 ls->tx_bcast_packets;
 155
 156         stats->obytes = ls->tx_ucast_bytes +
 157                 ls->tx_mcast_bytes +
 158                 ls->tx_bcast_bytes;
 159
 160         for (i = 0; i < lif->ntxqcqs; i++) {
 161                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 162                 stats->oerrors += tx_stats->drop;
 163         }
 164
 165         stats->oerrors +=
 166                 ls->tx_ucast_drop_packets +
 167                 ls->tx_mcast_drop_packets +
 168                 ls->tx_bcast_drop_packets;
 169
 170         stats->oerrors +=
 171                 ls->tx_dma_error +
 172                 ls->tx_queue_disabled +
 173                 ls->tx_desc_fetch_error +
 174                 ls->tx_desc_data_error;
 175
 176         for (i = 0; i < num_tx_q_counters; i++) {
 177                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 178                 stats->q_opackets[i] = tx_stats->packets;
 179                 stats->q_obytes[i] = tx_stats->bytes;
 180         }
 181 }
 182
 183 void
 184 ionic_lif_get_stats(const struct ionic_lif *lif,
 185                 struct rte_eth_stats *stats)
 186 {
 187         ionic_lif_get_abs_stats(lif, stats);
 188
 189         stats->ipackets  -= lif->stats_base.ipackets;
 190         stats->opackets  -= lif->stats_base.opackets;
 191         stats->ibytes    -= lif->stats_base.ibytes;
 192         stats->obytes    -= lif->stats_base.obytes;
 193         stats->imissed   -= lif->stats_base.imissed;
 194         stats->ierrors   -= lif->stats_base.ierrors;
 195         stats->oerrors   -= lif->stats_base.oerrors;
 196         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 197 }
 198
 199 void
 200 ionic_lif_reset_stats(struct ionic_lif *lif)
 201 {
 202         uint32_t i;
 203
 204         for (i = 0; i < lif->nrxqcqs; i++) {
 205                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 206                         sizeof(struct ionic_rx_stats));
 207                 memset(&lif->txqcqs[i]->stats.tx, 0,
 208                         sizeof(struct ionic_tx_stats));
 209         }
 210
 211         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 212 }
 213
 214 void
 215 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 216 {
 217         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 218         uint64_t *stats64 = (uint64_t *)stats;
 219         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 220         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 221
 222         for (i = 0; i < count; i++)
 223                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 224 }
 225
 226 void
 227 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 228 {
 229         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 230         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 231         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 232
 233         for (i = 0; i < count; i++)
 234                 lif_stats64_base[i] = lif_stats64[i];
 235 }
 236
 237 static int
 238 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 239 {
 240         struct ionic_admin_ctx ctx = {
 241                 .pending_work = true,
 242                 .cmd.rx_filter_add = {
 243                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 244                         .match = IONIC_RX_FILTER_MATCH_MAC,
 245                 },
 246         };
 247         int err;
 248
 249         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 250
 251         err = ionic_adminq_post_wait(lif, &ctx);
 252         if (err)
 253                 return err;
 254
 255         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 256                 ctx.comp.rx_filter_add.filter_id);
 257
 258         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 259 }
 260
 261 static int
 262 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 263 {
 264         struct ionic_admin_ctx ctx = {
 265                 .pending_work = true,
 266                 .cmd.rx_filter_del = {
 267                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 268                 },
 269         };
 270         struct ionic_rx_filter *f;
 271         int err;
 272
 273         IONIC_PRINT_CALL();
 274
 275         rte_spinlock_lock(&lif->rx_filters.lock);
 276
 277         f = ionic_rx_filter_by_addr(lif, addr);
 278         if (!f) {
 279                 rte_spinlock_unlock(&lif->rx_filters.lock);
 280                 return -ENOENT;
 281         }
 282
 283         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 284         ionic_rx_filter_free(f);
 285
 286         rte_spinlock_unlock(&lif->rx_filters.lock);
 287
 288         err = ionic_adminq_post_wait(lif, &ctx);
 289         if (err)
 290                 return err;
 291
 292         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 293                 ctx.cmd.rx_filter_del.filter_id);
 294
 295         return 0;
 296 }
 297
 298 int
 299 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 300                 struct rte_ether_addr *mac_addr,
 301                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 302 {
 303         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 304
 305         IONIC_PRINT_CALL();
 306
 307         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 308 }
 309
 310 void
 311 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index)
 312 {
 313         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 314         struct ionic_adapter *adapter = lif->adapter;
 315         struct rte_ether_addr *mac_addr;
 316
 317         IONIC_PRINT_CALL();
 318
 319         if (index >= adapter->max_mac_addrs) {
 320                 IONIC_PRINT(WARNING,
 321                         "Index %u is above MAC filter limit %u",
 322                         index, adapter->max_mac_addrs);
 323                 return;
 324         }
 325
 326         mac_addr = &eth_dev->data->mac_addrs[index];
 327
 328         if (!rte_is_valid_assigned_ether_addr(mac_addr))
 329                 return;
 330
 331         ionic_lif_addr_del(lif, (const uint8_t *)mac_addr);
 332 }
 333
 334 int
 335 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 336 {
 337         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 338
 339         IONIC_PRINT_CALL();
 340
 341         if (mac_addr == NULL) {
 342                 IONIC_PRINT(NOTICE, "New mac is null");
 343                 return -1;
 344         }
 345
 346         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 347                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 348                         lif->mac_addr);
 349                 ionic_lif_addr_del(lif, lif->mac_addr);
 350                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 351         }
 352
 353         IONIC_PRINT(INFO, "Updating mac addr");
 354
 355         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 356
 357         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 358 }
 359
 360 static int
 361 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 362 {
 363         struct ionic_admin_ctx ctx = {
 364                 .pending_work = true,
 365                 .cmd.rx_filter_add = {
 366                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 367                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 368                         .vlan.vlan = vid,
 369                 },
 370         };
 371         int err;
 372
 373         err = ionic_adminq_post_wait(lif, &ctx);
 374         if (err)
 375                 return err;
 376
 377         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 378                 ctx.comp.rx_filter_add.filter_id);
 379
 380         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 381 }
 382
 383 static int
 384 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 385 {
 386         struct ionic_admin_ctx ctx = {
 387                 .pending_work = true,
 388                 .cmd.rx_filter_del = {
 389                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 390                 },
 391         };
 392         struct ionic_rx_filter *f;
 393         int err;
 394
 395         IONIC_PRINT_CALL();
 396
 397         rte_spinlock_lock(&lif->rx_filters.lock);
 398
 399         f = ionic_rx_filter_by_vlan(lif, vid);
 400         if (!f) {
 401                 rte_spinlock_unlock(&lif->rx_filters.lock);
 402                 return -ENOENT;
 403         }
 404
 405         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 406         ionic_rx_filter_free(f);
 407         rte_spinlock_unlock(&lif->rx_filters.lock);
 408
 409         err = ionic_adminq_post_wait(lif, &ctx);
 410         if (err)
 411                 return err;
 412
 413         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 414                 ctx.cmd.rx_filter_del.filter_id);
 415
 416         return 0;
 417 }
 418
 419 int
 420 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 421                 int on)
 422 {
 423         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 424         int err;
 425
 426         if (on)
 427                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 428         else
 429                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 430
 431         return err;
 432 }
 433
 434 static void
 435 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 436 {
 437         struct ionic_admin_ctx ctx = {
 438                 .pending_work = true,
 439                 .cmd.rx_mode_set = {
 440                         .opcode = IONIC_CMD_RX_MODE_SET,
 441                         .lif_index = lif->index,
 442                         .rx_mode = rx_mode,
 443                 },
 444         };
 445         int err;
 446
 447         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 448                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 449         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 450                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 451         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 452                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 453         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 454                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 455         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 456                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 457
 458         err = ionic_adminq_post_wait(lif, &ctx);
 459         if (err)
 460                 IONIC_PRINT(ERR, "Failure setting RX mode");
 461 }
 462
 463 static void
 464 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 465 {
 466         if (lif->rx_mode != rx_mode) {
 467                 lif->rx_mode = rx_mode;
 468                 ionic_lif_rx_mode(lif, rx_mode);
 469         }
 470 }
 471
 472 int
 473 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 474 {
 475         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 476         uint32_t rx_mode = lif->rx_mode;
 477
 478         IONIC_PRINT_CALL();
 479
 480         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 481
 482         ionic_set_rx_mode(lif, rx_mode);
 483
 484         return 0;
 485 }
 486
 487 int
 488 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 489 {
 490         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 491         uint32_t rx_mode = lif->rx_mode;
 492
 493         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 494
 495         ionic_set_rx_mode(lif, rx_mode);
 496
 497         return 0;
 498 }
 499
 500 int
 501 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 502 {
 503         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 504         uint32_t rx_mode = lif->rx_mode;
 505
 506         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 507
 508         ionic_set_rx_mode(lif, rx_mode);
 509
 510         return 0;
 511 }
 512
 513 int
 514 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 515 {
 516         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 517         uint32_t rx_mode = lif->rx_mode;
 518
 519         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 520
 521         ionic_set_rx_mode(lif, rx_mode);
 522
 523         return 0;
 524 }
 525
 526 int
 527 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 528 {
 529         struct ionic_admin_ctx ctx = {
 530                 .pending_work = true,
 531                 .cmd.lif_setattr = {
 532                         .opcode = IONIC_CMD_LIF_SETATTR,
 533                         .index = lif->index,
 534                         .attr = IONIC_LIF_ATTR_MTU,
 535                         .mtu = new_mtu,
 536                 },
 537         };
 538         int err;
 539
 540         err = ionic_adminq_post_wait(lif, &ctx);
 541         if (err)
 542                 return err;
 543
 544         return 0;
 545 }
 546
 547 int
 548 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 549 {
 550         struct ionic_adapter *adapter = lif->adapter;
 551         struct ionic_dev *idev = &adapter->idev;
 552         unsigned long index;
 553
 554         /*
 555          * Note: interrupt handler is called for index = 0 only
 556          * (we use interrupts for the notifyq only anyway,
 557          * which hash index = 0)
 558          */
 559
 560         for (index = 0; index < adapter->nintrs; index++)
 561                 if (!adapter->intrs[index])
 562                         break;
 563
 564         if (index == adapter->nintrs)
 565                 return -ENOSPC;
 566
 567         adapter->intrs[index] = true;
 568
 569         ionic_intr_init(idev, intr, index);
 570
 571         return 0;
 572 }
 573
 574 void
 575 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 576 {
 577         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 578                 lif->adapter->intrs[intr->index] = false;
 579 }
 580
 581 static int
 582 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 583                 uint32_t index,
 584                 const char *base, uint32_t flags,
 585                 uint32_t num_descs,
 586                 uint32_t desc_size,
 587                 uint32_t cq_desc_size,
 588                 uint32_t sg_desc_size,
 589                 struct ionic_qcq **qcq)
 590 {
 591         struct ionic_dev *idev = &lif->adapter->idev;
 592         struct ionic_qcq *new;
 593         uint32_t q_size, cq_size, sg_size, total_size;
 594         void *q_base, *cq_base, *sg_base;
 595         rte_iova_t q_base_pa = 0;
 596         rte_iova_t cq_base_pa = 0;
 597         rte_iova_t sg_base_pa = 0;
 598         uint32_t socket_id = rte_socket_id();
 599         int err;
 600
 601         *qcq = NULL;
 602
 603         q_size  = num_descs * desc_size;
 604         cq_size = num_descs * cq_desc_size;
 605         sg_size = num_descs * sg_desc_size;
 606
 607         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 608                 RTE_ALIGN(cq_size, PAGE_SIZE);
 609         /*
 610          * Note: aligning q_size/cq_size is not enough due to cq_base address
 611          * aligning as q_base could be not aligned to the page.
 612          * Adding PAGE_SIZE.
 613          */
 614         total_size += PAGE_SIZE;
 615
 616         if (flags & IONIC_QCQ_F_SG) {
 617                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 618                 total_size += PAGE_SIZE;
 619         }
 620
 621         new = rte_zmalloc("ionic", sizeof(*new), 0);
 622         if (!new) {
 623                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 624                 return -ENOMEM;
 625         }
 626
 627         new->lif = lif;
 628         new->flags = flags;
 629
 630         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 631         if (!new->q.info) {
 632                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 633                 return -ENOMEM;
 634         }
 635
 636         new->q.type = type;
 637
 638         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 639                 desc_size, sg_desc_size);
 640         if (err) {
 641                 IONIC_PRINT(ERR, "Queue initialization failed");
 642                 return err;
 643         }
 644
 645         if (flags & IONIC_QCQ_F_INTR) {
 646                 err = ionic_intr_alloc(lif, &new->intr);
 647                 if (err)
 648                         return err;
 649
 650                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 651                         IONIC_INTR_MASK_SET);
 652         } else {
 653                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 654         }
 655
 656         err = ionic_cq_init(lif, &new->cq, &new->intr,
 657                 num_descs, cq_desc_size);
 658         if (err) {
 659                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 660                 goto err_out_free_intr;
 661         }
 662
 663         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 664                 base /* name */, index /* queue_idx */,
 665                 total_size, IONIC_ALIGN, socket_id);
 666
 667         if (!new->base_z) {
 668                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 669                 err = -ENOMEM;
 670                 goto err_out_free_intr;
 671         }
 672
 673         new->base = new->base_z->addr;
 674         new->base_pa = new->base_z->iova;
 675         new->total_size = total_size;
 676
 677         q_base = new->base;
 678         q_base_pa = new->base_pa;
 679
 680         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 681         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 682
 683         if (flags & IONIC_QCQ_F_SG) {
 684                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 685                         PAGE_SIZE);
 686                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 687                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 688         }
 689
 690         IONIC_PRINT(DEBUG, "Q-Base-PA = %ju CQ-Base-PA = %ju "
 691                 "SG-base-PA = %ju",
 692                 q_base_pa, cq_base_pa, sg_base_pa);
 693
 694         ionic_q_map(&new->q, q_base, q_base_pa);
 695         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 696         ionic_cq_bind(&new->cq, &new->q);
 697
 698         *qcq = new;
 699
 700         return 0;
 701
 702 err_out_free_intr:
 703         if (flags & IONIC_QCQ_F_INTR)
 704                 ionic_intr_free(lif, &new->intr);
 705
 706         return err;
 707 }
 708
 709 void
 710 ionic_qcq_free(struct ionic_qcq *qcq)
 711 {
 712         if (qcq->base_z) {
 713                 qcq->base = NULL;
 714                 qcq->base_pa = 0;
 715                 rte_memzone_free(qcq->base_z);
 716                 qcq->base_z = NULL;
 717         }
 718
 719         if (qcq->q.info) {
 720                 rte_free(qcq->q.info);
 721                 qcq->q.info = NULL;
 722         }
 723
 724         rte_free(qcq);
 725 }
 726
 727 int
 728 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 729                 struct ionic_qcq **qcq)
 730 {
 731         uint32_t flags;
 732         int err = -ENOMEM;
 733
 734         flags = IONIC_QCQ_F_SG;
 735         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 736                 nrxq_descs,
 737                 sizeof(struct ionic_rxq_desc),
 738                 sizeof(struct ionic_rxq_comp),
 739                 sizeof(struct ionic_rxq_sg_desc),
 740                 &lif->rxqcqs[index]);
 741         if (err)
 742                 return err;
 743
 744         *qcq = lif->rxqcqs[index];
 745
 746         return 0;
 747 }
 748
 749 int
 750 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 751                 struct ionic_qcq **qcq)
 752 {
 753         uint32_t flags;
 754         int err = -ENOMEM;
 755
 756         flags = IONIC_QCQ_F_SG;
 757         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 758                 ntxq_descs,
 759                 sizeof(struct ionic_txq_desc),
 760                 sizeof(struct ionic_txq_comp),
 761                 sizeof(struct ionic_txq_sg_desc),
 762                 &lif->txqcqs[index]);
 763         if (err)
 764                 return err;
 765
 766         *qcq = lif->txqcqs[index];
 767
 768         return 0;
 769 }
 770
 771 static int
 772 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 773 {
 774         uint32_t flags;
 775         int err = -ENOMEM;
 776
 777         flags = 0;
 778         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 779                 IONIC_ADMINQ_LENGTH,
 780                 sizeof(struct ionic_admin_cmd),
 781                 sizeof(struct ionic_admin_comp),
 782                 0,
 783                 &lif->adminqcq);
 784         if (err)
 785                 return err;
 786
 787         return 0;
 788 }
 789
 790 static int
 791 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 792 {
 793         uint32_t flags;
 794         int err = -ENOMEM;
 795
 796         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 797
 798         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 799                 flags,
 800                 IONIC_NOTIFYQ_LENGTH,
 801                 sizeof(struct ionic_notifyq_cmd),
 802                 sizeof(union ionic_notifyq_comp),
 803                 0,
 804                 &lif->notifyqcq);
 805         if (err)
 806                 return err;
 807
 808         return 0;
 809 }
 810
 811 static void *
 812 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 813 {
 814         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 815
 816         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 817                 return NULL;
 818
 819         return (void *)&vaddr[page_num << PAGE_SHIFT];
 820 }
 821
 822 int
 823 ionic_lif_alloc(struct ionic_lif *lif)
 824 {
 825         struct ionic_adapter *adapter = lif->adapter;
 826         uint32_t socket_id = rte_socket_id();
 827         int dbpage_num;
 828         int err;
 829
 830         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
 831
 832         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 833
 834         rte_spinlock_init(&lif->adminq_lock);
 835         rte_spinlock_init(&lif->adminq_service_lock);
 836
 837         dbpage_num = ionic_db_page_num(lif, 0);
 838
 839         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, dbpage_num);
 840         if (!lif->kern_dbpage) {
 841                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 842                 return -ENOMEM;
 843         }
 844
 845         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 846                 adapter->max_ntxqs_per_lif, 0);
 847
 848         if (!lif->txqcqs) {
 849                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 850                 return -ENOMEM;
 851         }
 852
 853         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 854                 adapter->max_nrxqs_per_lif, 0);
 855
 856         if (!lif->rxqcqs) {
 857                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 858                 return -ENOMEM;
 859         }
 860
 861         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 862
 863         err = ionic_notify_qcq_alloc(lif);
 864         if (err) {
 865                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 866                 return err;
 867         }
 868
 869         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 870
 871         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 872
 873         err = ionic_admin_qcq_alloc(lif);
 874         if (err) {
 875                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 876                 return err;
 877         }
 878
 879         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 880
 881         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 882
 883         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 884                 "lif_info", 0 /* queue_idx*/,
 885                 lif->info_sz, IONIC_ALIGN, socket_id);
 886         if (!lif->info_z) {
 887                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 888                 return -ENOMEM;
 889         }
 890
 891         lif->info = lif->info_z->addr;
 892         lif->info_pa = lif->info_z->iova;
 893
 894         return 0;
 895 }
 896
 897 void
 898 ionic_lif_free(struct ionic_lif *lif)
 899 {
 900         if (lif->notifyqcq) {
 901                 ionic_qcq_free(lif->notifyqcq);
 902                 lif->notifyqcq = NULL;
 903         }
 904
 905         if (lif->adminqcq) {
 906                 ionic_qcq_free(lif->adminqcq);
 907                 lif->adminqcq = NULL;
 908         }
 909
 910         if (lif->txqcqs) {
 911                 rte_free(lif->txqcqs);
 912                 lif->txqcqs = NULL;
 913         }
 914
 915         if (lif->rxqcqs) {
 916                 rte_free(lif->rxqcqs);
 917                 lif->rxqcqs = NULL;
 918         }
 919
 920         if (lif->info) {
 921                 rte_memzone_free(lif->info_z);
 922                 lif->info = NULL;
 923         }
 924 }
 925
 926 int
 927 ionic_lif_rss_config(struct ionic_lif *lif,
 928                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 929 {
 930         struct ionic_admin_ctx ctx = {
 931                 .pending_work = true,
 932                 .cmd.lif_setattr = {
 933                         .opcode = IONIC_CMD_LIF_SETATTR,
 934                         .attr = IONIC_LIF_ATTR_RSS,
 935                         .rss.types = types,
 936                         .rss.addr = lif->rss_ind_tbl_pa,
 937                 },
 938         };
 939         unsigned int i;
 940
 941         IONIC_PRINT_CALL();
 942
 943         lif->rss_types = types;
 944
 945         if (key)
 946                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 947
 948         if (indir)
 949                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 950                         lif->rss_ind_tbl[i] = indir[i];
 951
 952         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 953                IONIC_RSS_HASH_KEY_SIZE);
 954
 955         return ionic_adminq_post_wait(lif, &ctx);
 956 }
 957
 958 static int
 959 ionic_lif_rss_setup(struct ionic_lif *lif)
 960 {
 961         size_t tbl_size = sizeof(*lif->rss_ind_tbl) *
 962                 lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 963         static const uint8_t toeplitz_symmetric_key[] = {
 964                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 965                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 966                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 967                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 968                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 969         };
 970         uint32_t socket_id = rte_socket_id();
 971         uint32_t i;
 972         int err;
 973
 974         IONIC_PRINT_CALL();
 975
 976         lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 977                 "rss_ind_tbl",
 978                 0 /* queue_idx*/, tbl_size, IONIC_ALIGN, socket_id);
 979
 980         if (!lif->rss_ind_tbl_z) {
 981                 IONIC_PRINT(ERR, "OOM");
 982                 return -ENOMEM;
 983         }
 984
 985         lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 986         lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
 987
 988         /* Fill indirection table with 'default' values */
 989         for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 990                 lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
 991
 992         err = ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
 993                 toeplitz_symmetric_key, NULL);
 994         if (err)
 995                 return err;
 996
 997         return 0;
 998 }
 999
1000 static void
1001 ionic_lif_rss_teardown(struct ionic_lif *lif)
1002 {
1003         if (!lif->rss_ind_tbl)
1004                 return;
1005
1006         if (lif->rss_ind_tbl_z) {
1007                 /* Disable RSS on the NIC */
1008                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1009
1010                 lif->rss_ind_tbl = NULL;
1011                 lif->rss_ind_tbl_pa = 0;
1012                 rte_memzone_free(lif->rss_ind_tbl_z);
1013                 lif->rss_ind_tbl_z = NULL;
1014         }
1015 }
1016
1017 static void
1018 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1019 {
1020         struct ionic_dev *idev = &lif->adapter->idev;
1021
1022         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1023                 return;
1024
1025         if (qcq->flags & IONIC_QCQ_F_INTR)
1026                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1027                         IONIC_INTR_MASK_SET);
1028
1029         qcq->flags &= ~IONIC_QCQ_F_INITED;
1030 }
1031
1032 void
1033 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1034 {
1035         ionic_lif_qcq_deinit(qcq->lif, qcq);
1036 }
1037
1038 void
1039 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1040 {
1041         ionic_lif_qcq_deinit(qcq->lif, qcq);
1042 }
1043
1044 bool
1045 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1046                 void *cb_arg __rte_unused)
1047 {
1048         struct ionic_admin_comp *cq_desc_base = cq->base;
1049         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1050
1051         if (!color_match(cq_desc->color, cq->done_color))
1052                 return false;
1053
1054         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1055
1056         return true;
1057 }
1058
1059 /* This acts like ionic_napi */
1060 int
1061 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1062                 void *cb_arg)
1063 {
1064         struct ionic_cq *cq = &qcq->cq;
1065         uint32_t work_done;
1066
1067         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1068
1069         return work_done;
1070 }
1071
1072 static void
1073 ionic_link_status_check(struct ionic_lif *lif)
1074 {
1075         struct ionic_adapter *adapter = lif->adapter;
1076         bool link_up;
1077
1078         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1079
1080         if (!lif->info)
1081                 return;
1082
1083         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1084
1085         if ((link_up  && adapter->link_up) ||
1086             (!link_up && !adapter->link_up))
1087                 return;
1088
1089         if (link_up) {
1090                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1091                         lif->info->status.link_speed);
1092                 adapter->link_speed = lif->info->status.link_speed;
1093         } else {
1094                 IONIC_PRINT(DEBUG, "Link down");
1095         }
1096
1097         adapter->link_up = link_up;
1098 }
1099
1100 static bool
1101 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1102 {
1103         union ionic_notifyq_comp *cq_desc_base = cq->base;
1104         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1105         struct ionic_lif *lif = cb_arg;
1106
1107         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1108                 cq_desc->event.eid, cq_desc->event.ecode);
1109
1110         /* Have we run out of new completions to process? */
1111         if (!(cq_desc->event.eid > lif->last_eid))
1112                 return false;
1113
1114         lif->last_eid = cq_desc->event.eid;
1115
1116         switch (cq_desc->event.ecode) {
1117         case IONIC_EVENT_LINK_CHANGE:
1118                 IONIC_PRINT(DEBUG,
1119                         "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
1120                         cq_desc->event.eid,
1121                         cq_desc->link_change.link_status,
1122                         cq_desc->link_change.link_speed);
1123
1124                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1125
1126                 break;
1127         default:
1128                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1129                         cq_desc->event.ecode, cq_desc->event.eid);
1130                 break;
1131         }
1132
1133         return true;
1134 }
1135
1136 int
1137 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1138 {
1139         struct ionic_dev *idev = &lif->adapter->idev;
1140         struct ionic_qcq *qcq = lif->notifyqcq;
1141         uint32_t work_done;
1142
1143         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1144                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1145                 return -1;
1146         }
1147
1148         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1149                 IONIC_INTR_MASK_SET);
1150
1151         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1152
1153         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1154                 ionic_link_status_check(lif);
1155
1156         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1157                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1158
1159         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1160                 IONIC_INTR_MASK_CLEAR);
1161
1162         return 0;
1163 }
1164
1165 static int
1166 ionic_lif_adminq_init(struct ionic_lif *lif)
1167 {
1168         struct ionic_dev *idev = &lif->adapter->idev;
1169         struct ionic_qcq *qcq = lif->adminqcq;
1170         struct ionic_queue *q = &qcq->q;
1171         struct ionic_q_init_comp comp;
1172         int err;
1173
1174         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
1175         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1176         if (err)
1177                 return err;
1178
1179         ionic_dev_cmd_comp(idev, &comp);
1180
1181         q->hw_type = comp.hw_type;
1182         q->hw_index = comp.hw_index;
1183         q->db = ionic_db_map(lif, q);
1184
1185         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1186         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1187         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1188
1189         if (qcq->flags & IONIC_QCQ_F_INTR)
1190                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1191                         IONIC_INTR_MASK_CLEAR);
1192
1193         qcq->flags |= IONIC_QCQ_F_INITED;
1194
1195         return 0;
1196 }
1197
1198 static int
1199 ionic_lif_notifyq_init(struct ionic_lif *lif)
1200 {
1201         struct ionic_dev *idev = &lif->adapter->idev;
1202         struct ionic_qcq *qcq = lif->notifyqcq;
1203         struct ionic_queue *q = &qcq->q;
1204         int err;
1205
1206         struct ionic_admin_ctx ctx = {
1207                 .pending_work = true,
1208                 .cmd.q_init = {
1209                         .opcode = IONIC_CMD_Q_INIT,
1210                         .lif_index = lif->index,
1211                         .type = q->type,
1212                         .index = q->index,
1213                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1214                         .intr_index = qcq->intr.index,
1215                         .ring_size = rte_log2_u32(q->num_descs),
1216                         .ring_base = q->base_pa,
1217                 }
1218         };
1219
1220         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1221                 ctx.cmd.q_init.index);
1222         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1223                 ctx.cmd.q_init.ring_base);
1224         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1225                 ctx.cmd.q_init.ring_size);
1226
1227         err = ionic_adminq_post_wait(lif, &ctx);
1228         if (err)
1229                 return err;
1230
1231         q->hw_type = ctx.comp.q_init.hw_type;
1232         q->hw_index = ctx.comp.q_init.hw_index;
1233         q->db = NULL;
1234
1235         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1236         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1237         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1238
1239         if (qcq->flags & IONIC_QCQ_F_INTR)
1240                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1241                         IONIC_INTR_MASK_CLEAR);
1242
1243         qcq->flags |= IONIC_QCQ_F_INITED;
1244
1245         return 0;
1246 }
1247
1248 int
1249 ionic_lif_set_features(struct ionic_lif *lif)
1250 {
1251         struct ionic_admin_ctx ctx = {
1252                 .pending_work = true,
1253                 .cmd.lif_setattr = {
1254                         .opcode = IONIC_CMD_LIF_SETATTR,
1255                         .index = lif->index,
1256                         .attr = IONIC_LIF_ATTR_FEATURES,
1257                         .features = lif->features,
1258                 },
1259         };
1260         int err;
1261
1262         err = ionic_adminq_post_wait(lif, &ctx);
1263         if (err)
1264                 return err;
1265
1266         lif->hw_features = (ctx.cmd.lif_setattr.features &
1267                 ctx.comp.lif_setattr.features);
1268
1269         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1270                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1271         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1272                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1273         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1274                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1275         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1276                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1277         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1278                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1279         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1280                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1281         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1282                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1283         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1284                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1285         if (lif->hw_features & IONIC_ETH_HW_TSO)
1286                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1287         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1288                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1289         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1290                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1291         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1292                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1293         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1294                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1295         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1296                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1297         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1298                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1299         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1300                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1301         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1302                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1303
1304         return 0;
1305 }
1306
1307 int
1308 ionic_lif_txq_init(struct ionic_qcq *qcq)
1309 {
1310         struct ionic_queue *q = &qcq->q;
1311         struct ionic_lif *lif = qcq->lif;
1312         struct ionic_cq *cq = &qcq->cq;
1313         struct ionic_admin_ctx ctx = {
1314                 .pending_work = true,
1315                 .cmd.q_init = {
1316                         .opcode = IONIC_CMD_Q_INIT,
1317                         .lif_index = lif->index,
1318                         .type = q->type,
1319                         .index = q->index,
1320                         .flags = IONIC_QINIT_F_SG,
1321                         .intr_index = cq->bound_intr->index,
1322                         .ring_size = rte_log2_u32(q->num_descs),
1323                         .ring_base = q->base_pa,
1324                         .cq_ring_base = cq->base_pa,
1325                         .sg_ring_base = q->sg_base_pa,
1326                 },
1327         };
1328         int err;
1329
1330         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1331         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1332                 ctx.cmd.q_init.ring_base);
1333         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1334                 ctx.cmd.q_init.ring_size);
1335
1336         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1337         if (err)
1338                 return err;
1339
1340         q->hw_type = ctx.comp.q_init.hw_type;
1341         q->hw_index = ctx.comp.q_init.hw_index;
1342         q->db = ionic_db_map(lif, q);
1343
1344         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1345         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1346         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1347
1348         qcq->flags |= IONIC_QCQ_F_INITED;
1349
1350         return 0;
1351 }
1352
1353 int
1354 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1355 {
1356         struct ionic_queue *q = &qcq->q;
1357         struct ionic_lif *lif = qcq->lif;
1358         struct ionic_cq *cq = &qcq->cq;
1359         struct ionic_admin_ctx ctx = {
1360                 .pending_work = true,
1361                 .cmd.q_init = {
1362                         .opcode = IONIC_CMD_Q_INIT,
1363                         .lif_index = lif->index,
1364                         .type = q->type,
1365                         .index = q->index,
1366                         .flags = IONIC_QINIT_F_SG,
1367                         .intr_index = cq->bound_intr->index,
1368                         .ring_size = rte_log2_u32(q->num_descs),
1369                         .ring_base = q->base_pa,
1370                         .cq_ring_base = cq->base_pa,
1371                         .sg_ring_base = q->sg_base_pa,
1372                 },
1373         };
1374         int err;
1375
1376         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1377         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1378                 ctx.cmd.q_init.ring_base);
1379         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1380                 ctx.cmd.q_init.ring_size);
1381
1382         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1383         if (err)
1384                 return err;
1385
1386         q->hw_type = ctx.comp.q_init.hw_type;
1387         q->hw_index = ctx.comp.q_init.hw_index;
1388         q->db = ionic_db_map(lif, q);
1389
1390         qcq->flags |= IONIC_QCQ_F_INITED;
1391
1392         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1393         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1394         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1395
1396         return 0;
1397 }
1398
1399 static int
1400 ionic_station_set(struct ionic_lif *lif)
1401 {
1402         struct ionic_admin_ctx ctx = {
1403                 .pending_work = true,
1404                 .cmd.lif_getattr = {
1405                         .opcode = IONIC_CMD_LIF_GETATTR,
1406                         .index = lif->index,
1407                         .attr = IONIC_LIF_ATTR_MAC,
1408                 },
1409         };
1410         int err;
1411
1412         IONIC_PRINT_CALL();
1413
1414         err = ionic_adminq_post_wait(lif, &ctx);
1415         if (err)
1416                 return err;
1417
1418         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1419                         lif->mac_addr)) {
1420                 IONIC_PRINT(INFO, "deleting station MAC addr");
1421
1422                 ionic_lif_addr_del(lif, lif->mac_addr);
1423         }
1424
1425         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1426
1427         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1428                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1429                 return 0;
1430         }
1431
1432         IONIC_PRINT(DEBUG, "adding station MAC addr");
1433
1434         ionic_lif_addr_add(lif, lif->mac_addr);
1435
1436         return 0;
1437 }
1438
1439 static void
1440 ionic_lif_set_name(struct ionic_lif *lif)
1441 {
1442         struct ionic_admin_ctx ctx = {
1443                 .pending_work = true,
1444                 .cmd.lif_setattr = {
1445                         .opcode = IONIC_CMD_LIF_SETATTR,
1446                         .index = lif->index,
1447                         .attr = IONIC_LIF_ATTR_NAME,
1448                 },
1449         };
1450
1451         snprintf(ctx.cmd.lif_setattr.name, sizeof(ctx.cmd.lif_setattr.name),
1452                 "%d", lif->port_id);
1453
1454         ionic_adminq_post_wait(lif, &ctx);
1455 }
1456
1457 int
1458 ionic_lif_init(struct ionic_lif *lif)
1459 {
1460         struct ionic_dev *idev = &lif->adapter->idev;
1461         struct ionic_q_init_comp comp;
1462         int err;
1463
1464         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1465
1466         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
1467         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1468         ionic_dev_cmd_comp(idev, &comp);
1469         if (err)
1470                 return err;
1471
1472         lif->hw_index = comp.hw_index;
1473
1474         err = ionic_lif_adminq_init(lif);
1475         if (err)
1476                 return err;
1477
1478         err = ionic_lif_notifyq_init(lif);
1479         if (err)
1480                 goto err_out_adminq_deinit;
1481
1482         lif->features =
1483                   IONIC_ETH_HW_VLAN_TX_TAG
1484                 | IONIC_ETH_HW_VLAN_RX_STRIP
1485                 | IONIC_ETH_HW_VLAN_RX_FILTER
1486                 | IONIC_ETH_HW_RX_HASH
1487                 | IONIC_ETH_HW_TX_SG
1488                 | IONIC_ETH_HW_RX_SG
1489                 | IONIC_ETH_HW_TX_CSUM
1490                 | IONIC_ETH_HW_RX_CSUM
1491                 | IONIC_ETH_HW_TSO
1492                 | IONIC_ETH_HW_TSO_IPV6
1493                 | IONIC_ETH_HW_TSO_ECN;
1494
1495         err = ionic_lif_set_features(lif);
1496         if (err)
1497                 goto err_out_notifyq_deinit;
1498
1499         err = ionic_rx_filters_init(lif);
1500         if (err)
1501                 goto err_out_notifyq_deinit;
1502
1503         err = ionic_station_set(lif);
1504         if (err)
1505                 goto err_out_rx_filter_deinit;
1506
1507         ionic_lif_set_name(lif);
1508
1509         lif->state |= IONIC_LIF_F_INITED;
1510
1511         return 0;
1512
1513 err_out_rx_filter_deinit:
1514         ionic_rx_filters_deinit(lif);
1515
1516 err_out_notifyq_deinit:
1517         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1518
1519 err_out_adminq_deinit:
1520         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1521
1522         return err;
1523 }
1524
1525 void
1526 ionic_lif_deinit(struct ionic_lif *lif)
1527 {
1528         if (!(lif->state & IONIC_LIF_F_INITED))
1529                 return;
1530
1531         ionic_rx_filters_deinit(lif);
1532         ionic_lif_rss_teardown(lif);
1533         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1534         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1535
1536         lif->state &= ~IONIC_LIF_F_INITED;
1537 }
1538
1539 int
1540 ionic_lif_configure(struct ionic_lif *lif)
1541 {
1542         struct ionic_identity *ident = &lif->adapter->ident;
1543         uint32_t ntxqs_per_lif =
1544                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1545         uint32_t nrxqs_per_lif =
1546                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1547         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1548         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1549
1550         lif->port_id = lif->eth_dev->data->port_id;
1551
1552         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1553                 lif->port_id);
1554
1555         if (nrxqs > 0)
1556                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1557
1558         if (ntxqs > 0)
1559                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1560
1561         lif->nrxqcqs = nrxqs_per_lif;
1562         lif->ntxqcqs = ntxqs_per_lif;
1563
1564         return 0;
1565 }
1566
1567 int
1568 ionic_lif_start(struct ionic_lif *lif)
1569 {
1570         uint32_t rx_mode = 0;
1571         uint32_t i;
1572         int err;
1573
1574         IONIC_PRINT(DEBUG, "Setting RSS configuration on port %u",
1575                 lif->port_id);
1576
1577         err = ionic_lif_rss_setup(lif);
1578         if (err)
1579                 return err;
1580
1581         IONIC_PRINT(DEBUG, "Setting RX mode on port %u",
1582                 lif->port_id);
1583
1584         rx_mode |= IONIC_RX_MODE_F_UNICAST;
1585         rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1586         rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1587
1588         lif->rx_mode = 0; /* set by ionic_set_rx_mode */
1589
1590         ionic_set_rx_mode(lif, rx_mode);
1591
1592         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1593                 "on port %u",
1594                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1595
1596         for (i = 0; i < lif->nrxqcqs; i++) {
1597                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1598                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1599                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1600
1601                         if (err)
1602                                 return err;
1603                 }
1604         }
1605
1606         for (i = 0; i < lif->ntxqcqs; i++) {
1607                 struct ionic_qcq *txq = lif->txqcqs[i];
1608                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1609                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1610
1611                         if (err)
1612                                 return err;
1613                 }
1614         }
1615
1616         ionic_link_status_check(lif);
1617
1618         /* Carrier ON here */
1619
1620         return 0;
1621 }
1622
1623 int
1624 ionic_lif_identify(struct ionic_adapter *adapter)
1625 {
1626         struct ionic_dev *idev = &adapter->idev;
1627         struct ionic_identity *ident = &adapter->ident;
1628         int err;
1629         unsigned int i;
1630         unsigned int lif_words = sizeof(ident->lif.words) /
1631                 sizeof(ident->lif.words[0]);
1632         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1633                 sizeof(idev->dev_cmd->data[0]);
1634         unsigned int nwords;
1635
1636         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1637                 IONIC_IDENTITY_VERSION_1);
1638         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1639         if (err)
1640                 return (err);
1641
1642         nwords = RTE_MIN(lif_words, cmd_words);
1643         for (i = 0; i < nwords; i++)
1644                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1645
1646         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1647                 ident->lif.capabilities);
1648
1649         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1650                 ident->lif.eth.max_ucast_filters);
1651         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1652                 ident->lif.eth.max_mcast_filters);
1653
1654         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1655                 ident->lif.eth.config.features);
1656         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1657                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1658         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1659                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1660         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1661                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1662         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1663                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1664
1665         return 0;
1666 }
1667
1668 int
1669 ionic_lifs_size(struct ionic_adapter *adapter)
1670 {
1671         struct ionic_identity *ident = &adapter->ident;
1672         uint32_t nlifs = ident->dev.nlifs;
1673         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1674
1675         adapter->max_ntxqs_per_lif =
1676                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1677         adapter->max_nrxqs_per_lif =
1678                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1679
1680         nintrs = nlifs * 1 /* notifyq */;
1681
1682         if (nintrs > dev_nintrs) {
1683                 IONIC_PRINT(ERR, "At most %d intr queues supported, minimum required is %u",
1684                         dev_nintrs, nintrs);
1685                 return -ENOSPC;
1686         }
1687
1688         adapter->nintrs = nintrs;
1689
1690         return 0;
1691 }