drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .lif_index = lif->index,
  29                         .type = q->type,
  30                         .index = q->index,
  31                         .oper = IONIC_Q_ENABLE,
  32                 },
  33         };
  34
  35         if (qcq->flags & IONIC_QCQ_F_INTR) {
  36                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  37                         IONIC_INTR_MASK_CLEAR);
  38         }
  39
  40         return ionic_adminq_post_wait(lif, &ctx);
  41 }
  42
  43 int
  44 ionic_qcq_disable(struct ionic_qcq *qcq)
  45 {
  46         struct ionic_queue *q = &qcq->q;
  47         struct ionic_lif *lif = q->lif;
  48         struct ionic_dev *idev = &lif->adapter->idev;
  49         struct ionic_admin_ctx ctx = {
  50                 .pending_work = true,
  51                 .cmd.q_control = {
  52                         .opcode = IONIC_CMD_Q_CONTROL,
  53                         .lif_index = lif->index,
  54                         .type = q->type,
  55                         .index = q->index,
  56                         .oper = IONIC_Q_DISABLE,
  57                 },
  58         };
  59
  60         if (qcq->flags & IONIC_QCQ_F_INTR) {
  61                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  62                         IONIC_INTR_MASK_SET);
  63         }
  64
  65         return ionic_adminq_post_wait(lif, &ctx);
  66 }
  67
  68 int
  69 ionic_lif_stop(struct ionic_lif *lif __rte_unused)
  70 {
  71         /* Carrier OFF here */
  72
  73         return 0;
  74 }
  75
  76 void
  77 ionic_lif_reset(struct ionic_lif *lif)
  78 {
  79         struct ionic_dev *idev = &lif->adapter->idev;
  80         int err;
  81
  82         IONIC_PRINT_CALL();
  83
  84         ionic_dev_cmd_lif_reset(idev, lif->index);
  85         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  86         if (err)
  87                 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name);
  88 }
  89
  90 static void
  91 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  92 {
  93         struct ionic_lif_stats *ls = &lif->info->stats;
  94         uint32_t i;
  95         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  96                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  97         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  98                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  99
 100         memset(stats, 0, sizeof(*stats));
 101
 102         if (ls == NULL) {
 103                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 104                         lif->port_id);
 105                 return;
 106         }
 107
 108         /* RX */
 109
 110         stats->ipackets = ls->rx_ucast_packets +
 111                 ls->rx_mcast_packets +
 112                 ls->rx_bcast_packets;
 113
 114         stats->ibytes = ls->rx_ucast_bytes +
 115                 ls->rx_mcast_bytes +
 116                 ls->rx_bcast_bytes;
 117
 118         for (i = 0; i < lif->nrxqcqs; i++) {
 119                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 120                 stats->imissed +=
 121                         rx_stats->no_cb_arg +
 122                         rx_stats->bad_cq_status +
 123                         rx_stats->no_room +
 124                         rx_stats->bad_len;
 125         }
 126
 127         stats->imissed +=
 128                 ls->rx_ucast_drop_packets +
 129                 ls->rx_mcast_drop_packets +
 130                 ls->rx_bcast_drop_packets;
 131
 132         stats->imissed +=
 133                 ls->rx_queue_empty +
 134                 ls->rx_dma_error +
 135                 ls->rx_queue_disabled +
 136                 ls->rx_desc_fetch_error +
 137                 ls->rx_desc_data_error;
 138
 139         for (i = 0; i < num_rx_q_counters; i++) {
 140                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 141                 stats->q_ipackets[i] = rx_stats->packets;
 142                 stats->q_ibytes[i] = rx_stats->bytes;
 143                 stats->q_errors[i] =
 144                         rx_stats->no_cb_arg +
 145                         rx_stats->bad_cq_status +
 146                         rx_stats->no_room +
 147                         rx_stats->bad_len;
 148         }
 149
 150         /* TX */
 151
 152         stats->opackets = ls->tx_ucast_packets +
 153                 ls->tx_mcast_packets +
 154                 ls->tx_bcast_packets;
 155
 156         stats->obytes = ls->tx_ucast_bytes +
 157                 ls->tx_mcast_bytes +
 158                 ls->tx_bcast_bytes;
 159
 160         for (i = 0; i < lif->ntxqcqs; i++) {
 161                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 162                 stats->oerrors += tx_stats->drop;
 163         }
 164
 165         stats->oerrors +=
 166                 ls->tx_ucast_drop_packets +
 167                 ls->tx_mcast_drop_packets +
 168                 ls->tx_bcast_drop_packets;
 169
 170         stats->oerrors +=
 171                 ls->tx_dma_error +
 172                 ls->tx_queue_disabled +
 173                 ls->tx_desc_fetch_error +
 174                 ls->tx_desc_data_error;
 175
 176         for (i = 0; i < num_tx_q_counters; i++) {
 177                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 178                 stats->q_opackets[i] = tx_stats->packets;
 179                 stats->q_obytes[i] = tx_stats->bytes;
 180         }
 181 }
 182
 183 void
 184 ionic_lif_get_stats(const struct ionic_lif *lif,
 185                 struct rte_eth_stats *stats)
 186 {
 187         ionic_lif_get_abs_stats(lif, stats);
 188
 189         stats->ipackets  -= lif->stats_base.ipackets;
 190         stats->opackets  -= lif->stats_base.opackets;
 191         stats->ibytes    -= lif->stats_base.ibytes;
 192         stats->obytes    -= lif->stats_base.obytes;
 193         stats->imissed   -= lif->stats_base.imissed;
 194         stats->ierrors   -= lif->stats_base.ierrors;
 195         stats->oerrors   -= lif->stats_base.oerrors;
 196         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 197 }
 198
 199 void
 200 ionic_lif_reset_stats(struct ionic_lif *lif)
 201 {
 202         uint32_t i;
 203
 204         for (i = 0; i < lif->nrxqcqs; i++) {
 205                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 206                         sizeof(struct ionic_rx_stats));
 207                 memset(&lif->txqcqs[i]->stats.tx, 0,
 208                         sizeof(struct ionic_tx_stats));
 209         }
 210
 211         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 212 }
 213
 214 void
 215 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 216 {
 217         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 218         uint64_t *stats64 = (uint64_t *)stats;
 219         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 220         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 221
 222         for (i = 0; i < count; i++)
 223                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 224 }
 225
 226 void
 227 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 228 {
 229         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 230         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 231         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 232
 233         for (i = 0; i < count; i++)
 234                 lif_stats64_base[i] = lif_stats64[i];
 235 }
 236
 237 static int
 238 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 239 {
 240         struct ionic_admin_ctx ctx = {
 241                 .pending_work = true,
 242                 .cmd.rx_filter_add = {
 243                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 244                         .match = IONIC_RX_FILTER_MATCH_MAC,
 245                 },
 246         };
 247         int err;
 248
 249         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 250
 251         err = ionic_adminq_post_wait(lif, &ctx);
 252         if (err)
 253                 return err;
 254
 255         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 256                 ctx.comp.rx_filter_add.filter_id);
 257
 258         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 259 }
 260
 261 static int
 262 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 263 {
 264         struct ionic_admin_ctx ctx = {
 265                 .pending_work = true,
 266                 .cmd.rx_filter_del = {
 267                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 268                 },
 269         };
 270         struct ionic_rx_filter *f;
 271         int err;
 272
 273         IONIC_PRINT_CALL();
 274
 275         rte_spinlock_lock(&lif->rx_filters.lock);
 276
 277         f = ionic_rx_filter_by_addr(lif, addr);
 278         if (!f) {
 279                 rte_spinlock_unlock(&lif->rx_filters.lock);
 280                 return -ENOENT;
 281         }
 282
 283         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 284         ionic_rx_filter_free(f);
 285
 286         rte_spinlock_unlock(&lif->rx_filters.lock);
 287
 288         err = ionic_adminq_post_wait(lif, &ctx);
 289         if (err)
 290                 return err;
 291
 292         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 293                 ctx.cmd.rx_filter_del.filter_id);
 294
 295         return 0;
 296 }
 297
 298 int
 299 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 300                 struct rte_ether_addr *mac_addr,
 301                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 302 {
 303         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 304
 305         IONIC_PRINT_CALL();
 306
 307         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 308 }
 309
 310 void
 311 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index)
 312 {
 313         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 314         struct ionic_adapter *adapter = lif->adapter;
 315         struct rte_ether_addr *mac_addr;
 316
 317         IONIC_PRINT_CALL();
 318
 319         if (index >= adapter->max_mac_addrs) {
 320                 IONIC_PRINT(WARNING,
 321                         "Index %u is above MAC filter limit %u",
 322                         index, adapter->max_mac_addrs);
 323                 return;
 324         }
 325
 326         mac_addr = &eth_dev->data->mac_addrs[index];
 327
 328         if (!rte_is_valid_assigned_ether_addr(mac_addr))
 329                 return;
 330
 331         ionic_lif_addr_del(lif, (const uint8_t *)mac_addr);
 332 }
 333
 334 int
 335 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 336 {
 337         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 338
 339         IONIC_PRINT_CALL();
 340
 341         if (mac_addr == NULL) {
 342                 IONIC_PRINT(NOTICE, "New mac is null");
 343                 return -1;
 344         }
 345
 346         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 347                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 348                         lif->mac_addr);
 349                 ionic_lif_addr_del(lif, lif->mac_addr);
 350                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 351         }
 352
 353         IONIC_PRINT(INFO, "Updating mac addr");
 354
 355         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 356
 357         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 358 }
 359
 360 static int
 361 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 362 {
 363         struct ionic_admin_ctx ctx = {
 364                 .pending_work = true,
 365                 .cmd.rx_filter_add = {
 366                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 367                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 368                         .vlan.vlan = vid,
 369                 },
 370         };
 371         int err;
 372
 373         err = ionic_adminq_post_wait(lif, &ctx);
 374         if (err)
 375                 return err;
 376
 377         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 378                 ctx.comp.rx_filter_add.filter_id);
 379
 380         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 381 }
 382
 383 static int
 384 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 385 {
 386         struct ionic_admin_ctx ctx = {
 387                 .pending_work = true,
 388                 .cmd.rx_filter_del = {
 389                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 390                 },
 391         };
 392         struct ionic_rx_filter *f;
 393         int err;
 394
 395         IONIC_PRINT_CALL();
 396
 397         rte_spinlock_lock(&lif->rx_filters.lock);
 398
 399         f = ionic_rx_filter_by_vlan(lif, vid);
 400         if (!f) {
 401                 rte_spinlock_unlock(&lif->rx_filters.lock);
 402                 return -ENOENT;
 403         }
 404
 405         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 406         ionic_rx_filter_free(f);
 407         rte_spinlock_unlock(&lif->rx_filters.lock);
 408
 409         err = ionic_adminq_post_wait(lif, &ctx);
 410         if (err)
 411                 return err;
 412
 413         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 414                 ctx.cmd.rx_filter_del.filter_id);
 415
 416         return 0;
 417 }
 418
 419 int
 420 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 421                 int on)
 422 {
 423         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 424         int err;
 425
 426         if (on)
 427                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 428         else
 429                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 430
 431         return err;
 432 }
 433
 434 static void
 435 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 436 {
 437         struct ionic_admin_ctx ctx = {
 438                 .pending_work = true,
 439                 .cmd.rx_mode_set = {
 440                         .opcode = IONIC_CMD_RX_MODE_SET,
 441                         .lif_index = lif->index,
 442                         .rx_mode = rx_mode,
 443                 },
 444         };
 445         int err;
 446
 447         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 448                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 449         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 450                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 451         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 452                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 453         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 454                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 455         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 456                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 457
 458         err = ionic_adminq_post_wait(lif, &ctx);
 459         if (err)
 460                 IONIC_PRINT(ERR, "Failure setting RX mode");
 461 }
 462
 463 static void
 464 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 465 {
 466         if (lif->rx_mode != rx_mode) {
 467                 lif->rx_mode = rx_mode;
 468                 ionic_lif_rx_mode(lif, rx_mode);
 469         }
 470 }
 471
 472 int
 473 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 474 {
 475         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 476         uint32_t rx_mode = lif->rx_mode;
 477
 478         IONIC_PRINT_CALL();
 479
 480         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 481
 482         ionic_set_rx_mode(lif, rx_mode);
 483
 484         return 0;
 485 }
 486
 487 int
 488 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 489 {
 490         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 491         uint32_t rx_mode = lif->rx_mode;
 492
 493         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 494
 495         ionic_set_rx_mode(lif, rx_mode);
 496
 497         return 0;
 498 }
 499
 500 int
 501 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 502 {
 503         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 504         uint32_t rx_mode = lif->rx_mode;
 505
 506         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 507
 508         ionic_set_rx_mode(lif, rx_mode);
 509
 510         return 0;
 511 }
 512
 513 int
 514 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 515 {
 516         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 517         uint32_t rx_mode = lif->rx_mode;
 518
 519         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 520
 521         ionic_set_rx_mode(lif, rx_mode);
 522
 523         return 0;
 524 }
 525
 526 int
 527 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 528 {
 529         struct ionic_admin_ctx ctx = {
 530                 .pending_work = true,
 531                 .cmd.lif_setattr = {
 532                         .opcode = IONIC_CMD_LIF_SETATTR,
 533                         .index = lif->index,
 534                         .attr = IONIC_LIF_ATTR_MTU,
 535                         .mtu = new_mtu,
 536                 },
 537         };
 538         int err;
 539
 540         err = ionic_adminq_post_wait(lif, &ctx);
 541         if (err)
 542                 return err;
 543
 544         return 0;
 545 }
 546
 547 int
 548 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 549 {
 550         struct ionic_adapter *adapter = lif->adapter;
 551         struct ionic_dev *idev = &adapter->idev;
 552         unsigned long index;
 553
 554         /*
 555          * Note: interrupt handler is called for index = 0 only
 556          * (we use interrupts for the notifyq only anyway,
 557          * which has index = 0)
 558          */
 559
 560         for (index = 0; index < adapter->nintrs; index++)
 561                 if (!adapter->intrs[index])
 562                         break;
 563
 564         if (index == adapter->nintrs)
 565                 return -ENOSPC;
 566
 567         adapter->intrs[index] = true;
 568
 569         ionic_intr_init(idev, intr, index);
 570
 571         return 0;
 572 }
 573
 574 void
 575 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 576 {
 577         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 578                 lif->adapter->intrs[intr->index] = false;
 579 }
 580
 581 static int
 582 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 583                 uint32_t index,
 584                 const char *base, uint32_t flags,
 585                 uint32_t num_descs,
 586                 uint32_t desc_size,
 587                 uint32_t cq_desc_size,
 588                 uint32_t sg_desc_size,
 589                 struct ionic_qcq **qcq)
 590 {
 591         struct ionic_dev *idev = &lif->adapter->idev;
 592         struct ionic_qcq *new;
 593         uint32_t q_size, cq_size, sg_size, total_size;
 594         void *q_base, *cq_base, *sg_base;
 595         rte_iova_t q_base_pa = 0;
 596         rte_iova_t cq_base_pa = 0;
 597         rte_iova_t sg_base_pa = 0;
 598         uint32_t socket_id = rte_socket_id();
 599         int err;
 600
 601         *qcq = NULL;
 602
 603         q_size  = num_descs * desc_size;
 604         cq_size = num_descs * cq_desc_size;
 605         sg_size = num_descs * sg_desc_size;
 606
 607         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 608                 RTE_ALIGN(cq_size, PAGE_SIZE);
 609         /*
 610          * Note: aligning q_size/cq_size is not enough due to cq_base address
 611          * aligning as q_base could be not aligned to the page.
 612          * Adding PAGE_SIZE.
 613          */
 614         total_size += PAGE_SIZE;
 615
 616         if (flags & IONIC_QCQ_F_SG) {
 617                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 618                 total_size += PAGE_SIZE;
 619         }
 620
 621         new = rte_zmalloc("ionic", sizeof(*new), 0);
 622         if (!new) {
 623                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 624                 return -ENOMEM;
 625         }
 626
 627         new->lif = lif;
 628         new->flags = flags;
 629
 630         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 631         if (!new->q.info) {
 632                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 633                 return -ENOMEM;
 634         }
 635
 636         new->q.type = type;
 637
 638         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 639                 desc_size, sg_desc_size);
 640         if (err) {
 641                 IONIC_PRINT(ERR, "Queue initialization failed");
 642                 return err;
 643         }
 644
 645         if (flags & IONIC_QCQ_F_INTR) {
 646                 err = ionic_intr_alloc(lif, &new->intr);
 647                 if (err)
 648                         return err;
 649
 650                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 651                         IONIC_INTR_MASK_SET);
 652         } else {
 653                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 654         }
 655
 656         err = ionic_cq_init(lif, &new->cq, &new->intr,
 657                 num_descs, cq_desc_size);
 658         if (err) {
 659                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 660                 goto err_out_free_intr;
 661         }
 662
 663         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 664                 base /* name */, index /* queue_idx */,
 665                 total_size, IONIC_ALIGN, socket_id);
 666
 667         if (!new->base_z) {
 668                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 669                 err = -ENOMEM;
 670                 goto err_out_free_intr;
 671         }
 672
 673         new->base = new->base_z->addr;
 674         new->base_pa = new->base_z->iova;
 675         new->total_size = total_size;
 676
 677         q_base = new->base;
 678         q_base_pa = new->base_pa;
 679
 680         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 681         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 682
 683         if (flags & IONIC_QCQ_F_SG) {
 684                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 685                         PAGE_SIZE);
 686                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 687                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 688         }
 689
 690         IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 691                 "SG-base-PA = %#jx",
 692                 q_base_pa, cq_base_pa, sg_base_pa);
 693
 694         ionic_q_map(&new->q, q_base, q_base_pa);
 695         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 696         ionic_cq_bind(&new->cq, &new->q);
 697
 698         *qcq = new;
 699
 700         return 0;
 701
 702 err_out_free_intr:
 703         if (flags & IONIC_QCQ_F_INTR)
 704                 ionic_intr_free(lif, &new->intr);
 705
 706         return err;
 707 }
 708
 709 void
 710 ionic_qcq_free(struct ionic_qcq *qcq)
 711 {
 712         if (qcq->base_z) {
 713                 qcq->base = NULL;
 714                 qcq->base_pa = 0;
 715                 rte_memzone_free(qcq->base_z);
 716                 qcq->base_z = NULL;
 717         }
 718
 719         if (qcq->q.info) {
 720                 rte_free(qcq->q.info);
 721                 qcq->q.info = NULL;
 722         }
 723
 724         rte_free(qcq);
 725 }
 726
 727 int
 728 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 729                 struct ionic_qcq **qcq)
 730 {
 731         uint32_t flags;
 732         int err = -ENOMEM;
 733
 734         flags = IONIC_QCQ_F_SG;
 735         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 736                 nrxq_descs,
 737                 sizeof(struct ionic_rxq_desc),
 738                 sizeof(struct ionic_rxq_comp),
 739                 sizeof(struct ionic_rxq_sg_desc),
 740                 &lif->rxqcqs[index]);
 741         if (err)
 742                 return err;
 743
 744         *qcq = lif->rxqcqs[index];
 745
 746         return 0;
 747 }
 748
 749 int
 750 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 751                 struct ionic_qcq **qcq)
 752 {
 753         uint32_t flags;
 754         int err = -ENOMEM;
 755
 756         flags = IONIC_QCQ_F_SG;
 757         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 758                 ntxq_descs,
 759                 sizeof(struct ionic_txq_desc),
 760                 sizeof(struct ionic_txq_comp),
 761                 sizeof(struct ionic_txq_sg_desc),
 762                 &lif->txqcqs[index]);
 763         if (err)
 764                 return err;
 765
 766         *qcq = lif->txqcqs[index];
 767
 768         return 0;
 769 }
 770
 771 static int
 772 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 773 {
 774         uint32_t flags;
 775         int err = -ENOMEM;
 776
 777         flags = 0;
 778         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 779                 IONIC_ADMINQ_LENGTH,
 780                 sizeof(struct ionic_admin_cmd),
 781                 sizeof(struct ionic_admin_comp),
 782                 0,
 783                 &lif->adminqcq);
 784         if (err)
 785                 return err;
 786
 787         return 0;
 788 }
 789
 790 static int
 791 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 792 {
 793         uint32_t flags;
 794         int err = -ENOMEM;
 795
 796         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 797
 798         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 799                 flags,
 800                 IONIC_NOTIFYQ_LENGTH,
 801                 sizeof(struct ionic_notifyq_cmd),
 802                 sizeof(union ionic_notifyq_comp),
 803                 0,
 804                 &lif->notifyqcq);
 805         if (err)
 806                 return err;
 807
 808         return 0;
 809 }
 810
 811 static void *
 812 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 813 {
 814         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 815
 816         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 817                 return NULL;
 818
 819         return (void *)&vaddr[page_num << PAGE_SHIFT];
 820 }
 821
 822 int
 823 ionic_lif_alloc(struct ionic_lif *lif)
 824 {
 825         struct ionic_adapter *adapter = lif->adapter;
 826         uint32_t socket_id = rte_socket_id();
 827         int dbpage_num;
 828         int err;
 829
 830         /*
 831          * lif->name was zeroed on allocation.
 832          * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated.
 833          */
 834         memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1);
 835
 836         IONIC_PRINT(DEBUG, "LIF: %s", lif->name);
 837
 838         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 839
 840         rte_spinlock_init(&lif->adminq_lock);
 841         rte_spinlock_init(&lif->adminq_service_lock);
 842
 843         dbpage_num = ionic_db_page_num(lif, 0);
 844
 845         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, dbpage_num);
 846         if (!lif->kern_dbpage) {
 847                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 848                 return -ENOMEM;
 849         }
 850
 851         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 852                 adapter->max_ntxqs_per_lif, 0);
 853
 854         if (!lif->txqcqs) {
 855                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 856                 return -ENOMEM;
 857         }
 858
 859         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 860                 adapter->max_nrxqs_per_lif, 0);
 861
 862         if (!lif->rxqcqs) {
 863                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 864                 return -ENOMEM;
 865         }
 866
 867         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 868
 869         err = ionic_notify_qcq_alloc(lif);
 870         if (err) {
 871                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 872                 return err;
 873         }
 874
 875         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 876
 877         err = ionic_admin_qcq_alloc(lif);
 878         if (err) {
 879                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 880                 return err;
 881         }
 882
 883         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 884
 885         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 886
 887         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 888                 "lif_info", 0 /* queue_idx*/,
 889                 lif->info_sz, IONIC_ALIGN, socket_id);
 890         if (!lif->info_z) {
 891                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 892                 return -ENOMEM;
 893         }
 894
 895         lif->info = lif->info_z->addr;
 896         lif->info_pa = lif->info_z->iova;
 897
 898         return 0;
 899 }
 900
 901 void
 902 ionic_lif_free(struct ionic_lif *lif)
 903 {
 904         if (lif->notifyqcq) {
 905                 ionic_qcq_free(lif->notifyqcq);
 906                 lif->notifyqcq = NULL;
 907         }
 908
 909         if (lif->adminqcq) {
 910                 ionic_qcq_free(lif->adminqcq);
 911                 lif->adminqcq = NULL;
 912         }
 913
 914         if (lif->txqcqs) {
 915                 rte_free(lif->txqcqs);
 916                 lif->txqcqs = NULL;
 917         }
 918
 919         if (lif->rxqcqs) {
 920                 rte_free(lif->rxqcqs);
 921                 lif->rxqcqs = NULL;
 922         }
 923
 924         if (lif->info) {
 925                 rte_memzone_free(lif->info_z);
 926                 lif->info = NULL;
 927         }
 928 }
 929
 930 int
 931 ionic_lif_rss_config(struct ionic_lif *lif,
 932                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 933 {
 934         struct ionic_admin_ctx ctx = {
 935                 .pending_work = true,
 936                 .cmd.lif_setattr = {
 937                         .opcode = IONIC_CMD_LIF_SETATTR,
 938                         .attr = IONIC_LIF_ATTR_RSS,
 939                         .rss.types = types,
 940                         .rss.addr = lif->rss_ind_tbl_pa,
 941                 },
 942         };
 943         unsigned int i;
 944
 945         IONIC_PRINT_CALL();
 946
 947         lif->rss_types = types;
 948
 949         if (key)
 950                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 951
 952         if (indir)
 953                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 954                         lif->rss_ind_tbl[i] = indir[i];
 955
 956         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 957                IONIC_RSS_HASH_KEY_SIZE);
 958
 959         return ionic_adminq_post_wait(lif, &ctx);
 960 }
 961
 962 static int
 963 ionic_lif_rss_setup(struct ionic_lif *lif)
 964 {
 965         static const uint8_t toeplitz_symmetric_key[] = {
 966                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 967                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 968                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 969                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 970                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 971         };
 972         uint32_t i;
 973         uint16_t tbl_sz = lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 974
 975         IONIC_PRINT_CALL();
 976
 977         if (!lif->rss_ind_tbl_z) {
 978                 lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 979                                         "rss_ind_tbl", 0 /* queue_idx */,
 980                                         sizeof(*lif->rss_ind_tbl) * tbl_sz,
 981                                         IONIC_ALIGN, rte_socket_id());
 982                 if (!lif->rss_ind_tbl_z) {
 983                         IONIC_PRINT(ERR, "OOM");
 984                         return -ENOMEM;
 985                 }
 986
 987                 lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 988                 lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
 989         }
 990
 991         if (lif->rss_ind_tbl_nrxqcqs != lif->nrxqcqs) {
 992                 lif->rss_ind_tbl_nrxqcqs = lif->nrxqcqs;
 993
 994                 /* Fill indirection table with 'default' values */
 995                 for (i = 0; i < tbl_sz; i++)
 996                         lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
 997         }
 998
 999         return ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
1000                         toeplitz_symmetric_key, NULL);
1001 }
1002
1003 static void
1004 ionic_lif_rss_teardown(struct ionic_lif *lif)
1005 {
1006         if (!lif->rss_ind_tbl)
1007                 return;
1008
1009         if (lif->rss_ind_tbl_z) {
1010                 /* Disable RSS on the NIC */
1011                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1012
1013                 lif->rss_ind_tbl = NULL;
1014                 lif->rss_ind_tbl_pa = 0;
1015                 rte_memzone_free(lif->rss_ind_tbl_z);
1016                 lif->rss_ind_tbl_z = NULL;
1017         }
1018 }
1019
1020 static void
1021 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1022 {
1023         struct ionic_dev *idev = &lif->adapter->idev;
1024
1025         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1026                 return;
1027
1028         if (qcq->flags & IONIC_QCQ_F_INTR)
1029                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1030                         IONIC_INTR_MASK_SET);
1031
1032         qcq->flags &= ~IONIC_QCQ_F_INITED;
1033 }
1034
1035 void
1036 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1037 {
1038         ionic_lif_qcq_deinit(qcq->lif, qcq);
1039 }
1040
1041 void
1042 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1043 {
1044         ionic_lif_qcq_deinit(qcq->lif, qcq);
1045 }
1046
1047 bool
1048 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1049                 void *cb_arg __rte_unused)
1050 {
1051         struct ionic_admin_comp *cq_desc_base = cq->base;
1052         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1053
1054         if (!color_match(cq_desc->color, cq->done_color))
1055                 return false;
1056
1057         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1058
1059         return true;
1060 }
1061
1062 /* This acts like ionic_napi */
1063 int
1064 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1065                 void *cb_arg)
1066 {
1067         struct ionic_cq *cq = &qcq->cq;
1068         uint32_t work_done;
1069
1070         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1071
1072         return work_done;
1073 }
1074
1075 static void
1076 ionic_link_status_check(struct ionic_lif *lif)
1077 {
1078         struct ionic_adapter *adapter = lif->adapter;
1079         bool link_up;
1080
1081         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1082
1083         if (!lif->info)
1084                 return;
1085
1086         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1087
1088         if ((link_up  && adapter->link_up) ||
1089             (!link_up && !adapter->link_up))
1090                 return;
1091
1092         if (link_up) {
1093                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1094                         lif->info->status.link_speed);
1095                 adapter->link_speed = lif->info->status.link_speed;
1096         } else {
1097                 IONIC_PRINT(DEBUG, "Link down");
1098         }
1099
1100         adapter->link_up = link_up;
1101 }
1102
1103 static bool
1104 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1105 {
1106         union ionic_notifyq_comp *cq_desc_base = cq->base;
1107         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1108         struct ionic_lif *lif = cb_arg;
1109
1110         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1111                 cq_desc->event.eid, cq_desc->event.ecode);
1112
1113         /* Have we run out of new completions to process? */
1114         if (!(cq_desc->event.eid > lif->last_eid))
1115                 return false;
1116
1117         lif->last_eid = cq_desc->event.eid;
1118
1119         switch (cq_desc->event.ecode) {
1120         case IONIC_EVENT_LINK_CHANGE:
1121                 IONIC_PRINT(DEBUG,
1122                         "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
1123                         cq_desc->event.eid,
1124                         cq_desc->link_change.link_status,
1125                         cq_desc->link_change.link_speed);
1126
1127                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1128
1129                 break;
1130         default:
1131                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1132                         cq_desc->event.ecode, cq_desc->event.eid);
1133                 break;
1134         }
1135
1136         return true;
1137 }
1138
1139 int
1140 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1141 {
1142         struct ionic_dev *idev = &lif->adapter->idev;
1143         struct ionic_qcq *qcq = lif->notifyqcq;
1144         uint32_t work_done;
1145
1146         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1147                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1148                 return -1;
1149         }
1150
1151         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1152                 IONIC_INTR_MASK_SET);
1153
1154         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1155
1156         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1157                 ionic_link_status_check(lif);
1158
1159         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1160                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1161
1162         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1163                 IONIC_INTR_MASK_CLEAR);
1164
1165         return 0;
1166 }
1167
1168 static int
1169 ionic_lif_adminq_init(struct ionic_lif *lif)
1170 {
1171         struct ionic_dev *idev = &lif->adapter->idev;
1172         struct ionic_qcq *qcq = lif->adminqcq;
1173         struct ionic_queue *q = &qcq->q;
1174         struct ionic_q_init_comp comp;
1175         int err;
1176
1177         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
1178         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1179         if (err)
1180                 return err;
1181
1182         ionic_dev_cmd_comp(idev, &comp);
1183
1184         q->hw_type = comp.hw_type;
1185         q->hw_index = comp.hw_index;
1186         q->db = ionic_db_map(lif, q);
1187
1188         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1189         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1190         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1191
1192         if (qcq->flags & IONIC_QCQ_F_INTR)
1193                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1194                         IONIC_INTR_MASK_CLEAR);
1195
1196         qcq->flags |= IONIC_QCQ_F_INITED;
1197
1198         return 0;
1199 }
1200
1201 static int
1202 ionic_lif_notifyq_init(struct ionic_lif *lif)
1203 {
1204         struct ionic_dev *idev = &lif->adapter->idev;
1205         struct ionic_qcq *qcq = lif->notifyqcq;
1206         struct ionic_queue *q = &qcq->q;
1207         int err;
1208
1209         struct ionic_admin_ctx ctx = {
1210                 .pending_work = true,
1211                 .cmd.q_init = {
1212                         .opcode = IONIC_CMD_Q_INIT,
1213                         .lif_index = lif->index,
1214                         .type = q->type,
1215                         .index = q->index,
1216                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1217                         .intr_index = qcq->intr.index,
1218                         .ring_size = rte_log2_u32(q->num_descs),
1219                         .ring_base = q->base_pa,
1220                 }
1221         };
1222
1223         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1224                 ctx.cmd.q_init.index);
1225         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1226                 ctx.cmd.q_init.ring_base);
1227         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1228                 ctx.cmd.q_init.ring_size);
1229         IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver);
1230
1231         err = ionic_adminq_post_wait(lif, &ctx);
1232         if (err)
1233                 return err;
1234
1235         q->hw_type = ctx.comp.q_init.hw_type;
1236         q->hw_index = ctx.comp.q_init.hw_index;
1237         q->db = NULL;
1238
1239         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1240         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1241         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1242
1243         if (qcq->flags & IONIC_QCQ_F_INTR)
1244                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1245                         IONIC_INTR_MASK_CLEAR);
1246
1247         qcq->flags |= IONIC_QCQ_F_INITED;
1248
1249         return 0;
1250 }
1251
1252 int
1253 ionic_lif_set_features(struct ionic_lif *lif)
1254 {
1255         struct ionic_admin_ctx ctx = {
1256                 .pending_work = true,
1257                 .cmd.lif_setattr = {
1258                         .opcode = IONIC_CMD_LIF_SETATTR,
1259                         .index = lif->index,
1260                         .attr = IONIC_LIF_ATTR_FEATURES,
1261                         .features = lif->features,
1262                 },
1263         };
1264         int err;
1265
1266         err = ionic_adminq_post_wait(lif, &ctx);
1267         if (err)
1268                 return err;
1269
1270         lif->hw_features = (ctx.cmd.lif_setattr.features &
1271                 ctx.comp.lif_setattr.features);
1272
1273         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1274                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1275         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1276                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1277         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1278                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1279         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1280                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1281         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1282                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1283         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1284                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1285         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1286                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1287         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1288                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1289         if (lif->hw_features & IONIC_ETH_HW_TSO)
1290                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1291         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1292                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1293         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1294                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1295         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1296                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1297         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1298                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1299         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1300                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1301         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1302                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1303         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1304                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1305         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1306                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1307
1308         return 0;
1309 }
1310
1311 int
1312 ionic_lif_txq_init(struct ionic_qcq *qcq)
1313 {
1314         struct ionic_queue *q = &qcq->q;
1315         struct ionic_lif *lif = qcq->lif;
1316         struct ionic_cq *cq = &qcq->cq;
1317         struct ionic_admin_ctx ctx = {
1318                 .pending_work = true,
1319                 .cmd.q_init = {
1320                         .opcode = IONIC_CMD_Q_INIT,
1321                         .lif_index = lif->index,
1322                         .type = q->type,
1323                         .index = q->index,
1324                         .flags = IONIC_QINIT_F_SG,
1325                         .intr_index = cq->bound_intr->index,
1326                         .ring_size = rte_log2_u32(q->num_descs),
1327                         .ring_base = q->base_pa,
1328                         .cq_ring_base = cq->base_pa,
1329                         .sg_ring_base = q->sg_base_pa,
1330                 },
1331         };
1332         int err;
1333
1334         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1335         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1336                 ctx.cmd.q_init.ring_base);
1337         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1338                 ctx.cmd.q_init.ring_size);
1339         IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver);
1340
1341         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1342         if (err)
1343                 return err;
1344
1345         q->hw_type = ctx.comp.q_init.hw_type;
1346         q->hw_index = ctx.comp.q_init.hw_index;
1347         q->db = ionic_db_map(lif, q);
1348
1349         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1350         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1351         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1352
1353         qcq->flags |= IONIC_QCQ_F_INITED;
1354
1355         return 0;
1356 }
1357
1358 int
1359 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1360 {
1361         struct ionic_queue *q = &qcq->q;
1362         struct ionic_lif *lif = qcq->lif;
1363         struct ionic_cq *cq = &qcq->cq;
1364         struct ionic_admin_ctx ctx = {
1365                 .pending_work = true,
1366                 .cmd.q_init = {
1367                         .opcode = IONIC_CMD_Q_INIT,
1368                         .lif_index = lif->index,
1369                         .type = q->type,
1370                         .index = q->index,
1371                         .flags = IONIC_QINIT_F_SG,
1372                         .intr_index = cq->bound_intr->index,
1373                         .ring_size = rte_log2_u32(q->num_descs),
1374                         .ring_base = q->base_pa,
1375                         .cq_ring_base = cq->base_pa,
1376                         .sg_ring_base = q->sg_base_pa,
1377                 },
1378         };
1379         int err;
1380
1381         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1382         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1383                 ctx.cmd.q_init.ring_base);
1384         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1385                 ctx.cmd.q_init.ring_size);
1386         IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver);
1387
1388         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1389         if (err)
1390                 return err;
1391
1392         q->hw_type = ctx.comp.q_init.hw_type;
1393         q->hw_index = ctx.comp.q_init.hw_index;
1394         q->db = ionic_db_map(lif, q);
1395
1396         qcq->flags |= IONIC_QCQ_F_INITED;
1397
1398         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1399         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1400         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1401
1402         return 0;
1403 }
1404
1405 static int
1406 ionic_station_set(struct ionic_lif *lif)
1407 {
1408         struct ionic_admin_ctx ctx = {
1409                 .pending_work = true,
1410                 .cmd.lif_getattr = {
1411                         .opcode = IONIC_CMD_LIF_GETATTR,
1412                         .index = lif->index,
1413                         .attr = IONIC_LIF_ATTR_MAC,
1414                 },
1415         };
1416         int err;
1417
1418         IONIC_PRINT_CALL();
1419
1420         err = ionic_adminq_post_wait(lif, &ctx);
1421         if (err)
1422                 return err;
1423
1424         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1425                         lif->mac_addr)) {
1426                 IONIC_PRINT(INFO, "deleting station MAC addr");
1427
1428                 ionic_lif_addr_del(lif, lif->mac_addr);
1429         }
1430
1431         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1432
1433         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1434                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1435                 return 0;
1436         }
1437
1438         IONIC_PRINT(DEBUG, "adding station MAC addr");
1439
1440         ionic_lif_addr_add(lif, lif->mac_addr);
1441
1442         return 0;
1443 }
1444
1445 static void
1446 ionic_lif_set_name(struct ionic_lif *lif)
1447 {
1448         struct ionic_admin_ctx ctx = {
1449                 .pending_work = true,
1450                 .cmd.lif_setattr = {
1451                         .opcode = IONIC_CMD_LIF_SETATTR,
1452                         .index = lif->index,
1453                         .attr = IONIC_LIF_ATTR_NAME,
1454                 },
1455         };
1456
1457         memcpy(ctx.cmd.lif_setattr.name, lif->name,
1458                 sizeof(ctx.cmd.lif_setattr.name) - 1);
1459
1460         ionic_adminq_post_wait(lif, &ctx);
1461 }
1462
1463 int
1464 ionic_lif_init(struct ionic_lif *lif)
1465 {
1466         struct ionic_dev *idev = &lif->adapter->idev;
1467         struct ionic_q_init_comp comp;
1468         int err;
1469
1470         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1471
1472         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
1473         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1474         ionic_dev_cmd_comp(idev, &comp);
1475         if (err)
1476                 return err;
1477
1478         lif->hw_index = comp.hw_index;
1479
1480         err = ionic_lif_adminq_init(lif);
1481         if (err)
1482                 return err;
1483
1484         err = ionic_lif_notifyq_init(lif);
1485         if (err)
1486                 goto err_out_adminq_deinit;
1487
1488         lif->features =
1489                   IONIC_ETH_HW_VLAN_TX_TAG
1490                 | IONIC_ETH_HW_VLAN_RX_STRIP
1491                 | IONIC_ETH_HW_VLAN_RX_FILTER
1492                 | IONIC_ETH_HW_RX_HASH
1493                 | IONIC_ETH_HW_TX_SG
1494                 | IONIC_ETH_HW_RX_SG
1495                 | IONIC_ETH_HW_TX_CSUM
1496                 | IONIC_ETH_HW_RX_CSUM
1497                 | IONIC_ETH_HW_TSO
1498                 | IONIC_ETH_HW_TSO_IPV6
1499                 | IONIC_ETH_HW_TSO_ECN;
1500
1501         err = ionic_lif_set_features(lif);
1502         if (err)
1503                 goto err_out_notifyq_deinit;
1504
1505         err = ionic_rx_filters_init(lif);
1506         if (err)
1507                 goto err_out_notifyq_deinit;
1508
1509         err = ionic_station_set(lif);
1510         if (err)
1511                 goto err_out_rx_filter_deinit;
1512
1513         ionic_lif_set_name(lif);
1514
1515         lif->state |= IONIC_LIF_F_INITED;
1516
1517         return 0;
1518
1519 err_out_rx_filter_deinit:
1520         ionic_rx_filters_deinit(lif);
1521
1522 err_out_notifyq_deinit:
1523         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1524
1525 err_out_adminq_deinit:
1526         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1527
1528         return err;
1529 }
1530
1531 void
1532 ionic_lif_deinit(struct ionic_lif *lif)
1533 {
1534         if (!(lif->state & IONIC_LIF_F_INITED))
1535                 return;
1536
1537         ionic_rx_filters_deinit(lif);
1538         ionic_lif_rss_teardown(lif);
1539         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1540         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1541
1542         lif->state &= ~IONIC_LIF_F_INITED;
1543 }
1544
1545 int
1546 ionic_lif_configure(struct ionic_lif *lif)
1547 {
1548         struct ionic_identity *ident = &lif->adapter->ident;
1549         uint32_t ntxqs_per_lif =
1550                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1551         uint32_t nrxqs_per_lif =
1552                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1553         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1554         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1555
1556         lif->port_id = lif->eth_dev->data->port_id;
1557
1558         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1559                 lif->port_id);
1560
1561         if (nrxqs > 0)
1562                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1563
1564         if (ntxqs > 0)
1565                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1566
1567         lif->nrxqcqs = nrxqs_per_lif;
1568         lif->ntxqcqs = ntxqs_per_lif;
1569
1570         return 0;
1571 }
1572
1573 int
1574 ionic_lif_start(struct ionic_lif *lif)
1575 {
1576         uint32_t rx_mode;
1577         uint32_t i;
1578         int err;
1579
1580         err = ionic_lif_rss_setup(lif);
1581         if (err)
1582                 return err;
1583
1584         if (!lif->rx_mode) {
1585                 IONIC_PRINT(DEBUG, "Setting RX mode on %s",
1586                         lif->name);
1587
1588                 rx_mode  = IONIC_RX_MODE_F_UNICAST;
1589                 rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1590                 rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1591
1592                 ionic_set_rx_mode(lif, rx_mode);
1593         }
1594
1595         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1596                 "on port %u",
1597                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1598
1599         for (i = 0; i < lif->nrxqcqs; i++) {
1600                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1601                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1602                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1603
1604                         if (err)
1605                                 return err;
1606                 }
1607         }
1608
1609         for (i = 0; i < lif->ntxqcqs; i++) {
1610                 struct ionic_qcq *txq = lif->txqcqs[i];
1611                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1612                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1613
1614                         if (err)
1615                                 return err;
1616                 }
1617         }
1618
1619         ionic_link_status_check(lif);
1620
1621         /* Carrier ON here */
1622
1623         return 0;
1624 }
1625
1626 int
1627 ionic_lif_identify(struct ionic_adapter *adapter)
1628 {
1629         struct ionic_dev *idev = &adapter->idev;
1630         struct ionic_identity *ident = &adapter->ident;
1631         int err;
1632         unsigned int i;
1633         unsigned int lif_words = sizeof(ident->lif.words) /
1634                 sizeof(ident->lif.words[0]);
1635         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1636                 sizeof(idev->dev_cmd->data[0]);
1637         unsigned int nwords;
1638
1639         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1640                 IONIC_IDENTITY_VERSION_1);
1641         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1642         if (err)
1643                 return (err);
1644
1645         nwords = RTE_MIN(lif_words, cmd_words);
1646         for (i = 0; i < nwords; i++)
1647                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1648
1649         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1650                 ident->lif.capabilities);
1651
1652         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1653                 ident->lif.eth.max_ucast_filters);
1654         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1655                 ident->lif.eth.max_mcast_filters);
1656
1657         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1658                 ident->lif.eth.config.features);
1659         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1660                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1661         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1662                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1663         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1664                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1665         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1666                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1667
1668         return 0;
1669 }
1670
1671 int
1672 ionic_lifs_size(struct ionic_adapter *adapter)
1673 {
1674         struct ionic_identity *ident = &adapter->ident;
1675         uint32_t nlifs = ident->dev.nlifs;
1676         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1677
1678         adapter->max_ntxqs_per_lif =
1679                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1680         adapter->max_nrxqs_per_lif =
1681                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1682
1683         nintrs = nlifs * 1 /* notifyq */;
1684
1685         if (nintrs > dev_nintrs) {
1686                 IONIC_PRINT(ERR,
1687                         "At most %d intr supported, minimum req'd is %u",
1688                         dev_nintrs, nintrs);
1689                 return -ENOSPC;
1690         }
1691
1692         adapter->nintrs = nintrs;
1693
1694         return 0;
1695 }