drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13 #include "ionic_rxtx.h"
  14
  15 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  16 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  17
  18 int
  19 ionic_qcq_enable(struct ionic_qcq *qcq)
  20 {
  21         struct ionic_queue *q = &qcq->q;
  22         struct ionic_lif *lif = q->lif;
  23         struct ionic_dev *idev = &lif->adapter->idev;
  24         struct ionic_admin_ctx ctx = {
  25                 .pending_work = true,
  26                 .cmd.q_control = {
  27                         .opcode = IONIC_CMD_Q_CONTROL,
  28                         .lif_index = lif->index,
  29                         .type = q->type,
  30                         .index = q->index,
  31                         .oper = IONIC_Q_ENABLE,
  32                 },
  33         };
  34
  35         if (qcq->flags & IONIC_QCQ_F_INTR) {
  36                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  37                         IONIC_INTR_MASK_CLEAR);
  38         }
  39
  40         return ionic_adminq_post_wait(lif, &ctx);
  41 }
  42
  43 int
  44 ionic_qcq_disable(struct ionic_qcq *qcq)
  45 {
  46         struct ionic_queue *q = &qcq->q;
  47         struct ionic_lif *lif = q->lif;
  48         struct ionic_dev *idev = &lif->adapter->idev;
  49         struct ionic_admin_ctx ctx = {
  50                 .pending_work = true,
  51                 .cmd.q_control = {
  52                         .opcode = IONIC_CMD_Q_CONTROL,
  53                         .lif_index = lif->index,
  54                         .type = q->type,
  55                         .index = q->index,
  56                         .oper = IONIC_Q_DISABLE,
  57                 },
  58         };
  59
  60         if (qcq->flags & IONIC_QCQ_F_INTR) {
  61                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  62                         IONIC_INTR_MASK_SET);
  63         }
  64
  65         return ionic_adminq_post_wait(lif, &ctx);
  66 }
  67
  68 int
  69 ionic_lif_stop(struct ionic_lif *lif __rte_unused)
  70 {
  71         /* Carrier OFF here */
  72
  73         return 0;
  74 }
  75
  76 void
  77 ionic_lif_reset(struct ionic_lif *lif)
  78 {
  79         struct ionic_dev *idev = &lif->adapter->idev;
  80         int err;
  81
  82         IONIC_PRINT_CALL();
  83
  84         ionic_dev_cmd_lif_reset(idev, lif->index);
  85         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  86         if (err)
  87                 IONIC_PRINT(WARNING, "Failed to reset %s", lif->name);
  88 }
  89
  90 static void
  91 ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats)
  92 {
  93         struct ionic_lif_stats *ls = &lif->info->stats;
  94         uint32_t i;
  95         uint32_t num_rx_q_counters = RTE_MIN(lif->nrxqcqs, (uint32_t)
  96                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  97         uint32_t num_tx_q_counters = RTE_MIN(lif->ntxqcqs, (uint32_t)
  98                         RTE_ETHDEV_QUEUE_STAT_CNTRS);
  99
 100         memset(stats, 0, sizeof(*stats));
 101
 102         if (ls == NULL) {
 103                 IONIC_PRINT(DEBUG, "Stats on port %u not yet initialized",
 104                         lif->port_id);
 105                 return;
 106         }
 107
 108         /* RX */
 109
 110         stats->ipackets = ls->rx_ucast_packets +
 111                 ls->rx_mcast_packets +
 112                 ls->rx_bcast_packets;
 113
 114         stats->ibytes = ls->rx_ucast_bytes +
 115                 ls->rx_mcast_bytes +
 116                 ls->rx_bcast_bytes;
 117
 118         for (i = 0; i < lif->nrxqcqs; i++) {
 119                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 120                 stats->imissed +=
 121                         rx_stats->no_cb_arg +
 122                         rx_stats->bad_cq_status +
 123                         rx_stats->no_room +
 124                         rx_stats->bad_len;
 125         }
 126
 127         stats->imissed +=
 128                 ls->rx_ucast_drop_packets +
 129                 ls->rx_mcast_drop_packets +
 130                 ls->rx_bcast_drop_packets;
 131
 132         stats->imissed +=
 133                 ls->rx_queue_empty +
 134                 ls->rx_dma_error +
 135                 ls->rx_queue_disabled +
 136                 ls->rx_desc_fetch_error +
 137                 ls->rx_desc_data_error;
 138
 139         for (i = 0; i < num_rx_q_counters; i++) {
 140                 struct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats.rx;
 141                 stats->q_ipackets[i] = rx_stats->packets;
 142                 stats->q_ibytes[i] = rx_stats->bytes;
 143                 stats->q_errors[i] =
 144                         rx_stats->no_cb_arg +
 145                         rx_stats->bad_cq_status +
 146                         rx_stats->no_room +
 147                         rx_stats->bad_len;
 148         }
 149
 150         /* TX */
 151
 152         stats->opackets = ls->tx_ucast_packets +
 153                 ls->tx_mcast_packets +
 154                 ls->tx_bcast_packets;
 155
 156         stats->obytes = ls->tx_ucast_bytes +
 157                 ls->tx_mcast_bytes +
 158                 ls->tx_bcast_bytes;
 159
 160         for (i = 0; i < lif->ntxqcqs; i++) {
 161                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 162                 stats->oerrors += tx_stats->drop;
 163         }
 164
 165         stats->oerrors +=
 166                 ls->tx_ucast_drop_packets +
 167                 ls->tx_mcast_drop_packets +
 168                 ls->tx_bcast_drop_packets;
 169
 170         stats->oerrors +=
 171                 ls->tx_dma_error +
 172                 ls->tx_queue_disabled +
 173                 ls->tx_desc_fetch_error +
 174                 ls->tx_desc_data_error;
 175
 176         for (i = 0; i < num_tx_q_counters; i++) {
 177                 struct ionic_tx_stats *tx_stats = &lif->txqcqs[i]->stats.tx;
 178                 stats->q_opackets[i] = tx_stats->packets;
 179                 stats->q_obytes[i] = tx_stats->bytes;
 180         }
 181 }
 182
 183 void
 184 ionic_lif_get_stats(const struct ionic_lif *lif,
 185                 struct rte_eth_stats *stats)
 186 {
 187         ionic_lif_get_abs_stats(lif, stats);
 188
 189         stats->ipackets  -= lif->stats_base.ipackets;
 190         stats->opackets  -= lif->stats_base.opackets;
 191         stats->ibytes    -= lif->stats_base.ibytes;
 192         stats->obytes    -= lif->stats_base.obytes;
 193         stats->imissed   -= lif->stats_base.imissed;
 194         stats->ierrors   -= lif->stats_base.ierrors;
 195         stats->oerrors   -= lif->stats_base.oerrors;
 196         stats->rx_nombuf -= lif->stats_base.rx_nombuf;
 197 }
 198
 199 void
 200 ionic_lif_reset_stats(struct ionic_lif *lif)
 201 {
 202         uint32_t i;
 203
 204         for (i = 0; i < lif->nrxqcqs; i++) {
 205                 memset(&lif->rxqcqs[i]->stats.rx, 0,
 206                         sizeof(struct ionic_rx_stats));
 207                 memset(&lif->txqcqs[i]->stats.tx, 0,
 208                         sizeof(struct ionic_tx_stats));
 209         }
 210
 211         ionic_lif_get_abs_stats(lif, &lif->stats_base);
 212 }
 213
 214 void
 215 ionic_lif_get_hw_stats(struct ionic_lif *lif, struct ionic_lif_stats *stats)
 216 {
 217         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 218         uint64_t *stats64 = (uint64_t *)stats;
 219         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 220         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 221
 222         for (i = 0; i < count; i++)
 223                 stats64[i] = lif_stats64[i] - lif_stats64_base[i];
 224 }
 225
 226 void
 227 ionic_lif_reset_hw_stats(struct ionic_lif *lif)
 228 {
 229         uint16_t i, count = sizeof(struct ionic_lif_stats) / sizeof(uint64_t);
 230         uint64_t *lif_stats64 = (uint64_t *)&lif->info->stats;
 231         uint64_t *lif_stats64_base = (uint64_t *)&lif->lif_stats_base;
 232
 233         for (i = 0; i < count; i++)
 234                 lif_stats64_base[i] = lif_stats64[i];
 235 }
 236
 237 static int
 238 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
 239 {
 240         struct ionic_admin_ctx ctx = {
 241                 .pending_work = true,
 242                 .cmd.rx_filter_add = {
 243                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 244                         .match = IONIC_RX_FILTER_MATCH_MAC,
 245                 },
 246         };
 247         int err;
 248
 249         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
 250
 251         err = ionic_adminq_post_wait(lif, &ctx);
 252         if (err)
 253                 return err;
 254
 255         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 256                 ctx.comp.rx_filter_add.filter_id);
 257
 258         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 259 }
 260
 261 static int
 262 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 263 {
 264         struct ionic_admin_ctx ctx = {
 265                 .pending_work = true,
 266                 .cmd.rx_filter_del = {
 267                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 268                 },
 269         };
 270         struct ionic_rx_filter *f;
 271         int err;
 272
 273         IONIC_PRINT_CALL();
 274
 275         rte_spinlock_lock(&lif->rx_filters.lock);
 276
 277         f = ionic_rx_filter_by_addr(lif, addr);
 278         if (!f) {
 279                 rte_spinlock_unlock(&lif->rx_filters.lock);
 280                 return -ENOENT;
 281         }
 282
 283         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 284         ionic_rx_filter_free(f);
 285
 286         rte_spinlock_unlock(&lif->rx_filters.lock);
 287
 288         err = ionic_adminq_post_wait(lif, &ctx);
 289         if (err)
 290                 return err;
 291
 292         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 293                 ctx.cmd.rx_filter_del.filter_id);
 294
 295         return 0;
 296 }
 297
 298 int
 299 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 300                 struct rte_ether_addr *mac_addr,
 301                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 302 {
 303         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 304
 305         IONIC_PRINT_CALL();
 306
 307         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 308 }
 309
 310 void
 311 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index)
 312 {
 313         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 314         struct ionic_adapter *adapter = lif->adapter;
 315         struct rte_ether_addr *mac_addr;
 316
 317         IONIC_PRINT_CALL();
 318
 319         if (index >= adapter->max_mac_addrs) {
 320                 IONIC_PRINT(WARNING,
 321                         "Index %u is above MAC filter limit %u",
 322                         index, adapter->max_mac_addrs);
 323                 return;
 324         }
 325
 326         mac_addr = &eth_dev->data->mac_addrs[index];
 327
 328         if (!rte_is_valid_assigned_ether_addr(mac_addr))
 329                 return;
 330
 331         ionic_lif_addr_del(lif, (const uint8_t *)mac_addr);
 332 }
 333
 334 int
 335 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 336 {
 337         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 338
 339         IONIC_PRINT_CALL();
 340
 341         if (mac_addr == NULL) {
 342                 IONIC_PRINT(NOTICE, "New mac is null");
 343                 return -1;
 344         }
 345
 346         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 347                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 348                         lif->mac_addr);
 349                 ionic_lif_addr_del(lif, lif->mac_addr);
 350                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 351         }
 352
 353         IONIC_PRINT(INFO, "Updating mac addr");
 354
 355         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 356
 357         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 358 }
 359
 360 static int
 361 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 362 {
 363         struct ionic_admin_ctx ctx = {
 364                 .pending_work = true,
 365                 .cmd.rx_filter_add = {
 366                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 367                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 368                         .vlan.vlan = vid,
 369                 },
 370         };
 371         int err;
 372
 373         err = ionic_adminq_post_wait(lif, &ctx);
 374         if (err)
 375                 return err;
 376
 377         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 378                 ctx.comp.rx_filter_add.filter_id);
 379
 380         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 381 }
 382
 383 static int
 384 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 385 {
 386         struct ionic_admin_ctx ctx = {
 387                 .pending_work = true,
 388                 .cmd.rx_filter_del = {
 389                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 390                 },
 391         };
 392         struct ionic_rx_filter *f;
 393         int err;
 394
 395         IONIC_PRINT_CALL();
 396
 397         rte_spinlock_lock(&lif->rx_filters.lock);
 398
 399         f = ionic_rx_filter_by_vlan(lif, vid);
 400         if (!f) {
 401                 rte_spinlock_unlock(&lif->rx_filters.lock);
 402                 return -ENOENT;
 403         }
 404
 405         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 406         ionic_rx_filter_free(f);
 407         rte_spinlock_unlock(&lif->rx_filters.lock);
 408
 409         err = ionic_adminq_post_wait(lif, &ctx);
 410         if (err)
 411                 return err;
 412
 413         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 414                 ctx.cmd.rx_filter_del.filter_id);
 415
 416         return 0;
 417 }
 418
 419 int
 420 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 421                 int on)
 422 {
 423         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 424         int err;
 425
 426         if (on)
 427                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 428         else
 429                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 430
 431         return err;
 432 }
 433
 434 static void
 435 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 436 {
 437         struct ionic_admin_ctx ctx = {
 438                 .pending_work = true,
 439                 .cmd.rx_mode_set = {
 440                         .opcode = IONIC_CMD_RX_MODE_SET,
 441                         .lif_index = lif->index,
 442                         .rx_mode = rx_mode,
 443                 },
 444         };
 445         int err;
 446
 447         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 448                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 449         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 450                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 451         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 452                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 453         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 454                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 455         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 456                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 457
 458         err = ionic_adminq_post_wait(lif, &ctx);
 459         if (err)
 460                 IONIC_PRINT(ERR, "Failure setting RX mode");
 461 }
 462
 463 static void
 464 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 465 {
 466         if (lif->rx_mode != rx_mode) {
 467                 lif->rx_mode = rx_mode;
 468                 ionic_lif_rx_mode(lif, rx_mode);
 469         }
 470 }
 471
 472 int
 473 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 474 {
 475         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 476         uint32_t rx_mode = lif->rx_mode;
 477
 478         IONIC_PRINT_CALL();
 479
 480         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 481
 482         ionic_set_rx_mode(lif, rx_mode);
 483
 484         return 0;
 485 }
 486
 487 int
 488 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 489 {
 490         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 491         uint32_t rx_mode = lif->rx_mode;
 492
 493         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 494
 495         ionic_set_rx_mode(lif, rx_mode);
 496
 497         return 0;
 498 }
 499
 500 int
 501 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 502 {
 503         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 504         uint32_t rx_mode = lif->rx_mode;
 505
 506         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 507
 508         ionic_set_rx_mode(lif, rx_mode);
 509
 510         return 0;
 511 }
 512
 513 int
 514 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 515 {
 516         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 517         uint32_t rx_mode = lif->rx_mode;
 518
 519         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 520
 521         ionic_set_rx_mode(lif, rx_mode);
 522
 523         return 0;
 524 }
 525
 526 int
 527 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 528 {
 529         struct ionic_admin_ctx ctx = {
 530                 .pending_work = true,
 531                 .cmd.lif_setattr = {
 532                         .opcode = IONIC_CMD_LIF_SETATTR,
 533                         .index = lif->index,
 534                         .attr = IONIC_LIF_ATTR_MTU,
 535                         .mtu = new_mtu,
 536                 },
 537         };
 538         int err;
 539
 540         err = ionic_adminq_post_wait(lif, &ctx);
 541         if (err)
 542                 return err;
 543
 544         return 0;
 545 }
 546
 547 int
 548 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 549 {
 550         struct ionic_adapter *adapter = lif->adapter;
 551         struct ionic_dev *idev = &adapter->idev;
 552         unsigned long index;
 553
 554         /*
 555          * Note: interrupt handler is called for index = 0 only
 556          * (we use interrupts for the notifyq only anyway,
 557          * which has index = 0)
 558          */
 559
 560         for (index = 0; index < adapter->nintrs; index++)
 561                 if (!adapter->intrs[index])
 562                         break;
 563
 564         if (index == adapter->nintrs)
 565                 return -ENOSPC;
 566
 567         adapter->intrs[index] = true;
 568
 569         ionic_intr_init(idev, intr, index);
 570
 571         return 0;
 572 }
 573
 574 void
 575 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 576 {
 577         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 578                 lif->adapter->intrs[intr->index] = false;
 579 }
 580
 581 static int
 582 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 583                 uint32_t index,
 584                 const char *base, uint32_t flags,
 585                 uint32_t num_descs,
 586                 uint32_t desc_size,
 587                 uint32_t cq_desc_size,
 588                 uint32_t sg_desc_size,
 589                 struct ionic_qcq **qcq)
 590 {
 591         struct ionic_dev *idev = &lif->adapter->idev;
 592         struct ionic_qcq *new;
 593         uint32_t q_size, cq_size, sg_size, total_size;
 594         void *q_base, *cq_base, *sg_base;
 595         rte_iova_t q_base_pa = 0;
 596         rte_iova_t cq_base_pa = 0;
 597         rte_iova_t sg_base_pa = 0;
 598         uint32_t socket_id = rte_socket_id();
 599         int err;
 600
 601         *qcq = NULL;
 602
 603         q_size  = num_descs * desc_size;
 604         cq_size = num_descs * cq_desc_size;
 605         sg_size = num_descs * sg_desc_size;
 606
 607         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 608                 RTE_ALIGN(cq_size, PAGE_SIZE);
 609         /*
 610          * Note: aligning q_size/cq_size is not enough due to cq_base address
 611          * aligning as q_base could be not aligned to the page.
 612          * Adding PAGE_SIZE.
 613          */
 614         total_size += PAGE_SIZE;
 615
 616         if (flags & IONIC_QCQ_F_SG) {
 617                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 618                 total_size += PAGE_SIZE;
 619         }
 620
 621         new = rte_zmalloc("ionic", sizeof(*new), 0);
 622         if (!new) {
 623                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 624                 return -ENOMEM;
 625         }
 626
 627         new->lif = lif;
 628         new->flags = flags;
 629
 630         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 631         if (!new->q.info) {
 632                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 633                 return -ENOMEM;
 634         }
 635
 636         new->q.type = type;
 637
 638         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 639                 desc_size, sg_desc_size);
 640         if (err) {
 641                 IONIC_PRINT(ERR, "Queue initialization failed");
 642                 return err;
 643         }
 644
 645         if (flags & IONIC_QCQ_F_INTR) {
 646                 err = ionic_intr_alloc(lif, &new->intr);
 647                 if (err)
 648                         return err;
 649
 650                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 651                         IONIC_INTR_MASK_SET);
 652         } else {
 653                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 654         }
 655
 656         err = ionic_cq_init(lif, &new->cq, &new->intr,
 657                 num_descs, cq_desc_size);
 658         if (err) {
 659                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 660                 goto err_out_free_intr;
 661         }
 662
 663         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 664                 base /* name */, index /* queue_idx */,
 665                 total_size, IONIC_ALIGN, socket_id);
 666
 667         if (!new->base_z) {
 668                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 669                 err = -ENOMEM;
 670                 goto err_out_free_intr;
 671         }
 672
 673         new->base = new->base_z->addr;
 674         new->base_pa = new->base_z->iova;
 675         new->total_size = total_size;
 676
 677         q_base = new->base;
 678         q_base_pa = new->base_pa;
 679
 680         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 681         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 682
 683         if (flags & IONIC_QCQ_F_SG) {
 684                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 685                         PAGE_SIZE);
 686                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 687                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 688         }
 689
 690         IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 691                 "SG-base-PA = %#jx",
 692                 q_base_pa, cq_base_pa, sg_base_pa);
 693
 694         ionic_q_map(&new->q, q_base, q_base_pa);
 695         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 696         ionic_cq_bind(&new->cq, &new->q);
 697
 698         *qcq = new;
 699
 700         return 0;
 701
 702 err_out_free_intr:
 703         if (flags & IONIC_QCQ_F_INTR)
 704                 ionic_intr_free(lif, &new->intr);
 705
 706         return err;
 707 }
 708
 709 void
 710 ionic_qcq_free(struct ionic_qcq *qcq)
 711 {
 712         if (qcq->base_z) {
 713                 qcq->base = NULL;
 714                 qcq->base_pa = 0;
 715                 rte_memzone_free(qcq->base_z);
 716                 qcq->base_z = NULL;
 717         }
 718
 719         if (qcq->q.info) {
 720                 rte_free(qcq->q.info);
 721                 qcq->q.info = NULL;
 722         }
 723
 724         rte_free(qcq);
 725 }
 726
 727 int
 728 ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t nrxq_descs,
 729                 struct ionic_qcq **qcq)
 730 {
 731         uint32_t flags;
 732         int err = -ENOMEM;
 733
 734         flags = IONIC_QCQ_F_SG;
 735         err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, index, "rx", flags,
 736                 nrxq_descs,
 737                 sizeof(struct ionic_rxq_desc),
 738                 sizeof(struct ionic_rxq_comp),
 739                 sizeof(struct ionic_rxq_sg_desc),
 740                 &lif->rxqcqs[index]);
 741         if (err)
 742                 return err;
 743
 744         *qcq = lif->rxqcqs[index];
 745
 746         return 0;
 747 }
 748
 749 int
 750 ionic_tx_qcq_alloc(struct ionic_lif *lif, uint32_t index, uint16_t ntxq_descs,
 751                 struct ionic_qcq **qcq)
 752 {
 753         uint32_t flags;
 754         int err = -ENOMEM;
 755
 756         flags = IONIC_QCQ_F_SG;
 757         err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, index, "tx", flags,
 758                 ntxq_descs,
 759                 sizeof(struct ionic_txq_desc),
 760                 sizeof(struct ionic_txq_comp),
 761                 sizeof(struct ionic_txq_sg_desc),
 762                 &lif->txqcqs[index]);
 763         if (err)
 764                 return err;
 765
 766         *qcq = lif->txqcqs[index];
 767
 768         return 0;
 769 }
 770
 771 static int
 772 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 773 {
 774         uint32_t flags;
 775         int err = -ENOMEM;
 776
 777         flags = 0;
 778         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 779                 IONIC_ADMINQ_LENGTH,
 780                 sizeof(struct ionic_admin_cmd),
 781                 sizeof(struct ionic_admin_comp),
 782                 0,
 783                 &lif->adminqcq);
 784         if (err)
 785                 return err;
 786
 787         return 0;
 788 }
 789
 790 static int
 791 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 792 {
 793         uint32_t flags;
 794         int err = -ENOMEM;
 795
 796         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 797
 798         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 799                 flags,
 800                 IONIC_NOTIFYQ_LENGTH,
 801                 sizeof(struct ionic_notifyq_cmd),
 802                 sizeof(union ionic_notifyq_comp),
 803                 0,
 804                 &lif->notifyqcq);
 805         if (err)
 806                 return err;
 807
 808         return 0;
 809 }
 810
 811 static void *
 812 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 813 {
 814         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 815
 816         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 817                 return NULL;
 818
 819         return (void *)&vaddr[page_num << PAGE_SHIFT];
 820 }
 821
 822 int
 823 ionic_lif_alloc(struct ionic_lif *lif)
 824 {
 825         struct ionic_adapter *adapter = lif->adapter;
 826         uint32_t socket_id = rte_socket_id();
 827         int dbpage_num;
 828         int err;
 829
 830         /*
 831          * lif->name was zeroed on allocation.
 832          * Copy (sizeof() - 1) bytes to ensure that it is NULL terminated.
 833          */
 834         memcpy(lif->name, lif->eth_dev->data->name, sizeof(lif->name) - 1);
 835
 836         IONIC_PRINT(DEBUG, "LIF: %s", lif->name);
 837
 838         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 839
 840         rte_spinlock_init(&lif->adminq_lock);
 841         rte_spinlock_init(&lif->adminq_service_lock);
 842
 843         dbpage_num = ionic_db_page_num(lif, 0);
 844
 845         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, dbpage_num);
 846         if (!lif->kern_dbpage) {
 847                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 848                 return -ENOMEM;
 849         }
 850
 851         lif->txqcqs = rte_zmalloc("ionic", sizeof(*lif->txqcqs) *
 852                 adapter->max_ntxqs_per_lif, 0);
 853
 854         if (!lif->txqcqs) {
 855                 IONIC_PRINT(ERR, "Cannot allocate tx queues array");
 856                 return -ENOMEM;
 857         }
 858
 859         lif->rxqcqs = rte_zmalloc("ionic", sizeof(*lif->rxqcqs) *
 860                 adapter->max_nrxqs_per_lif, 0);
 861
 862         if (!lif->rxqcqs) {
 863                 IONIC_PRINT(ERR, "Cannot allocate rx queues array");
 864                 return -ENOMEM;
 865         }
 866
 867         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 868
 869         err = ionic_notify_qcq_alloc(lif);
 870         if (err) {
 871                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 872                 return err;
 873         }
 874
 875         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 876
 877         err = ionic_admin_qcq_alloc(lif);
 878         if (err) {
 879                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 880                 return err;
 881         }
 882
 883         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 884
 885         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 886
 887         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 888                 "lif_info", 0 /* queue_idx*/,
 889                 lif->info_sz, IONIC_ALIGN, socket_id);
 890         if (!lif->info_z) {
 891                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 892                 return -ENOMEM;
 893         }
 894
 895         lif->info = lif->info_z->addr;
 896         lif->info_pa = lif->info_z->iova;
 897
 898         return 0;
 899 }
 900
 901 void
 902 ionic_lif_free(struct ionic_lif *lif)
 903 {
 904         if (lif->notifyqcq) {
 905                 ionic_qcq_free(lif->notifyqcq);
 906                 lif->notifyqcq = NULL;
 907         }
 908
 909         if (lif->adminqcq) {
 910                 ionic_qcq_free(lif->adminqcq);
 911                 lif->adminqcq = NULL;
 912         }
 913
 914         if (lif->txqcqs) {
 915                 rte_free(lif->txqcqs);
 916                 lif->txqcqs = NULL;
 917         }
 918
 919         if (lif->rxqcqs) {
 920                 rte_free(lif->rxqcqs);
 921                 lif->rxqcqs = NULL;
 922         }
 923
 924         if (lif->info) {
 925                 rte_memzone_free(lif->info_z);
 926                 lif->info = NULL;
 927         }
 928 }
 929
 930 int
 931 ionic_lif_rss_config(struct ionic_lif *lif,
 932                 const uint16_t types, const uint8_t *key, const uint32_t *indir)
 933 {
 934         struct ionic_admin_ctx ctx = {
 935                 .pending_work = true,
 936                 .cmd.lif_setattr = {
 937                         .opcode = IONIC_CMD_LIF_SETATTR,
 938                         .attr = IONIC_LIF_ATTR_RSS,
 939                         .rss.types = types,
 940                         .rss.addr = lif->rss_ind_tbl_pa,
 941                 },
 942         };
 943         unsigned int i;
 944
 945         IONIC_PRINT_CALL();
 946
 947         lif->rss_types = types;
 948
 949         if (key)
 950                 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
 951
 952         if (indir)
 953                 for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 954                         lif->rss_ind_tbl[i] = indir[i];
 955
 956         memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
 957                IONIC_RSS_HASH_KEY_SIZE);
 958
 959         return ionic_adminq_post_wait(lif, &ctx);
 960 }
 961
 962 static int
 963 ionic_lif_rss_setup(struct ionic_lif *lif)
 964 {
 965         size_t tbl_size = sizeof(*lif->rss_ind_tbl) *
 966                 lif->adapter->ident.lif.eth.rss_ind_tbl_sz;
 967         static const uint8_t toeplitz_symmetric_key[] = {
 968                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 969                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 970                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 971                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 972                 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A, 0x6D, 0x5A,
 973         };
 974         uint32_t socket_id = rte_socket_id();
 975         uint32_t i;
 976         int err;
 977
 978         IONIC_PRINT_CALL();
 979
 980         lif->rss_ind_tbl_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 981                 "rss_ind_tbl",
 982                 0 /* queue_idx*/, tbl_size, IONIC_ALIGN, socket_id);
 983
 984         if (!lif->rss_ind_tbl_z) {
 985                 IONIC_PRINT(ERR, "OOM");
 986                 return -ENOMEM;
 987         }
 988
 989         lif->rss_ind_tbl = lif->rss_ind_tbl_z->addr;
 990         lif->rss_ind_tbl_pa = lif->rss_ind_tbl_z->iova;
 991
 992         /* Fill indirection table with 'default' values */
 993         for (i = 0; i < lif->adapter->ident.lif.eth.rss_ind_tbl_sz; i++)
 994                 lif->rss_ind_tbl[i] = i % lif->nrxqcqs;
 995
 996         err = ionic_lif_rss_config(lif, IONIC_RSS_OFFLOAD_ALL,
 997                 toeplitz_symmetric_key, NULL);
 998         if (err)
 999                 return err;
1000
1001         return 0;
1002 }
1003
1004 static void
1005 ionic_lif_rss_teardown(struct ionic_lif *lif)
1006 {
1007         if (!lif->rss_ind_tbl)
1008                 return;
1009
1010         if (lif->rss_ind_tbl_z) {
1011                 /* Disable RSS on the NIC */
1012                 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1013
1014                 lif->rss_ind_tbl = NULL;
1015                 lif->rss_ind_tbl_pa = 0;
1016                 rte_memzone_free(lif->rss_ind_tbl_z);
1017                 lif->rss_ind_tbl_z = NULL;
1018         }
1019 }
1020
1021 static void
1022 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
1023 {
1024         struct ionic_dev *idev = &lif->adapter->idev;
1025
1026         if (!(qcq->flags & IONIC_QCQ_F_INITED))
1027                 return;
1028
1029         if (qcq->flags & IONIC_QCQ_F_INTR)
1030                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1031                         IONIC_INTR_MASK_SET);
1032
1033         qcq->flags &= ~IONIC_QCQ_F_INITED;
1034 }
1035
1036 void
1037 ionic_lif_txq_deinit(struct ionic_qcq *qcq)
1038 {
1039         ionic_lif_qcq_deinit(qcq->lif, qcq);
1040 }
1041
1042 void
1043 ionic_lif_rxq_deinit(struct ionic_qcq *qcq)
1044 {
1045         ionic_lif_qcq_deinit(qcq->lif, qcq);
1046 }
1047
1048 bool
1049 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
1050                 void *cb_arg __rte_unused)
1051 {
1052         struct ionic_admin_comp *cq_desc_base = cq->base;
1053         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
1054
1055         if (!color_match(cq_desc->color, cq->done_color))
1056                 return false;
1057
1058         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
1059
1060         return true;
1061 }
1062
1063 /* This acts like ionic_napi */
1064 int
1065 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
1066                 void *cb_arg)
1067 {
1068         struct ionic_cq *cq = &qcq->cq;
1069         uint32_t work_done;
1070
1071         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
1072
1073         return work_done;
1074 }
1075
1076 static void
1077 ionic_link_status_check(struct ionic_lif *lif)
1078 {
1079         struct ionic_adapter *adapter = lif->adapter;
1080         bool link_up;
1081
1082         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
1083
1084         if (!lif->info)
1085                 return;
1086
1087         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
1088
1089         if ((link_up  && adapter->link_up) ||
1090             (!link_up && !adapter->link_up))
1091                 return;
1092
1093         if (link_up) {
1094                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
1095                         lif->info->status.link_speed);
1096                 adapter->link_speed = lif->info->status.link_speed;
1097         } else {
1098                 IONIC_PRINT(DEBUG, "Link down");
1099         }
1100
1101         adapter->link_up = link_up;
1102 }
1103
1104 static bool
1105 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
1106 {
1107         union ionic_notifyq_comp *cq_desc_base = cq->base;
1108         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
1109         struct ionic_lif *lif = cb_arg;
1110
1111         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
1112                 cq_desc->event.eid, cq_desc->event.ecode);
1113
1114         /* Have we run out of new completions to process? */
1115         if (!(cq_desc->event.eid > lif->last_eid))
1116                 return false;
1117
1118         lif->last_eid = cq_desc->event.eid;
1119
1120         switch (cq_desc->event.ecode) {
1121         case IONIC_EVENT_LINK_CHANGE:
1122                 IONIC_PRINT(DEBUG,
1123                         "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
1124                         cq_desc->event.eid,
1125                         cq_desc->link_change.link_status,
1126                         cq_desc->link_change.link_speed);
1127
1128                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
1129
1130                 break;
1131         default:
1132                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
1133                         cq_desc->event.ecode, cq_desc->event.eid);
1134                 break;
1135         }
1136
1137         return true;
1138 }
1139
1140 int
1141 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
1142 {
1143         struct ionic_dev *idev = &lif->adapter->idev;
1144         struct ionic_qcq *qcq = lif->notifyqcq;
1145         uint32_t work_done;
1146
1147         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
1148                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
1149                 return -1;
1150         }
1151
1152         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1153                 IONIC_INTR_MASK_SET);
1154
1155         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
1156
1157         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
1158                 ionic_link_status_check(lif);
1159
1160         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
1161                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
1162
1163         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1164                 IONIC_INTR_MASK_CLEAR);
1165
1166         return 0;
1167 }
1168
1169 static int
1170 ionic_lif_adminq_init(struct ionic_lif *lif)
1171 {
1172         struct ionic_dev *idev = &lif->adapter->idev;
1173         struct ionic_qcq *qcq = lif->adminqcq;
1174         struct ionic_queue *q = &qcq->q;
1175         struct ionic_q_init_comp comp;
1176         int err;
1177
1178         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
1179         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1180         if (err)
1181                 return err;
1182
1183         ionic_dev_cmd_comp(idev, &comp);
1184
1185         q->hw_type = comp.hw_type;
1186         q->hw_index = comp.hw_index;
1187         q->db = ionic_db_map(lif, q);
1188
1189         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
1190         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
1191         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
1192
1193         if (qcq->flags & IONIC_QCQ_F_INTR)
1194                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1195                         IONIC_INTR_MASK_CLEAR);
1196
1197         qcq->flags |= IONIC_QCQ_F_INITED;
1198
1199         return 0;
1200 }
1201
1202 static int
1203 ionic_lif_notifyq_init(struct ionic_lif *lif)
1204 {
1205         struct ionic_dev *idev = &lif->adapter->idev;
1206         struct ionic_qcq *qcq = lif->notifyqcq;
1207         struct ionic_queue *q = &qcq->q;
1208         int err;
1209
1210         struct ionic_admin_ctx ctx = {
1211                 .pending_work = true,
1212                 .cmd.q_init = {
1213                         .opcode = IONIC_CMD_Q_INIT,
1214                         .lif_index = lif->index,
1215                         .type = q->type,
1216                         .index = q->index,
1217                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
1218                         .intr_index = qcq->intr.index,
1219                         .ring_size = rte_log2_u32(q->num_descs),
1220                         .ring_base = q->base_pa,
1221                 }
1222         };
1223
1224         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
1225                 ctx.cmd.q_init.index);
1226         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
1227                 ctx.cmd.q_init.ring_base);
1228         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
1229                 ctx.cmd.q_init.ring_size);
1230         IONIC_PRINT(DEBUG, "notifyq_init.ver %u", ctx.cmd.q_init.ver);
1231
1232         err = ionic_adminq_post_wait(lif, &ctx);
1233         if (err)
1234                 return err;
1235
1236         q->hw_type = ctx.comp.q_init.hw_type;
1237         q->hw_index = ctx.comp.q_init.hw_index;
1238         q->db = NULL;
1239
1240         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
1241         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
1242         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
1243
1244         if (qcq->flags & IONIC_QCQ_F_INTR)
1245                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
1246                         IONIC_INTR_MASK_CLEAR);
1247
1248         qcq->flags |= IONIC_QCQ_F_INITED;
1249
1250         return 0;
1251 }
1252
1253 int
1254 ionic_lif_set_features(struct ionic_lif *lif)
1255 {
1256         struct ionic_admin_ctx ctx = {
1257                 .pending_work = true,
1258                 .cmd.lif_setattr = {
1259                         .opcode = IONIC_CMD_LIF_SETATTR,
1260                         .index = lif->index,
1261                         .attr = IONIC_LIF_ATTR_FEATURES,
1262                         .features = lif->features,
1263                 },
1264         };
1265         int err;
1266
1267         err = ionic_adminq_post_wait(lif, &ctx);
1268         if (err)
1269                 return err;
1270
1271         lif->hw_features = (ctx.cmd.lif_setattr.features &
1272                 ctx.comp.lif_setattr.features);
1273
1274         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1275                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
1276         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1277                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
1278         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1279                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
1280         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1281                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
1282         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1283                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
1284         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
1285                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
1286         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1287                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
1288         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1289                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
1290         if (lif->hw_features & IONIC_ETH_HW_TSO)
1291                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
1292         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1293                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
1294         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1295                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
1296         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1297                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
1298         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1299                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
1300         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1301                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
1302         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1303                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
1304         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1305                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
1306         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1307                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
1308
1309         return 0;
1310 }
1311
1312 int
1313 ionic_lif_txq_init(struct ionic_qcq *qcq)
1314 {
1315         struct ionic_queue *q = &qcq->q;
1316         struct ionic_lif *lif = qcq->lif;
1317         struct ionic_cq *cq = &qcq->cq;
1318         struct ionic_admin_ctx ctx = {
1319                 .pending_work = true,
1320                 .cmd.q_init = {
1321                         .opcode = IONIC_CMD_Q_INIT,
1322                         .lif_index = lif->index,
1323                         .type = q->type,
1324                         .index = q->index,
1325                         .flags = IONIC_QINIT_F_SG,
1326                         .intr_index = cq->bound_intr->index,
1327                         .ring_size = rte_log2_u32(q->num_descs),
1328                         .ring_base = q->base_pa,
1329                         .cq_ring_base = cq->base_pa,
1330                         .sg_ring_base = q->sg_base_pa,
1331                 },
1332         };
1333         int err;
1334
1335         IONIC_PRINT(DEBUG, "txq_init.index %d", ctx.cmd.q_init.index);
1336         IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "",
1337                 ctx.cmd.q_init.ring_base);
1338         IONIC_PRINT(DEBUG, "txq_init.ring_size %d",
1339                 ctx.cmd.q_init.ring_size);
1340         IONIC_PRINT(DEBUG, "txq_init.ver %u", ctx.cmd.q_init.ver);
1341
1342         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1343         if (err)
1344                 return err;
1345
1346         q->hw_type = ctx.comp.q_init.hw_type;
1347         q->hw_index = ctx.comp.q_init.hw_index;
1348         q->db = ionic_db_map(lif, q);
1349
1350         IONIC_PRINT(DEBUG, "txq->hw_type %d", q->hw_type);
1351         IONIC_PRINT(DEBUG, "txq->hw_index %d", q->hw_index);
1352         IONIC_PRINT(DEBUG, "txq->db %p", q->db);
1353
1354         qcq->flags |= IONIC_QCQ_F_INITED;
1355
1356         return 0;
1357 }
1358
1359 int
1360 ionic_lif_rxq_init(struct ionic_qcq *qcq)
1361 {
1362         struct ionic_queue *q = &qcq->q;
1363         struct ionic_lif *lif = qcq->lif;
1364         struct ionic_cq *cq = &qcq->cq;
1365         struct ionic_admin_ctx ctx = {
1366                 .pending_work = true,
1367                 .cmd.q_init = {
1368                         .opcode = IONIC_CMD_Q_INIT,
1369                         .lif_index = lif->index,
1370                         .type = q->type,
1371                         .index = q->index,
1372                         .flags = IONIC_QINIT_F_SG,
1373                         .intr_index = cq->bound_intr->index,
1374                         .ring_size = rte_log2_u32(q->num_descs),
1375                         .ring_base = q->base_pa,
1376                         .cq_ring_base = cq->base_pa,
1377                         .sg_ring_base = q->sg_base_pa,
1378                 },
1379         };
1380         int err;
1381
1382         IONIC_PRINT(DEBUG, "rxq_init.index %d", ctx.cmd.q_init.index);
1383         IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "",
1384                 ctx.cmd.q_init.ring_base);
1385         IONIC_PRINT(DEBUG, "rxq_init.ring_size %d",
1386                 ctx.cmd.q_init.ring_size);
1387         IONIC_PRINT(DEBUG, "rxq_init.ver %u", ctx.cmd.q_init.ver);
1388
1389         err = ionic_adminq_post_wait(qcq->lif, &ctx);
1390         if (err)
1391                 return err;
1392
1393         q->hw_type = ctx.comp.q_init.hw_type;
1394         q->hw_index = ctx.comp.q_init.hw_index;
1395         q->db = ionic_db_map(lif, q);
1396
1397         qcq->flags |= IONIC_QCQ_F_INITED;
1398
1399         IONIC_PRINT(DEBUG, "rxq->hw_type %d", q->hw_type);
1400         IONIC_PRINT(DEBUG, "rxq->hw_index %d", q->hw_index);
1401         IONIC_PRINT(DEBUG, "rxq->db %p", q->db);
1402
1403         return 0;
1404 }
1405
1406 static int
1407 ionic_station_set(struct ionic_lif *lif)
1408 {
1409         struct ionic_admin_ctx ctx = {
1410                 .pending_work = true,
1411                 .cmd.lif_getattr = {
1412                         .opcode = IONIC_CMD_LIF_GETATTR,
1413                         .index = lif->index,
1414                         .attr = IONIC_LIF_ATTR_MAC,
1415                 },
1416         };
1417         int err;
1418
1419         IONIC_PRINT_CALL();
1420
1421         err = ionic_adminq_post_wait(lif, &ctx);
1422         if (err)
1423                 return err;
1424
1425         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1426                         lif->mac_addr)) {
1427                 IONIC_PRINT(INFO, "deleting station MAC addr");
1428
1429                 ionic_lif_addr_del(lif, lif->mac_addr);
1430         }
1431
1432         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1433
1434         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1435                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1436                 return 0;
1437         }
1438
1439         IONIC_PRINT(DEBUG, "adding station MAC addr");
1440
1441         ionic_lif_addr_add(lif, lif->mac_addr);
1442
1443         return 0;
1444 }
1445
1446 static void
1447 ionic_lif_set_name(struct ionic_lif *lif)
1448 {
1449         struct ionic_admin_ctx ctx = {
1450                 .pending_work = true,
1451                 .cmd.lif_setattr = {
1452                         .opcode = IONIC_CMD_LIF_SETATTR,
1453                         .index = lif->index,
1454                         .attr = IONIC_LIF_ATTR_NAME,
1455                 },
1456         };
1457
1458         memcpy(ctx.cmd.lif_setattr.name, lif->name,
1459                 sizeof(ctx.cmd.lif_setattr.name) - 1);
1460
1461         ionic_adminq_post_wait(lif, &ctx);
1462 }
1463
1464 int
1465 ionic_lif_init(struct ionic_lif *lif)
1466 {
1467         struct ionic_dev *idev = &lif->adapter->idev;
1468         struct ionic_q_init_comp comp;
1469         int err;
1470
1471         memset(&lif->stats_base, 0, sizeof(lif->stats_base));
1472
1473         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
1474         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1475         ionic_dev_cmd_comp(idev, &comp);
1476         if (err)
1477                 return err;
1478
1479         lif->hw_index = comp.hw_index;
1480
1481         err = ionic_lif_adminq_init(lif);
1482         if (err)
1483                 return err;
1484
1485         err = ionic_lif_notifyq_init(lif);
1486         if (err)
1487                 goto err_out_adminq_deinit;
1488
1489         lif->features =
1490                   IONIC_ETH_HW_VLAN_TX_TAG
1491                 | IONIC_ETH_HW_VLAN_RX_STRIP
1492                 | IONIC_ETH_HW_VLAN_RX_FILTER
1493                 | IONIC_ETH_HW_RX_HASH
1494                 | IONIC_ETH_HW_TX_SG
1495                 | IONIC_ETH_HW_RX_SG
1496                 | IONIC_ETH_HW_TX_CSUM
1497                 | IONIC_ETH_HW_RX_CSUM
1498                 | IONIC_ETH_HW_TSO
1499                 | IONIC_ETH_HW_TSO_IPV6
1500                 | IONIC_ETH_HW_TSO_ECN;
1501
1502         err = ionic_lif_set_features(lif);
1503         if (err)
1504                 goto err_out_notifyq_deinit;
1505
1506         err = ionic_rx_filters_init(lif);
1507         if (err)
1508                 goto err_out_notifyq_deinit;
1509
1510         err = ionic_station_set(lif);
1511         if (err)
1512                 goto err_out_rx_filter_deinit;
1513
1514         ionic_lif_set_name(lif);
1515
1516         lif->state |= IONIC_LIF_F_INITED;
1517
1518         return 0;
1519
1520 err_out_rx_filter_deinit:
1521         ionic_rx_filters_deinit(lif);
1522
1523 err_out_notifyq_deinit:
1524         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1525
1526 err_out_adminq_deinit:
1527         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1528
1529         return err;
1530 }
1531
1532 void
1533 ionic_lif_deinit(struct ionic_lif *lif)
1534 {
1535         if (!(lif->state & IONIC_LIF_F_INITED))
1536                 return;
1537
1538         ionic_rx_filters_deinit(lif);
1539         ionic_lif_rss_teardown(lif);
1540         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1541         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1542
1543         lif->state &= ~IONIC_LIF_F_INITED;
1544 }
1545
1546 int
1547 ionic_lif_configure(struct ionic_lif *lif)
1548 {
1549         struct ionic_identity *ident = &lif->adapter->ident;
1550         uint32_t ntxqs_per_lif =
1551                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1552         uint32_t nrxqs_per_lif =
1553                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1554         uint32_t nrxqs = lif->eth_dev->data->nb_rx_queues;
1555         uint32_t ntxqs = lif->eth_dev->data->nb_tx_queues;
1556
1557         lif->port_id = lif->eth_dev->data->port_id;
1558
1559         IONIC_PRINT(DEBUG, "Configuring LIF on port %u",
1560                 lif->port_id);
1561
1562         if (nrxqs > 0)
1563                 nrxqs_per_lif = RTE_MIN(nrxqs_per_lif, nrxqs);
1564
1565         if (ntxqs > 0)
1566                 ntxqs_per_lif = RTE_MIN(ntxqs_per_lif, ntxqs);
1567
1568         lif->nrxqcqs = nrxqs_per_lif;
1569         lif->ntxqcqs = ntxqs_per_lif;
1570
1571         return 0;
1572 }
1573
1574 int
1575 ionic_lif_start(struct ionic_lif *lif)
1576 {
1577         uint32_t rx_mode = 0;
1578         uint32_t i;
1579         int err;
1580
1581         IONIC_PRINT(DEBUG, "Setting RSS configuration on port %u",
1582                 lif->port_id);
1583
1584         err = ionic_lif_rss_setup(lif);
1585         if (err)
1586                 return err;
1587
1588         IONIC_PRINT(DEBUG, "Setting RX mode on port %u",
1589                 lif->port_id);
1590
1591         rx_mode |= IONIC_RX_MODE_F_UNICAST;
1592         rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1593         rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1594
1595         lif->rx_mode = 0; /* set by ionic_set_rx_mode */
1596
1597         ionic_set_rx_mode(lif, rx_mode);
1598
1599         IONIC_PRINT(DEBUG, "Starting %u RX queues and %u TX queues "
1600                 "on port %u",
1601                 lif->nrxqcqs, lif->ntxqcqs, lif->port_id);
1602
1603         for (i = 0; i < lif->nrxqcqs; i++) {
1604                 struct ionic_qcq *rxq = lif->rxqcqs[i];
1605                 if (!(rxq->flags & IONIC_QCQ_F_DEFERRED)) {
1606                         err = ionic_dev_rx_queue_start(lif->eth_dev, i);
1607
1608                         if (err)
1609                                 return err;
1610                 }
1611         }
1612
1613         for (i = 0; i < lif->ntxqcqs; i++) {
1614                 struct ionic_qcq *txq = lif->txqcqs[i];
1615                 if (!(txq->flags & IONIC_QCQ_F_DEFERRED)) {
1616                         err = ionic_dev_tx_queue_start(lif->eth_dev, i);
1617
1618                         if (err)
1619                                 return err;
1620                 }
1621         }
1622
1623         ionic_link_status_check(lif);
1624
1625         /* Carrier ON here */
1626
1627         return 0;
1628 }
1629
1630 int
1631 ionic_lif_identify(struct ionic_adapter *adapter)
1632 {
1633         struct ionic_dev *idev = &adapter->idev;
1634         struct ionic_identity *ident = &adapter->ident;
1635         int err;
1636         unsigned int i;
1637         unsigned int lif_words = sizeof(ident->lif.words) /
1638                 sizeof(ident->lif.words[0]);
1639         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1640                 sizeof(idev->dev_cmd->data[0]);
1641         unsigned int nwords;
1642
1643         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1644                 IONIC_IDENTITY_VERSION_1);
1645         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1646         if (err)
1647                 return (err);
1648
1649         nwords = RTE_MIN(lif_words, cmd_words);
1650         for (i = 0; i < nwords; i++)
1651                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1652
1653         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1654                 ident->lif.capabilities);
1655
1656         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1657                 ident->lif.eth.max_ucast_filters);
1658         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1659                 ident->lif.eth.max_mcast_filters);
1660
1661         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1662                 ident->lif.eth.config.features);
1663         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1664                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1665         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1666                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1667         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1668                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1669         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1670                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1671
1672         return 0;
1673 }
1674
1675 int
1676 ionic_lifs_size(struct ionic_adapter *adapter)
1677 {
1678         struct ionic_identity *ident = &adapter->ident;
1679         uint32_t nlifs = ident->dev.nlifs;
1680         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1681
1682         adapter->max_ntxqs_per_lif =
1683                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1684         adapter->max_nrxqs_per_lif =
1685                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1686
1687         nintrs = nlifs * 1 /* notifyq */;
1688
1689         if (nintrs > dev_nintrs) {
1690                 IONIC_PRINT(ERR,
1691                         "At most %d intr supported, minimum req'd is %u",
1692                         dev_nintrs, nintrs);
1693                 return -ENOSPC;
1694         }
1695
1696         adapter->nintrs = nintrs;
1697
1698         return 0;
1699 }