drivers/net/ionic/ionic_lif.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_malloc.h>
   6 #include <rte_ethdev_driver.h>
   7
   8 #include "ionic.h"
   9 #include "ionic_logs.h"
  10 #include "ionic_lif.h"
  11 #include "ionic_ethdev.h"
  12 #include "ionic_rx_filter.h"
  13
  14 static int ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr);
  15 static int ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr);
  16
  17 int
  18 ionic_qcq_enable(struct ionic_qcq *qcq)
  19 {
  20         struct ionic_queue *q = &qcq->q;
  21         struct ionic_lif *lif = q->lif;
  22         struct ionic_dev *idev = &lif->adapter->idev;
  23         struct ionic_admin_ctx ctx = {
  24                 .pending_work = true,
  25                 .cmd.q_control = {
  26                         .opcode = IONIC_CMD_Q_CONTROL,
  27                         .lif_index = lif->index,
  28                         .type = q->type,
  29                         .index = q->index,
  30                         .oper = IONIC_Q_ENABLE,
  31                 },
  32         };
  33
  34         if (qcq->flags & IONIC_QCQ_F_INTR) {
  35                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  36                         IONIC_INTR_MASK_CLEAR);
  37         }
  38
  39         return ionic_adminq_post_wait(lif, &ctx);
  40 }
  41
  42 int
  43 ionic_qcq_disable(struct ionic_qcq *qcq)
  44 {
  45         struct ionic_queue *q = &qcq->q;
  46         struct ionic_lif *lif = q->lif;
  47         struct ionic_dev *idev = &lif->adapter->idev;
  48         struct ionic_admin_ctx ctx = {
  49                 .pending_work = true,
  50                 .cmd.q_control = {
  51                         .opcode = IONIC_CMD_Q_CONTROL,
  52                         .lif_index = lif->index,
  53                         .type = q->type,
  54                         .index = q->index,
  55                         .oper = IONIC_Q_DISABLE,
  56                 },
  57         };
  58
  59         if (qcq->flags & IONIC_QCQ_F_INTR) {
  60                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
  61                         IONIC_INTR_MASK_SET);
  62         }
  63
  64         return ionic_adminq_post_wait(lif, &ctx);
  65 }
  66
  67 int
  68 ionic_lif_stop(struct ionic_lif *lif __rte_unused)
  69 {
  70         /* Carrier OFF here */
  71
  72         return 0;
  73 }
  74
  75 void
  76 ionic_lif_reset(struct ionic_lif *lif)
  77 {
  78         struct ionic_dev *idev = &lif->adapter->idev;
  79
  80         IONIC_PRINT_CALL();
  81
  82         ionic_dev_cmd_lif_reset(idev, lif->index);
  83         ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
  84 }
  85
  86 static int
  87 ionic_lif_addr_add(struct ionic_lif *lif, const uint8_t *addr)
  88 {
  89         struct ionic_admin_ctx ctx = {
  90                 .pending_work = true,
  91                 .cmd.rx_filter_add = {
  92                         .opcode = IONIC_CMD_RX_FILTER_ADD,
  93                         .match = IONIC_RX_FILTER_MATCH_MAC,
  94                 },
  95         };
  96         int err;
  97
  98         memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, RTE_ETHER_ADDR_LEN);
  99
 100         err = ionic_adminq_post_wait(lif, &ctx);
 101         if (err)
 102                 return err;
 103
 104         IONIC_PRINT(INFO, "rx_filter add (id %d)",
 105                 ctx.comp.rx_filter_add.filter_id);
 106
 107         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 108 }
 109
 110 static int
 111 ionic_lif_addr_del(struct ionic_lif *lif, const uint8_t *addr)
 112 {
 113         struct ionic_admin_ctx ctx = {
 114                 .pending_work = true,
 115                 .cmd.rx_filter_del = {
 116                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 117                 },
 118         };
 119         struct ionic_rx_filter *f;
 120         int err;
 121
 122         IONIC_PRINT_CALL();
 123
 124         rte_spinlock_lock(&lif->rx_filters.lock);
 125
 126         f = ionic_rx_filter_by_addr(lif, addr);
 127         if (!f) {
 128                 rte_spinlock_unlock(&lif->rx_filters.lock);
 129                 return -ENOENT;
 130         }
 131
 132         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 133         ionic_rx_filter_free(f);
 134
 135         rte_spinlock_unlock(&lif->rx_filters.lock);
 136
 137         err = ionic_adminq_post_wait(lif, &ctx);
 138         if (err)
 139                 return err;
 140
 141         IONIC_PRINT(INFO, "rx_filter del (id %d)",
 142                 ctx.cmd.rx_filter_del.filter_id);
 143
 144         return 0;
 145 }
 146
 147 int
 148 ionic_dev_add_mac(struct rte_eth_dev *eth_dev,
 149                 struct rte_ether_addr *mac_addr,
 150                 uint32_t index __rte_unused, uint32_t pool __rte_unused)
 151 {
 152         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 153
 154         IONIC_PRINT_CALL();
 155
 156         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 157 }
 158
 159 void
 160 ionic_dev_remove_mac(struct rte_eth_dev *eth_dev, uint32_t index __rte_unused)
 161 {
 162         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 163         struct ionic_adapter *adapter = lif->adapter;
 164
 165         IONIC_PRINT_CALL();
 166
 167         if (index >= adapter->max_mac_addrs) {
 168                 IONIC_PRINT(WARNING,
 169                         "Index %u is above MAC filter limit %u",
 170                         index, adapter->max_mac_addrs);
 171                 return;
 172         }
 173
 174         if (!rte_is_valid_assigned_ether_addr(&eth_dev->data->mac_addrs[index]))
 175                 return;
 176
 177         ionic_lif_addr_del(lif, (const uint8_t *)
 178                 &eth_dev->data->mac_addrs[index]);
 179 }
 180
 181 int
 182 ionic_dev_set_mac(struct rte_eth_dev *eth_dev, struct rte_ether_addr *mac_addr)
 183 {
 184         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 185
 186         IONIC_PRINT_CALL();
 187
 188         if (mac_addr == NULL) {
 189                 IONIC_PRINT(NOTICE, "New mac is null");
 190                 return -1;
 191         }
 192
 193         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
 194                 IONIC_PRINT(INFO, "Deleting mac addr %pM",
 195                         lif->mac_addr);
 196                 ionic_lif_addr_del(lif, lif->mac_addr);
 197                 memset(lif->mac_addr, 0, RTE_ETHER_ADDR_LEN);
 198         }
 199
 200         IONIC_PRINT(INFO, "Updating mac addr");
 201
 202         rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)lif->mac_addr);
 203
 204         return ionic_lif_addr_add(lif, (const uint8_t *)mac_addr);
 205 }
 206
 207 static int
 208 ionic_vlan_rx_add_vid(struct ionic_lif *lif, uint16_t vid)
 209 {
 210         struct ionic_admin_ctx ctx = {
 211                 .pending_work = true,
 212                 .cmd.rx_filter_add = {
 213                         .opcode = IONIC_CMD_RX_FILTER_ADD,
 214                         .match = IONIC_RX_FILTER_MATCH_VLAN,
 215                         .vlan.vlan = vid,
 216                 },
 217         };
 218         int err;
 219
 220         err = ionic_adminq_post_wait(lif, &ctx);
 221         if (err)
 222                 return err;
 223
 224         IONIC_PRINT(INFO, "rx_filter add VLAN %d (id %d)", vid,
 225                 ctx.comp.rx_filter_add.filter_id);
 226
 227         return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, &ctx);
 228 }
 229
 230 static int
 231 ionic_vlan_rx_kill_vid(struct ionic_lif *lif, uint16_t vid)
 232 {
 233         struct ionic_admin_ctx ctx = {
 234                 .pending_work = true,
 235                 .cmd.rx_filter_del = {
 236                         .opcode = IONIC_CMD_RX_FILTER_DEL,
 237                 },
 238         };
 239         struct ionic_rx_filter *f;
 240         int err;
 241
 242         IONIC_PRINT_CALL();
 243
 244         rte_spinlock_lock(&lif->rx_filters.lock);
 245
 246         f = ionic_rx_filter_by_vlan(lif, vid);
 247         if (!f) {
 248                 rte_spinlock_unlock(&lif->rx_filters.lock);
 249                 return -ENOENT;
 250         }
 251
 252         ctx.cmd.rx_filter_del.filter_id = f->filter_id;
 253         ionic_rx_filter_free(f);
 254         rte_spinlock_unlock(&lif->rx_filters.lock);
 255
 256         err = ionic_adminq_post_wait(lif, &ctx);
 257         if (err)
 258                 return err;
 259
 260         IONIC_PRINT(INFO, "rx_filter del VLAN %d (id %d)", vid,
 261                 ctx.cmd.rx_filter_del.filter_id);
 262
 263         return 0;
 264 }
 265
 266 int
 267 ionic_dev_vlan_filter_set(struct rte_eth_dev *eth_dev, uint16_t vlan_id,
 268                 int on)
 269 {
 270         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 271         int err;
 272
 273         if (on)
 274                 err = ionic_vlan_rx_add_vid(lif, vlan_id);
 275         else
 276                 err = ionic_vlan_rx_kill_vid(lif, vlan_id);
 277
 278         return err;
 279 }
 280
 281 static void
 282 ionic_lif_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 283 {
 284         struct ionic_admin_ctx ctx = {
 285                 .pending_work = true,
 286                 .cmd.rx_mode_set = {
 287                         .opcode = IONIC_CMD_RX_MODE_SET,
 288                         .lif_index = lif->index,
 289                         .rx_mode = rx_mode,
 290                 },
 291         };
 292         int err;
 293
 294         if (rx_mode & IONIC_RX_MODE_F_UNICAST)
 295                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_UNICAST");
 296         if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
 297                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_MULTICAST");
 298         if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
 299                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_BROADCAST");
 300         if (rx_mode & IONIC_RX_MODE_F_PROMISC)
 301                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_PROMISC");
 302         if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
 303                 IONIC_PRINT(DEBUG, "rx_mode IONIC_RX_MODE_F_ALLMULTI");
 304
 305         err = ionic_adminq_post_wait(lif, &ctx);
 306         if (err)
 307                 IONIC_PRINT(ERR, "Failure setting RX mode");
 308 }
 309
 310 static void
 311 ionic_set_rx_mode(struct ionic_lif *lif, uint32_t rx_mode)
 312 {
 313         if (lif->rx_mode != rx_mode) {
 314                 lif->rx_mode = rx_mode;
 315                 ionic_lif_rx_mode(lif, rx_mode);
 316         }
 317 }
 318
 319 int
 320 ionic_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 321 {
 322         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 323         uint32_t rx_mode = lif->rx_mode;
 324
 325         IONIC_PRINT_CALL();
 326
 327         rx_mode |= IONIC_RX_MODE_F_PROMISC;
 328
 329         ionic_set_rx_mode(lif, rx_mode);
 330
 331         return 0;
 332 }
 333
 334 int
 335 ionic_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 336 {
 337         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 338         uint32_t rx_mode = lif->rx_mode;
 339
 340         rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
 341
 342         ionic_set_rx_mode(lif, rx_mode);
 343
 344         return 0;
 345 }
 346
 347 int
 348 ionic_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 349 {
 350         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 351         uint32_t rx_mode = lif->rx_mode;
 352
 353         rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
 354
 355         ionic_set_rx_mode(lif, rx_mode);
 356
 357         return 0;
 358 }
 359
 360 int
 361 ionic_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 362 {
 363         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 364         uint32_t rx_mode = lif->rx_mode;
 365
 366         rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
 367
 368         ionic_set_rx_mode(lif, rx_mode);
 369
 370         return 0;
 371 }
 372
 373 int
 374 ionic_lif_change_mtu(struct ionic_lif *lif, int new_mtu)
 375 {
 376         struct ionic_admin_ctx ctx = {
 377                 .pending_work = true,
 378                 .cmd.lif_setattr = {
 379                         .opcode = IONIC_CMD_LIF_SETATTR,
 380                         .index = lif->index,
 381                         .attr = IONIC_LIF_ATTR_MTU,
 382                         .mtu = new_mtu,
 383                 },
 384         };
 385         int err;
 386
 387         err = ionic_adminq_post_wait(lif, &ctx);
 388         if (err)
 389                 return err;
 390
 391         lif->mtu = new_mtu;
 392
 393         return 0;
 394 }
 395
 396 int
 397 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 398 {
 399         struct ionic_adapter *adapter = lif->adapter;
 400         struct ionic_dev *idev = &adapter->idev;
 401         unsigned long index;
 402
 403         /*
 404          * Note: interrupt handler is called for index = 0 only
 405          * (we use interrupts for the notifyq only anyway,
 406          * which hash index = 0)
 407          */
 408
 409         for (index = 0; index < adapter->nintrs; index++)
 410                 if (!adapter->intrs[index])
 411                         break;
 412
 413         if (index == adapter->nintrs)
 414                 return -ENOSPC;
 415
 416         adapter->intrs[index] = true;
 417
 418         ionic_intr_init(idev, intr, index);
 419
 420         return 0;
 421 }
 422
 423 void
 424 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
 425 {
 426         if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
 427                 lif->adapter->intrs[intr->index] = false;
 428 }
 429
 430 static int
 431 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
 432                 uint32_t index,
 433                 const char *base, uint32_t flags,
 434                 uint32_t num_descs,
 435                 uint32_t desc_size,
 436                 uint32_t cq_desc_size,
 437                 uint32_t sg_desc_size,
 438                 uint32_t pid, struct ionic_qcq **qcq)
 439 {
 440         struct ionic_dev *idev = &lif->adapter->idev;
 441         struct ionic_qcq *new;
 442         uint32_t q_size, cq_size, sg_size, total_size;
 443         void *q_base, *cq_base, *sg_base;
 444         rte_iova_t q_base_pa = 0;
 445         rte_iova_t cq_base_pa = 0;
 446         rte_iova_t sg_base_pa = 0;
 447         uint32_t socket_id = rte_socket_id();
 448         int err;
 449
 450         *qcq = NULL;
 451
 452         q_size  = num_descs * desc_size;
 453         cq_size = num_descs * cq_desc_size;
 454         sg_size = num_descs * sg_desc_size;
 455
 456         total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
 457                 RTE_ALIGN(cq_size, PAGE_SIZE);
 458         /*
 459          * Note: aligning q_size/cq_size is not enough due to cq_base address
 460          * aligning as q_base could be not aligned to the page.
 461          * Adding PAGE_SIZE.
 462          */
 463         total_size += PAGE_SIZE;
 464
 465         if (flags & IONIC_QCQ_F_SG) {
 466                 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
 467                 total_size += PAGE_SIZE;
 468         }
 469
 470         new = rte_zmalloc("ionic", sizeof(*new), 0);
 471         if (!new) {
 472                 IONIC_PRINT(ERR, "Cannot allocate queue structure");
 473                 return -ENOMEM;
 474         }
 475
 476         new->lif = lif;
 477         new->flags = flags;
 478
 479         new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
 480         if (!new->q.info) {
 481                 IONIC_PRINT(ERR, "Cannot allocate queue info");
 482                 return -ENOMEM;
 483         }
 484
 485         new->q.type = type;
 486
 487         err = ionic_q_init(lif, idev, &new->q, index, num_descs,
 488                 desc_size, sg_desc_size, pid);
 489         if (err) {
 490                 IONIC_PRINT(ERR, "Queue initialization failed");
 491                 return err;
 492         }
 493
 494         if (flags & IONIC_QCQ_F_INTR) {
 495                 err = ionic_intr_alloc(lif, &new->intr);
 496                 if (err)
 497                         return err;
 498
 499                 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
 500                         IONIC_INTR_MASK_SET);
 501         } else {
 502                 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 503         }
 504
 505         err = ionic_cq_init(lif, &new->cq, &new->intr,
 506                 num_descs, cq_desc_size);
 507         if (err) {
 508                 IONIC_PRINT(ERR, "Completion queue initialization failed");
 509                 goto err_out_free_intr;
 510         }
 511
 512         new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 513                 base /* name */, index /* queue_idx */,
 514                 total_size, IONIC_ALIGN, socket_id);
 515
 516         if (!new->base_z) {
 517                 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
 518                 err = -ENOMEM;
 519                 goto err_out_free_intr;
 520         }
 521
 522         new->base = new->base_z->addr;
 523         new->base_pa = new->base_z->iova;
 524         new->total_size = total_size;
 525
 526         q_base = new->base;
 527         q_base_pa = new->base_pa;
 528
 529         cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
 530         cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
 531
 532         if (flags & IONIC_QCQ_F_SG) {
 533                 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
 534                         PAGE_SIZE);
 535                 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
 536                 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 537         }
 538
 539         IONIC_PRINT(DEBUG, "Q-Base-PA = %ju CQ-Base-PA = %ju "
 540                 "SG-base-PA = %ju",
 541                 q_base_pa, cq_base_pa, sg_base_pa);
 542
 543         ionic_q_map(&new->q, q_base, q_base_pa);
 544         ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 545         ionic_cq_bind(&new->cq, &new->q);
 546
 547         *qcq = new;
 548
 549         return 0;
 550
 551 err_out_free_intr:
 552         if (flags & IONIC_QCQ_F_INTR)
 553                 ionic_intr_free(lif, &new->intr);
 554
 555         return err;
 556 }
 557
 558 void
 559 ionic_qcq_free(struct ionic_qcq *qcq)
 560 {
 561         if (qcq->base_z) {
 562                 qcq->base = NULL;
 563                 qcq->base_pa = 0;
 564                 rte_memzone_free(qcq->base_z);
 565                 qcq->base_z = NULL;
 566         }
 567
 568         if (qcq->q.info) {
 569                 rte_free(qcq->q.info);
 570                 qcq->q.info = NULL;
 571         }
 572
 573         rte_free(qcq);
 574 }
 575
 576 static int
 577 ionic_admin_qcq_alloc(struct ionic_lif *lif)
 578 {
 579         uint32_t flags;
 580         int err = -ENOMEM;
 581
 582         flags = 0;
 583         err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 584                 IONIC_ADMINQ_LENGTH,
 585                 sizeof(struct ionic_admin_cmd),
 586                 sizeof(struct ionic_admin_comp),
 587                 0,
 588                 lif->kern_pid, &lif->adminqcq);
 589         if (err)
 590                 return err;
 591
 592         return 0;
 593 }
 594
 595 static int
 596 ionic_notify_qcq_alloc(struct ionic_lif *lif)
 597 {
 598         uint32_t flags;
 599         int err = -ENOMEM;
 600
 601         flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
 602
 603         err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
 604                 flags,
 605                 IONIC_NOTIFYQ_LENGTH,
 606                 sizeof(struct ionic_notifyq_cmd),
 607                 sizeof(union ionic_notifyq_comp),
 608                 0,
 609                 lif->kern_pid, &lif->notifyqcq);
 610         if (err)
 611                 return err;
 612
 613         return 0;
 614 }
 615
 616 static void *
 617 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
 618 {
 619         char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
 620
 621         if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
 622                 return NULL;
 623
 624         return (void *)&vaddr[page_num << PAGE_SHIFT];
 625 }
 626
 627 int
 628 ionic_lif_alloc(struct ionic_lif *lif)
 629 {
 630         struct ionic_adapter *adapter = lif->adapter;
 631         uint32_t socket_id = rte_socket_id();
 632         int dbpage_num;
 633         int err;
 634
 635         snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
 636
 637         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 638
 639         rte_spinlock_init(&lif->adminq_lock);
 640         rte_spinlock_init(&lif->adminq_service_lock);
 641
 642         lif->kern_pid = 0;
 643
 644         dbpage_num = ionic_db_page_num(lif, 0);
 645
 646         lif->kern_dbpage = ionic_bus_map_dbpage(adapter, dbpage_num);
 647         if (!lif->kern_dbpage) {
 648                 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
 649                 return -ENOMEM;
 650         }
 651
 652         IONIC_PRINT(DEBUG, "Allocating Notify Queue");
 653
 654         err = ionic_notify_qcq_alloc(lif);
 655         if (err) {
 656                 IONIC_PRINT(ERR, "Cannot allocate notify queue");
 657                 return err;
 658         }
 659
 660         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 661
 662         IONIC_PRINT(DEBUG, "Allocating Admin Queue");
 663
 664         err = ionic_admin_qcq_alloc(lif);
 665         if (err) {
 666                 IONIC_PRINT(ERR, "Cannot allocate admin queue");
 667                 return err;
 668         }
 669
 670         IONIC_PRINT(DEBUG, "Allocating Lif Info");
 671
 672         lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
 673
 674         lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
 675                 "lif_info", 0 /* queue_idx*/,
 676                 lif->info_sz, IONIC_ALIGN, socket_id);
 677         if (!lif->info_z) {
 678                 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
 679                 return -ENOMEM;
 680         }
 681
 682         lif->info = lif->info_z->addr;
 683         lif->info_pa = lif->info_z->iova;
 684
 685         return 0;
 686 }
 687
 688 void
 689 ionic_lif_free(struct ionic_lif *lif)
 690 {
 691         if (lif->notifyqcq) {
 692                 ionic_qcq_free(lif->notifyqcq);
 693                 lif->notifyqcq = NULL;
 694         }
 695
 696         if (lif->adminqcq) {
 697                 ionic_qcq_free(lif->adminqcq);
 698                 lif->adminqcq = NULL;
 699         }
 700
 701         if (lif->info) {
 702                 rte_memzone_free(lif->info_z);
 703                 lif->info = NULL;
 704         }
 705 }
 706
 707 static void
 708 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
 709 {
 710         struct ionic_dev *idev = &lif->adapter->idev;
 711
 712         if (!(qcq->flags & IONIC_QCQ_F_INITED))
 713                 return;
 714
 715         if (qcq->flags & IONIC_QCQ_F_INTR)
 716                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 717                         IONIC_INTR_MASK_SET);
 718
 719         qcq->flags &= ~IONIC_QCQ_F_INITED;
 720 }
 721
 722 bool
 723 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
 724                 void *cb_arg __rte_unused)
 725 {
 726         struct ionic_admin_comp *cq_desc_base = cq->base;
 727         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
 728
 729         if (!color_match(cq_desc->color, cq->done_color))
 730                 return false;
 731
 732         ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
 733
 734         return true;
 735 }
 736
 737 /* This acts like ionic_napi */
 738 int
 739 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
 740                 void *cb_arg)
 741 {
 742         struct ionic_cq *cq = &qcq->cq;
 743         uint32_t work_done;
 744
 745         work_done = ionic_cq_service(cq, budget, cb, cb_arg);
 746
 747         return work_done;
 748 }
 749
 750 static void
 751 ionic_link_status_check(struct ionic_lif *lif)
 752 {
 753         struct ionic_adapter *adapter = lif->adapter;
 754         bool link_up;
 755
 756         lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
 757
 758         if (!lif->info)
 759                 return;
 760
 761         link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
 762
 763         if ((link_up  && adapter->link_up) ||
 764             (!link_up && !adapter->link_up))
 765                 return;
 766
 767         if (link_up) {
 768                 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
 769                         lif->info->status.link_speed);
 770                 adapter->link_speed = lif->info->status.link_speed;
 771         } else {
 772                 IONIC_PRINT(DEBUG, "Link down");
 773         }
 774
 775         adapter->link_up = link_up;
 776 }
 777
 778 static bool
 779 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
 780 {
 781         union ionic_notifyq_comp *cq_desc_base = cq->base;
 782         union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
 783         struct ionic_lif *lif = cb_arg;
 784
 785         IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
 786                 cq_desc->event.eid, cq_desc->event.ecode);
 787
 788         /* Have we run out of new completions to process? */
 789         if (!(cq_desc->event.eid > lif->last_eid))
 790                 return false;
 791
 792         lif->last_eid = cq_desc->event.eid;
 793
 794         switch (cq_desc->event.ecode) {
 795         case IONIC_EVENT_LINK_CHANGE:
 796                 IONIC_PRINT(DEBUG,
 797                         "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
 798                         cq_desc->event.eid,
 799                         cq_desc->link_change.link_status,
 800                         cq_desc->link_change.link_speed);
 801
 802                 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
 803
 804                 break;
 805         default:
 806                 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
 807                         cq_desc->event.ecode, cq_desc->event.eid);
 808                 break;
 809         }
 810
 811         return true;
 812 }
 813
 814 int
 815 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
 816 {
 817         struct ionic_dev *idev = &lif->adapter->idev;
 818         struct ionic_qcq *qcq = lif->notifyqcq;
 819         uint32_t work_done;
 820
 821         if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
 822                 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
 823                 return -1;
 824         }
 825
 826         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 827                 IONIC_INTR_MASK_SET);
 828
 829         work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
 830
 831         if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
 832                 ionic_link_status_check(lif);
 833
 834         ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
 835                 work_done, IONIC_INTR_CRED_RESET_COALESCE);
 836
 837         ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 838                 IONIC_INTR_MASK_CLEAR);
 839
 840         return 0;
 841 }
 842
 843 static int
 844 ionic_lif_adminq_init(struct ionic_lif *lif)
 845 {
 846         struct ionic_dev *idev = &lif->adapter->idev;
 847         struct ionic_qcq *qcq = lif->adminqcq;
 848         struct ionic_queue *q = &qcq->q;
 849         struct ionic_q_init_comp comp;
 850         int err;
 851
 852         ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
 853         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
 854         if (err)
 855                 return err;
 856
 857         ionic_dev_cmd_comp(idev, &comp);
 858
 859         q->hw_type = comp.hw_type;
 860         q->hw_index = comp.hw_index;
 861         q->db = ionic_db_map(lif, q);
 862
 863         IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
 864         IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
 865         IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
 866
 867         if (qcq->flags & IONIC_QCQ_F_INTR)
 868                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 869                         IONIC_INTR_MASK_CLEAR);
 870
 871         qcq->flags |= IONIC_QCQ_F_INITED;
 872
 873         return 0;
 874 }
 875
 876 static int
 877 ionic_lif_notifyq_init(struct ionic_lif *lif)
 878 {
 879         struct ionic_dev *idev = &lif->adapter->idev;
 880         struct ionic_qcq *qcq = lif->notifyqcq;
 881         struct ionic_queue *q = &qcq->q;
 882         int err;
 883
 884         struct ionic_admin_ctx ctx = {
 885                 .pending_work = true,
 886                 .cmd.q_init = {
 887                         .opcode = IONIC_CMD_Q_INIT,
 888                         .lif_index = lif->index,
 889                         .type = q->type,
 890                         .index = q->index,
 891                         .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
 892                         .intr_index = qcq->intr.index,
 893                         .pid = q->pid,
 894                         .ring_size = rte_log2_u32(q->num_descs),
 895                         .ring_base = q->base_pa,
 896                 }
 897         };
 898
 899         IONIC_PRINT(DEBUG, "notifyq_init.pid %d", ctx.cmd.q_init.pid);
 900         IONIC_PRINT(DEBUG, "notifyq_init.index %d",
 901                 ctx.cmd.q_init.index);
 902         IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
 903                 ctx.cmd.q_init.ring_base);
 904         IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
 905                 ctx.cmd.q_init.ring_size);
 906
 907         err = ionic_adminq_post_wait(lif, &ctx);
 908         if (err)
 909                 return err;
 910
 911         q->hw_type = ctx.comp.q_init.hw_type;
 912         q->hw_index = ctx.comp.q_init.hw_index;
 913         q->db = NULL;
 914
 915         IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
 916         IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
 917         IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
 918
 919         if (qcq->flags & IONIC_QCQ_F_INTR)
 920                 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 921                         IONIC_INTR_MASK_CLEAR);
 922
 923         qcq->flags |= IONIC_QCQ_F_INITED;
 924
 925         return 0;
 926 }
 927
 928 int
 929 ionic_lif_set_features(struct ionic_lif *lif)
 930 {
 931         struct ionic_admin_ctx ctx = {
 932                 .pending_work = true,
 933                 .cmd.lif_setattr = {
 934                         .opcode = IONIC_CMD_LIF_SETATTR,
 935                         .index = lif->index,
 936                         .attr = IONIC_LIF_ATTR_FEATURES,
 937                         .features = lif->features,
 938                 },
 939         };
 940         int err;
 941
 942         err = ionic_adminq_post_wait(lif, &ctx);
 943         if (err)
 944                 return err;
 945
 946         lif->hw_features = (ctx.cmd.lif_setattr.features &
 947                 ctx.comp.lif_setattr.features);
 948
 949         if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
 950                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_TX_TAG");
 951         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
 952                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_STRIP");
 953         if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
 954                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_VLAN_RX_FILTER");
 955         if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
 956                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_HASH");
 957         if (lif->hw_features & IONIC_ETH_HW_TX_SG)
 958                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_SG");
 959         if (lif->hw_features & IONIC_ETH_HW_RX_SG)
 960                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_SG");
 961         if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
 962                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TX_CSUM");
 963         if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
 964                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_RX_CSUM");
 965         if (lif->hw_features & IONIC_ETH_HW_TSO)
 966                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO");
 967         if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
 968                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPV6");
 969         if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
 970                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_ECN");
 971         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
 972                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE");
 973         if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
 974                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_GRE_CSUM");
 975         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
 976                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP4");
 977         if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
 978                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_IPXIP6");
 979         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
 980                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP");
 981         if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
 982                 IONIC_PRINT(DEBUG, "feature IONIC_ETH_HW_TSO_UDP_CSUM");
 983
 984         return 0;
 985 }
 986
 987 static int
 988 ionic_station_set(struct ionic_lif *lif)
 989 {
 990         struct ionic_admin_ctx ctx = {
 991                 .pending_work = true,
 992                 .cmd.lif_getattr = {
 993                         .opcode = IONIC_CMD_LIF_GETATTR,
 994                         .index = lif->index,
 995                         .attr = IONIC_LIF_ATTR_MAC,
 996                 },
 997         };
 998         int err;
 999
1000         IONIC_PRINT_CALL();
1001
1002         err = ionic_adminq_post_wait(lif, &ctx);
1003         if (err)
1004                 return err;
1005
1006         if (!rte_is_zero_ether_addr((struct rte_ether_addr *)
1007                         lif->mac_addr)) {
1008                 IONIC_PRINT(INFO, "deleting station MAC addr");
1009
1010                 ionic_lif_addr_del(lif, lif->mac_addr);
1011         }
1012
1013         memcpy(lif->mac_addr, ctx.comp.lif_getattr.mac, RTE_ETHER_ADDR_LEN);
1014
1015         if (rte_is_zero_ether_addr((struct rte_ether_addr *)lif->mac_addr)) {
1016                 IONIC_PRINT(NOTICE, "empty MAC addr (VF?)");
1017                 return 0;
1018         }
1019
1020         IONIC_PRINT(DEBUG, "adding station MAC addr");
1021
1022         ionic_lif_addr_add(lif, lif->mac_addr);
1023
1024         return 0;
1025 }
1026
1027 static void
1028 ionic_lif_set_name(struct ionic_lif *lif)
1029 {
1030         struct ionic_admin_ctx ctx = {
1031                 .pending_work = true,
1032                 .cmd.lif_setattr = {
1033                         .opcode = IONIC_CMD_LIF_SETATTR,
1034                         .index = lif->index,
1035                         .attr = IONIC_LIF_ATTR_NAME,
1036                 },
1037         };
1038
1039         snprintf(ctx.cmd.lif_setattr.name, sizeof(ctx.cmd.lif_setattr.name),
1040                 "%d", lif->port_id);
1041
1042         ionic_adminq_post_wait(lif, &ctx);
1043 }
1044
1045 int
1046 ionic_lif_init(struct ionic_lif *lif)
1047 {
1048         struct ionic_dev *idev = &lif->adapter->idev;
1049         struct ionic_q_init_comp comp;
1050         int err;
1051
1052         ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
1053         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1054         ionic_dev_cmd_comp(idev, &comp);
1055         if (err)
1056                 return err;
1057
1058         lif->hw_index = comp.hw_index;
1059
1060         err = ionic_lif_adminq_init(lif);
1061         if (err)
1062                 return err;
1063
1064         err = ionic_lif_notifyq_init(lif);
1065         if (err)
1066                 goto err_out_adminq_deinit;
1067
1068         lif->features = 0;
1069
1070         err = ionic_lif_set_features(lif);
1071         if (err)
1072                 goto err_out_notifyq_deinit;
1073
1074         err = ionic_rx_filters_init(lif);
1075         if (err)
1076                 goto err_out_notifyq_deinit;
1077
1078         err = ionic_station_set(lif);
1079         if (err)
1080                 goto err_out_rx_filter_deinit;
1081
1082         ionic_lif_set_name(lif);
1083
1084         lif->state |= IONIC_LIF_F_INITED;
1085
1086         return 0;
1087
1088 err_out_rx_filter_deinit:
1089         ionic_rx_filters_deinit(lif);
1090
1091 err_out_notifyq_deinit:
1092         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1093
1094 err_out_adminq_deinit:
1095         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1096
1097         return err;
1098 }
1099
1100 void
1101 ionic_lif_deinit(struct ionic_lif *lif)
1102 {
1103         if (!(lif->state & IONIC_LIF_F_INITED))
1104                 return;
1105
1106         ionic_rx_filters_deinit(lif);
1107         ionic_lif_qcq_deinit(lif, lif->notifyqcq);
1108         ionic_lif_qcq_deinit(lif, lif->adminqcq);
1109
1110         lif->state &= ~IONIC_LIF_F_INITED;
1111 }
1112
1113 int
1114 ionic_lif_configure(struct ionic_lif *lif)
1115 {
1116         lif->port_id = lif->eth_dev->data->port_id;
1117
1118         return 0;
1119 }
1120
1121 int
1122 ionic_lif_start(struct ionic_lif *lif)
1123 {
1124         uint32_t rx_mode = 0;
1125
1126         IONIC_PRINT(DEBUG, "Setting RX mode on port %u",
1127                 lif->port_id);
1128
1129         rx_mode |= IONIC_RX_MODE_F_UNICAST;
1130         rx_mode |= IONIC_RX_MODE_F_MULTICAST;
1131         rx_mode |= IONIC_RX_MODE_F_BROADCAST;
1132
1133         lif->rx_mode = 0; /* set by ionic_set_rx_mode */
1134
1135         ionic_set_rx_mode(lif, rx_mode);
1136
1137         ionic_link_status_check(lif);
1138
1139         /* Carrier ON here */
1140
1141         return 0;
1142 }
1143
1144 int
1145 ionic_lif_identify(struct ionic_adapter *adapter)
1146 {
1147         struct ionic_dev *idev = &adapter->idev;
1148         struct ionic_identity *ident = &adapter->ident;
1149         int err;
1150         unsigned int i;
1151         unsigned int lif_words = sizeof(ident->lif.words) /
1152                 sizeof(ident->lif.words[0]);
1153         unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
1154                 sizeof(idev->dev_cmd->data[0]);
1155         unsigned int nwords;
1156
1157         ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
1158                 IONIC_IDENTITY_VERSION_1);
1159         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
1160         if (err)
1161                 return (err);
1162
1163         nwords = RTE_MIN(lif_words, cmd_words);
1164         for (i = 0; i < nwords; i++)
1165                 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
1166
1167         IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
1168                 ident->lif.capabilities);
1169
1170         IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
1171                 ident->lif.eth.max_ucast_filters);
1172         IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
1173                 ident->lif.eth.max_mcast_filters);
1174
1175         IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
1176                 ident->lif.eth.config.features);
1177         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
1178                 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
1179         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
1180                 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
1181         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
1182                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
1183         IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
1184                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
1185
1186         return 0;
1187 }
1188
1189 int
1190 ionic_lifs_size(struct ionic_adapter *adapter)
1191 {
1192         struct ionic_identity *ident = &adapter->ident;
1193         uint32_t nlifs = ident->dev.nlifs;
1194         uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
1195
1196         adapter->max_ntxqs_per_lif =
1197                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
1198         adapter->max_nrxqs_per_lif =
1199                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
1200
1201         nintrs = nlifs * 1 /* notifyq */;
1202
1203         if (nintrs > dev_nintrs) {
1204                 IONIC_PRINT(ERR, "At most %d intr queues supported, minimum required is %u",
1205                         dev_nintrs, nintrs);
1206                 return -ENOSPC;
1207         }
1208
1209         adapter->nintrs = nintrs;
1210
1211         return 0;
1212 }