drivers/net/ionic/ionic_ethdev.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_pci.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_ethdev.h>
   8 #include <rte_ethdev_driver.h>
   9 #include <rte_malloc.h>
  10 #include <rte_ethdev_pci.h>
  11
  12 #include "ionic_logs.h"
  13 #include "ionic.h"
  14 #include "ionic_dev.h"
  15 #include "ionic_mac_api.h"
  16 #include "ionic_lif.h"
  17 #include "ionic_ethdev.h"
  18 #include "ionic_rxtx.h"
  19
  20 static int  eth_ionic_dev_init(struct rte_eth_dev *eth_dev, void *init_params);
  21 static int  eth_ionic_dev_uninit(struct rte_eth_dev *eth_dev);
  22 static int  ionic_dev_info_get(struct rte_eth_dev *eth_dev,
  23         struct rte_eth_dev_info *dev_info);
  24 static int  ionic_dev_configure(struct rte_eth_dev *dev);
  25 static int  ionic_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
  26 static int  ionic_dev_start(struct rte_eth_dev *dev);
  27 static void ionic_dev_stop(struct rte_eth_dev *dev);
  28 static void ionic_dev_close(struct rte_eth_dev *dev);
  29 static int  ionic_dev_set_link_up(struct rte_eth_dev *dev);
  30 static int  ionic_dev_set_link_down(struct rte_eth_dev *dev);
  31 static int  ionic_dev_link_update(struct rte_eth_dev *eth_dev,
  32         int wait_to_complete);
  33 static int  ionic_flow_ctrl_get(struct rte_eth_dev *eth_dev,
  34         struct rte_eth_fc_conf *fc_conf);
  35 static int  ionic_flow_ctrl_set(struct rte_eth_dev *eth_dev,
  36         struct rte_eth_fc_conf *fc_conf);
  37 static int  ionic_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask);
  38 static int  ionic_dev_rss_reta_update(struct rte_eth_dev *eth_dev,
  39         struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size);
  40 static int  ionic_dev_rss_reta_query(struct rte_eth_dev *eth_dev,
  41         struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size);
  42 static int  ionic_dev_rss_hash_conf_get(struct rte_eth_dev *eth_dev,
  43         struct rte_eth_rss_conf *rss_conf);
  44 static int  ionic_dev_rss_hash_update(struct rte_eth_dev *eth_dev,
  45         struct rte_eth_rss_conf *rss_conf);
  46
  47 int ionic_logtype;
  48
  49 static const struct rte_pci_id pci_id_ionic_map[] = {
  50         { RTE_PCI_DEVICE(IONIC_PENSANDO_VENDOR_ID, IONIC_DEV_ID_ETH_PF) },
  51         { RTE_PCI_DEVICE(IONIC_PENSANDO_VENDOR_ID, IONIC_DEV_ID_ETH_VF) },
  52         { RTE_PCI_DEVICE(IONIC_PENSANDO_VENDOR_ID, IONIC_DEV_ID_ETH_MGMT) },
  53         { .vendor_id = 0, /* sentinel */ },
  54 };
  55
  56 static const struct rte_eth_desc_lim rx_desc_lim = {
  57         .nb_max = IONIC_MAX_RING_DESC,
  58         .nb_min = IONIC_MIN_RING_DESC,
  59         .nb_align = 1,
  60 };
  61
  62 static const struct rte_eth_desc_lim tx_desc_lim = {
  63         .nb_max = IONIC_MAX_RING_DESC,
  64         .nb_min = IONIC_MIN_RING_DESC,
  65         .nb_align = 1,
  66         .nb_seg_max = IONIC_TX_MAX_SG_ELEMS,
  67         .nb_mtu_seg_max = IONIC_TX_MAX_SG_ELEMS,
  68 };
  69
  70 static const struct eth_dev_ops ionic_eth_dev_ops = {
  71         .dev_infos_get          = ionic_dev_info_get,
  72         .dev_configure          = ionic_dev_configure,
  73         .mtu_set                = ionic_dev_mtu_set,
  74         .dev_start              = ionic_dev_start,
  75         .dev_stop               = ionic_dev_stop,
  76         .dev_close              = ionic_dev_close,
  77         .link_update            = ionic_dev_link_update,
  78         .dev_set_link_up        = ionic_dev_set_link_up,
  79         .dev_set_link_down      = ionic_dev_set_link_down,
  80         .mac_addr_add           = ionic_dev_add_mac,
  81         .mac_addr_remove        = ionic_dev_remove_mac,
  82         .mac_addr_set           = ionic_dev_set_mac,
  83         .vlan_filter_set        = ionic_dev_vlan_filter_set,
  84         .promiscuous_enable     = ionic_dev_promiscuous_enable,
  85         .promiscuous_disable    = ionic_dev_promiscuous_disable,
  86         .allmulticast_enable    = ionic_dev_allmulticast_enable,
  87         .allmulticast_disable   = ionic_dev_allmulticast_disable,
  88         .flow_ctrl_get          = ionic_flow_ctrl_get,
  89         .flow_ctrl_set          = ionic_flow_ctrl_set,
  90         .rxq_info_get           = ionic_rxq_info_get,
  91         .txq_info_get           = ionic_txq_info_get,
  92         .rx_queue_setup         = ionic_dev_rx_queue_setup,
  93         .rx_queue_release       = ionic_dev_rx_queue_release,
  94         .rx_queue_start         = ionic_dev_rx_queue_start,
  95         .rx_queue_stop          = ionic_dev_rx_queue_stop,
  96         .tx_queue_setup         = ionic_dev_tx_queue_setup,
  97         .tx_queue_release       = ionic_dev_tx_queue_release,
  98         .tx_queue_start         = ionic_dev_tx_queue_start,
  99         .tx_queue_stop          = ionic_dev_tx_queue_stop,
 100         .vlan_offload_set       = ionic_vlan_offload_set,
 101         .reta_update            = ionic_dev_rss_reta_update,
 102         .reta_query             = ionic_dev_rss_reta_query,
 103         .rss_hash_conf_get      = ionic_dev_rss_hash_conf_get,
 104         .rss_hash_update        = ionic_dev_rss_hash_update,
 105 };
 106
 107 /*
 108  * Set device link up, enable tx.
 109  */
 110 static int
 111 ionic_dev_set_link_up(struct rte_eth_dev *eth_dev)
 112 {
 113         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 114         struct ionic_adapter *adapter = lif->adapter;
 115         struct ionic_dev *idev = &adapter->idev;
 116         int err;
 117
 118         IONIC_PRINT_CALL();
 119
 120         ionic_dev_cmd_port_state(idev, IONIC_PORT_ADMIN_STATE_UP);
 121
 122         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
 123         if (err) {
 124                 IONIC_PRINT(WARNING, "Failed to bring port UP");
 125                 return err;
 126         }
 127
 128         return 0;
 129 }
 130
 131 /*
 132  * Set device link down, disable tx.
 133  */
 134 static int
 135 ionic_dev_set_link_down(struct rte_eth_dev *eth_dev)
 136 {
 137         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 138         struct ionic_adapter *adapter = lif->adapter;
 139         struct ionic_dev *idev = &adapter->idev;
 140         int err;
 141
 142         IONIC_PRINT_CALL();
 143
 144         ionic_dev_cmd_port_state(idev, IONIC_PORT_ADMIN_STATE_DOWN);
 145
 146         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
 147         if (err) {
 148                 IONIC_PRINT(WARNING, "Failed to bring port DOWN");
 149                 return err;
 150         }
 151
 152         return 0;
 153 }
 154
 155 static int
 156 ionic_dev_link_update(struct rte_eth_dev *eth_dev,
 157                 int wait_to_complete __rte_unused)
 158 {
 159         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 160         struct ionic_adapter *adapter = lif->adapter;
 161         struct rte_eth_link link;
 162
 163         IONIC_PRINT_CALL();
 164
 165         /* Initialize */
 166         memset(&link, 0, sizeof(link));
 167         link.link_autoneg = ETH_LINK_AUTONEG;
 168
 169         if (!adapter->link_up) {
 170                 /* Interface is down */
 171                 link.link_status = ETH_LINK_DOWN;
 172                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
 173                 link.link_speed = ETH_SPEED_NUM_NONE;
 174         } else {
 175                 /* Interface is up */
 176                 link.link_status = ETH_LINK_UP;
 177                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
 178                 switch (adapter->link_speed) {
 179                 case  10000:
 180                         link.link_speed = ETH_SPEED_NUM_10G;
 181                         break;
 182                 case  25000:
 183                         link.link_speed = ETH_SPEED_NUM_25G;
 184                         break;
 185                 case  40000:
 186                         link.link_speed = ETH_SPEED_NUM_40G;
 187                         break;
 188                 case  50000:
 189                         link.link_speed = ETH_SPEED_NUM_50G;
 190                         break;
 191                 case 100000:
 192                         link.link_speed = ETH_SPEED_NUM_100G;
 193                         break;
 194                 default:
 195                         link.link_speed = ETH_SPEED_NUM_NONE;
 196                         break;
 197                 }
 198         }
 199
 200         return rte_eth_linkstatus_set(eth_dev, &link);
 201 }
 202
 203 /**
 204  * Interrupt handler triggered by NIC for handling
 205  * specific interrupt.
 206  *
 207  * @param param
 208  *  The address of parameter registered before.
 209  *
 210  * @return
 211  *  void
 212  */
 213 static void
 214 ionic_dev_interrupt_handler(void *param)
 215 {
 216         struct ionic_adapter *adapter = (struct ionic_adapter *)param;
 217         uint32_t i;
 218
 219         IONIC_PRINT(DEBUG, "->");
 220
 221         for (i = 0; i < adapter->nlifs; i++) {
 222                 if (adapter->lifs[i])
 223                         ionic_notifyq_handler(adapter->lifs[i], -1);
 224         }
 225 }
 226
 227 static int
 228 ionic_dev_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu)
 229 {
 230         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 231         uint32_t max_frame_size;
 232         int err;
 233
 234         IONIC_PRINT_CALL();
 235
 236         /*
 237          * Note: mtu check against IONIC_MIN_MTU, IONIC_MAX_MTU
 238          * is done by the the API.
 239          */
 240
 241         /*
 242          * Max frame size is MTU + Ethernet header + VLAN + QinQ
 243          * (plus ETHER_CRC_LEN if the adapter is able to keep CRC)
 244          */
 245         max_frame_size = mtu + RTE_ETHER_HDR_LEN + 4 + 4;
 246
 247         if (eth_dev->data->dev_conf.rxmode.max_rx_pkt_len < max_frame_size)
 248                 return -EINVAL;
 249
 250         err = ionic_lif_change_mtu(lif, mtu);
 251         if (err)
 252                 return err;
 253
 254         return 0;
 255 }
 256
 257 static int
 258 ionic_dev_info_get(struct rte_eth_dev *eth_dev,
 259                 struct rte_eth_dev_info *dev_info)
 260 {
 261         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 262         struct ionic_adapter *adapter = lif->adapter;
 263         struct ionic_identity *ident = &adapter->ident;
 264
 265         IONIC_PRINT_CALL();
 266
 267         dev_info->max_rx_queues = (uint16_t)
 268                 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
 269         dev_info->max_tx_queues = (uint16_t)
 270                 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
 271         /* Also add ETHER_CRC_LEN if the adapter is able to keep CRC */
 272         dev_info->min_rx_bufsize = IONIC_MIN_MTU + RTE_ETHER_HDR_LEN;
 273         dev_info->max_rx_pktlen = IONIC_MAX_MTU + RTE_ETHER_HDR_LEN;
 274         dev_info->max_mac_addrs = adapter->max_mac_addrs;
 275         dev_info->min_mtu = IONIC_MIN_MTU;
 276         dev_info->max_mtu = IONIC_MAX_MTU;
 277
 278         dev_info->hash_key_size = IONIC_RSS_HASH_KEY_SIZE;
 279         dev_info->reta_size = ident->lif.eth.rss_ind_tbl_sz;
 280         dev_info->flow_type_rss_offloads = IONIC_ETH_RSS_OFFLOAD_ALL;
 281
 282         dev_info->speed_capa =
 283                 ETH_LINK_SPEED_10G |
 284                 ETH_LINK_SPEED_25G |
 285                 ETH_LINK_SPEED_40G |
 286                 ETH_LINK_SPEED_50G |
 287                 ETH_LINK_SPEED_100G;
 288
 289         /*
 290          * Per-queue capabilities. Actually most of the offloads are enabled
 291          * by default on the port and can be used on selected queues (by adding
 292          * packet flags at runtime when required)
 293          */
 294
 295         dev_info->rx_queue_offload_capa =
 296                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 297                 DEV_RX_OFFLOAD_UDP_CKSUM |
 298                 DEV_RX_OFFLOAD_TCP_CKSUM |
 299                 0;
 300
 301         dev_info->tx_queue_offload_capa =
 302                 DEV_TX_OFFLOAD_VLAN_INSERT |
 303                 0;
 304
 305         /*
 306          * Per-port capabilities
 307          * See ionic_set_features to request and check supported features
 308          */
 309
 310         dev_info->rx_offload_capa = dev_info->rx_queue_offload_capa |
 311                 DEV_RX_OFFLOAD_JUMBO_FRAME |
 312                 DEV_RX_OFFLOAD_VLAN_FILTER |
 313                 DEV_RX_OFFLOAD_VLAN_STRIP |
 314                 DEV_RX_OFFLOAD_SCATTER |
 315                 0;
 316
 317         dev_info->tx_offload_capa = dev_info->tx_queue_offload_capa |
 318                 DEV_TX_OFFLOAD_MULTI_SEGS |
 319                 DEV_TX_OFFLOAD_TCP_TSO |
 320                 0;
 321
 322         dev_info->rx_desc_lim = rx_desc_lim;
 323         dev_info->tx_desc_lim = tx_desc_lim;
 324
 325         /* Driver-preferred Rx/Tx parameters */
 326         dev_info->default_rxportconf.burst_size = 32;
 327         dev_info->default_txportconf.burst_size = 32;
 328         dev_info->default_rxportconf.nb_queues = 1;
 329         dev_info->default_txportconf.nb_queues = 1;
 330         dev_info->default_rxportconf.ring_size = IONIC_DEF_TXRX_DESC;
 331         dev_info->default_txportconf.ring_size = IONIC_DEF_TXRX_DESC;
 332
 333         return 0;
 334 }
 335
 336 static int
 337 ionic_flow_ctrl_get(struct rte_eth_dev *eth_dev,
 338                 struct rte_eth_fc_conf *fc_conf)
 339 {
 340         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 341         struct ionic_adapter *adapter = lif->adapter;
 342         struct ionic_dev *idev = &adapter->idev;
 343
 344         if (idev->port_info) {
 345                 fc_conf->autoneg = idev->port_info->config.an_enable;
 346
 347                 if (idev->port_info->config.pause_type)
 348                         fc_conf->mode = RTE_FC_FULL;
 349                 else
 350                         fc_conf->mode = RTE_FC_NONE;
 351         }
 352
 353         return 0;
 354 }
 355
 356 static int
 357 ionic_flow_ctrl_set(struct rte_eth_dev *eth_dev,
 358                 struct rte_eth_fc_conf *fc_conf)
 359 {
 360         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 361         struct ionic_adapter *adapter = lif->adapter;
 362         struct ionic_dev *idev = &adapter->idev;
 363         uint8_t pause_type = IONIC_PORT_PAUSE_TYPE_NONE;
 364         uint8_t an_enable;
 365
 366         switch (fc_conf->mode) {
 367         case RTE_FC_NONE:
 368                 pause_type = IONIC_PORT_PAUSE_TYPE_NONE;
 369                 break;
 370         case RTE_FC_FULL:
 371                 pause_type = IONIC_PORT_PAUSE_TYPE_LINK;
 372                 break;
 373         case RTE_FC_RX_PAUSE:
 374         case RTE_FC_TX_PAUSE:
 375                 return -ENOTSUP;
 376         }
 377
 378         an_enable = fc_conf->autoneg;
 379
 380         ionic_dev_cmd_port_pause(idev, pause_type);
 381         ionic_dev_cmd_port_autoneg(idev, an_enable);
 382
 383         return 0;
 384 }
 385
 386 static int
 387 ionic_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask)
 388 {
 389         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 390         struct rte_eth_rxmode *rxmode;
 391         rxmode = &eth_dev->data->dev_conf.rxmode;
 392         int i;
 393
 394         if (mask & ETH_VLAN_STRIP_MASK) {
 395                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) {
 396                         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 397                                 struct ionic_qcq *rxq =
 398                                         eth_dev->data->rx_queues[i];
 399                                 rxq->offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
 400                         }
 401                         lif->features |= IONIC_ETH_HW_VLAN_RX_STRIP;
 402                 } else {
 403                         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 404                                 struct ionic_qcq *rxq =
 405                                         eth_dev->data->rx_queues[i];
 406                                 rxq->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
 407                         }
 408                         lif->features &= ~IONIC_ETH_HW_VLAN_RX_STRIP;
 409                 }
 410         }
 411
 412         if (mask & ETH_VLAN_FILTER_MASK) {
 413                 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
 414                         lif->features |= IONIC_ETH_HW_VLAN_RX_FILTER;
 415                 else
 416                         lif->features &= ~IONIC_ETH_HW_VLAN_RX_FILTER;
 417         }
 418
 419         ionic_lif_set_features(lif);
 420
 421         return 0;
 422 }
 423
 424 static int
 425 ionic_dev_rss_reta_update(struct rte_eth_dev *eth_dev,
 426                 struct rte_eth_rss_reta_entry64 *reta_conf,
 427                 uint16_t reta_size)
 428 {
 429         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 430         struct ionic_adapter *adapter = lif->adapter;
 431         struct ionic_identity *ident = &adapter->ident;
 432         uint32_t i, j, index, num;
 433
 434         IONIC_PRINT_CALL();
 435
 436         if (!lif->rss_ind_tbl) {
 437                 IONIC_PRINT(ERR, "RSS RETA not initialized, "
 438                         "can't update the table");
 439                 return -EINVAL;
 440         }
 441
 442         if (reta_size != ident->lif.eth.rss_ind_tbl_sz) {
 443                 IONIC_PRINT(ERR, "The size of hash lookup table configured "
 444                         "(%d) doesn't match the number hardware can supported "
 445                         "(%d)",
 446                         reta_size, ident->lif.eth.rss_ind_tbl_sz);
 447                 return -EINVAL;
 448         }
 449
 450         num = lif->adapter->ident.lif.eth.rss_ind_tbl_sz / RTE_RETA_GROUP_SIZE;
 451
 452         for (i = 0; i < num; i++) {
 453                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) {
 454                         if (reta_conf[i].mask & ((uint64_t)1 << j)) {
 455                                 index = (i * RTE_RETA_GROUP_SIZE) + j;
 456                                 lif->rss_ind_tbl[index] = reta_conf[i].reta[j];
 457                         }
 458                 }
 459         }
 460
 461         return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
 462 }
 463
 464 static int
 465 ionic_dev_rss_reta_query(struct rte_eth_dev *eth_dev,
 466                 struct rte_eth_rss_reta_entry64 *reta_conf,
 467                 uint16_t reta_size)
 468 {
 469         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 470         struct ionic_adapter *adapter = lif->adapter;
 471         struct ionic_identity *ident = &adapter->ident;
 472         int i, num;
 473
 474         IONIC_PRINT_CALL();
 475
 476         if (reta_size != ident->lif.eth.rss_ind_tbl_sz) {
 477                 IONIC_PRINT(ERR, "The size of hash lookup table configured "
 478                         "(%d) doesn't match the number hardware can supported "
 479                         "(%d)",
 480                         reta_size, ident->lif.eth.rss_ind_tbl_sz);
 481                 return -EINVAL;
 482         }
 483
 484         if (!lif->rss_ind_tbl) {
 485                 IONIC_PRINT(ERR, "RSS RETA has not been built yet");
 486                 return -EINVAL;
 487         }
 488
 489         num = reta_size / RTE_RETA_GROUP_SIZE;
 490
 491         for (i = 0; i < num; i++) {
 492                 memcpy(reta_conf->reta,
 493                         &lif->rss_ind_tbl[i * RTE_RETA_GROUP_SIZE],
 494                         RTE_RETA_GROUP_SIZE);
 495                 reta_conf++;
 496         }
 497
 498         return 0;
 499 }
 500
 501 static int
 502 ionic_dev_rss_hash_conf_get(struct rte_eth_dev *eth_dev,
 503                 struct rte_eth_rss_conf *rss_conf)
 504 {
 505         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 506         uint64_t rss_hf = 0;
 507
 508         IONIC_PRINT_CALL();
 509
 510         if (!lif->rss_ind_tbl) {
 511                 IONIC_PRINT(NOTICE, "RSS not enabled");
 512                 return 0;
 513         }
 514
 515         /* Get key value (if not null, rss_key is 40-byte) */
 516         if (rss_conf->rss_key != NULL &&
 517                         rss_conf->rss_key_len >= IONIC_RSS_HASH_KEY_SIZE)
 518                 memcpy(rss_conf->rss_key, lif->rss_hash_key,
 519                         IONIC_RSS_HASH_KEY_SIZE);
 520
 521         if (lif->rss_types & IONIC_RSS_TYPE_IPV4)
 522                 rss_hf |= ETH_RSS_IPV4;
 523         if (lif->rss_types & IONIC_RSS_TYPE_IPV4_TCP)
 524                 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
 525         if (lif->rss_types & IONIC_RSS_TYPE_IPV4_UDP)
 526                 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
 527         if (lif->rss_types & IONIC_RSS_TYPE_IPV6)
 528                 rss_hf |= ETH_RSS_IPV6;
 529         if (lif->rss_types & IONIC_RSS_TYPE_IPV6_TCP)
 530                 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
 531         if (lif->rss_types & IONIC_RSS_TYPE_IPV6_UDP)
 532                 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
 533
 534         rss_conf->rss_hf = rss_hf;
 535
 536         return 0;
 537 }
 538
 539 static int
 540 ionic_dev_rss_hash_update(struct rte_eth_dev *eth_dev,
 541                 struct rte_eth_rss_conf *rss_conf)
 542 {
 543         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 544         uint32_t rss_types = 0;
 545         uint8_t *key = NULL;
 546
 547         IONIC_PRINT_CALL();
 548
 549         if (rss_conf->rss_key)
 550                 key = rss_conf->rss_key;
 551
 552         if ((rss_conf->rss_hf & IONIC_ETH_RSS_OFFLOAD_ALL) == 0) {
 553                 /*
 554                  * Can't disable rss through hash flags,
 555                  * if it is enabled by default during init
 556                  */
 557                 if (lif->rss_ind_tbl)
 558                         return -EINVAL;
 559         } else {
 560                 /* Can't enable rss if disabled by default during init */
 561                 if (!lif->rss_ind_tbl)
 562                         return -EINVAL;
 563
 564                 if (rss_conf->rss_hf & ETH_RSS_IPV4)
 565                         rss_types |= IONIC_RSS_TYPE_IPV4;
 566                 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
 567                         rss_types |= IONIC_RSS_TYPE_IPV4_TCP;
 568                 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
 569                         rss_types |= IONIC_RSS_TYPE_IPV4_UDP;
 570                 if (rss_conf->rss_hf & ETH_RSS_IPV6)
 571                         rss_types |= IONIC_RSS_TYPE_IPV6;
 572                 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
 573                         rss_types |= IONIC_RSS_TYPE_IPV6_TCP;
 574                 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
 575                         rss_types |= IONIC_RSS_TYPE_IPV6_UDP;
 576
 577                 ionic_lif_rss_config(lif, rss_types, key, NULL);
 578         }
 579
 580         return 0;
 581 }
 582
 583 static int
 584 ionic_dev_configure(struct rte_eth_dev *eth_dev)
 585 {
 586         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 587         int err;
 588
 589         IONIC_PRINT_CALL();
 590
 591         err = ionic_lif_configure(lif);
 592         if (err) {
 593                 IONIC_PRINT(ERR, "Cannot configure LIF: %d", err);
 594                 return err;
 595         }
 596
 597         return 0;
 598 }
 599
 600 static inline uint32_t
 601 ionic_parse_link_speeds(uint16_t link_speeds)
 602 {
 603         if (link_speeds & ETH_LINK_SPEED_100G)
 604                 return 100000;
 605         else if (link_speeds & ETH_LINK_SPEED_50G)
 606                 return 50000;
 607         else if (link_speeds & ETH_LINK_SPEED_40G)
 608                 return 40000;
 609         else if (link_speeds & ETH_LINK_SPEED_25G)
 610                 return 25000;
 611         else if (link_speeds & ETH_LINK_SPEED_10G)
 612                 return 10000;
 613         else
 614                 return 0;
 615 }
 616
 617 /*
 618  * Configure device link speed and setup link.
 619  * It returns 0 on success.
 620  */
 621 static int
 622 ionic_dev_start(struct rte_eth_dev *eth_dev)
 623 {
 624         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
 625         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 626         struct ionic_adapter *adapter = lif->adapter;
 627         struct ionic_dev *idev = &adapter->idev;
 628         uint32_t allowed_speeds;
 629         int err;
 630
 631         IONIC_PRINT_CALL();
 632
 633         allowed_speeds =
 634                 ETH_LINK_SPEED_FIXED |
 635                 ETH_LINK_SPEED_10G |
 636                 ETH_LINK_SPEED_25G |
 637                 ETH_LINK_SPEED_40G |
 638                 ETH_LINK_SPEED_50G |
 639                 ETH_LINK_SPEED_100G;
 640
 641         if (dev_conf->link_speeds & ~allowed_speeds) {
 642                 IONIC_PRINT(ERR, "Invalid link setting");
 643                 return -EINVAL;
 644         }
 645
 646         err = ionic_lif_start(lif);
 647         if (err) {
 648                 IONIC_PRINT(ERR, "Cannot start LIF: %d", err);
 649                 return err;
 650         }
 651
 652         if (eth_dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
 653                 uint32_t speed = ionic_parse_link_speeds(dev_conf->link_speeds);
 654
 655                 if (speed)
 656                         ionic_dev_cmd_port_speed(idev, speed);
 657         }
 658
 659         ionic_dev_link_update(eth_dev, 0);
 660
 661         return 0;
 662 }
 663
 664 /*
 665  * Stop device: disable rx and tx functions to allow for reconfiguring.
 666  */
 667 static void
 668 ionic_dev_stop(struct rte_eth_dev *eth_dev)
 669 {
 670         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 671         int err;
 672
 673         IONIC_PRINT_CALL();
 674
 675         err = ionic_lif_stop(lif);
 676         if (err)
 677                 IONIC_PRINT(ERR, "Cannot stop LIF: %d", err);
 678 }
 679
 680 /*
 681  * Reset and stop device.
 682  */
 683 static void
 684 ionic_dev_close(struct rte_eth_dev *eth_dev)
 685 {
 686         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 687         int err;
 688
 689         IONIC_PRINT_CALL();
 690
 691         err = ionic_lif_stop(lif);
 692         if (err) {
 693                 IONIC_PRINT(ERR, "Cannot stop LIF: %d", err);
 694                 return;
 695         }
 696
 697         err = eth_ionic_dev_uninit(eth_dev);
 698         if (err) {
 699                 IONIC_PRINT(ERR, "Cannot destroy LIF: %d", err);
 700                 return;
 701         }
 702 }
 703
 704 static int
 705 eth_ionic_dev_init(struct rte_eth_dev *eth_dev, void *init_params)
 706 {
 707         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 708         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 709         struct ionic_adapter *adapter = (struct ionic_adapter *)init_params;
 710         int err;
 711
 712         IONIC_PRINT_CALL();
 713
 714         eth_dev->dev_ops = &ionic_eth_dev_ops;
 715         eth_dev->rx_pkt_burst = &ionic_recv_pkts;
 716         eth_dev->tx_pkt_burst = &ionic_xmit_pkts;
 717         eth_dev->tx_pkt_prepare = &ionic_prep_pkts;
 718
 719         /* Multi-process not supported, primary does initialization anyway */
 720         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 721                 return 0;
 722
 723         rte_eth_copy_pci_info(eth_dev, pci_dev);
 724
 725         lif->index = adapter->nlifs;
 726         lif->eth_dev = eth_dev;
 727         lif->adapter = adapter;
 728         adapter->lifs[adapter->nlifs] = lif;
 729
 730         IONIC_PRINT(DEBUG, "Up to %u MAC addresses supported",
 731                 adapter->max_mac_addrs);
 732
 733         /* Allocate memory for storing MAC addresses */
 734         eth_dev->data->mac_addrs = rte_zmalloc("ionic",
 735                 RTE_ETHER_ADDR_LEN * adapter->max_mac_addrs, 0);
 736
 737         if (eth_dev->data->mac_addrs == NULL) {
 738                 IONIC_PRINT(ERR, "Failed to allocate %u bytes needed to "
 739                         "store MAC addresses",
 740                         RTE_ETHER_ADDR_LEN * adapter->max_mac_addrs);
 741                 err = -ENOMEM;
 742                 goto err;
 743         }
 744
 745         err = ionic_lif_alloc(lif);
 746         if (err) {
 747                 IONIC_PRINT(ERR, "Cannot allocate LIFs: %d, aborting",
 748                         err);
 749                 goto err;
 750         }
 751
 752         err = ionic_lif_init(lif);
 753         if (err) {
 754                 IONIC_PRINT(ERR, "Cannot init LIFs: %d, aborting", err);
 755                 goto err_free_lif;
 756         }
 757
 758         /* Copy the MAC address */
 759         rte_ether_addr_copy((struct rte_ether_addr *)lif->mac_addr,
 760                 &eth_dev->data->mac_addrs[0]);
 761
 762         IONIC_PRINT(DEBUG, "Port %u initialized", eth_dev->data->port_id);
 763
 764         return 0;
 765
 766 err_free_lif:
 767         ionic_lif_free(lif);
 768 err:
 769         return err;
 770 }
 771
 772 static int
 773 eth_ionic_dev_uninit(struct rte_eth_dev *eth_dev)
 774 {
 775         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 776         struct ionic_adapter *adapter = lif->adapter;
 777
 778         IONIC_PRINT_CALL();
 779
 780         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 781                 return 0;
 782
 783         adapter->lifs[lif->index] = NULL;
 784
 785         ionic_lif_deinit(lif);
 786         ionic_lif_free(lif);
 787
 788         eth_dev->dev_ops = NULL;
 789         eth_dev->rx_pkt_burst = NULL;
 790         eth_dev->tx_pkt_burst = NULL;
 791         eth_dev->tx_pkt_prepare = NULL;
 792
 793         return 0;
 794 }
 795
 796 static int
 797 ionic_configure_intr(struct ionic_adapter *adapter)
 798 {
 799         struct rte_pci_device *pci_dev = adapter->pci_dev;
 800         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 801         int err;
 802
 803         IONIC_PRINT(DEBUG, "Configuring %u intrs", adapter->nintrs);
 804
 805         if (rte_intr_efd_enable(intr_handle, adapter->nintrs)) {
 806                 IONIC_PRINT(ERR, "Fail to create eventfd");
 807                 return -1;
 808         }
 809
 810         if (rte_intr_dp_is_en(intr_handle))
 811                 IONIC_PRINT(DEBUG,
 812                         "Packet I/O interrupt on datapath is enabled");
 813
 814         if (!intr_handle->intr_vec) {
 815                 intr_handle->intr_vec = rte_zmalloc("intr_vec",
 816                         adapter->nintrs * sizeof(int), 0);
 817
 818                 if (!intr_handle->intr_vec) {
 819                         IONIC_PRINT(ERR, "Failed to allocate %u vectors",
 820                                 adapter->nintrs);
 821                         return -ENOMEM;
 822                 }
 823         }
 824
 825         err = rte_intr_callback_register(intr_handle,
 826                 ionic_dev_interrupt_handler,
 827                 adapter);
 828
 829         if (err) {
 830                 IONIC_PRINT(ERR,
 831                         "Failure registering interrupts handler (%d)",
 832                         err);
 833                 return err;
 834         }
 835
 836         /* enable intr mapping */
 837         err = rte_intr_enable(intr_handle);
 838
 839         if (err) {
 840                 IONIC_PRINT(ERR, "Failure enabling interrupts (%d)", err);
 841                 return err;
 842         }
 843
 844         return 0;
 845 }
 846
 847 static void
 848 ionic_unconfigure_intr(struct ionic_adapter *adapter)
 849 {
 850         struct rte_pci_device *pci_dev = adapter->pci_dev;
 851         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 852
 853         rte_intr_disable(intr_handle);
 854
 855         rte_intr_callback_unregister(intr_handle,
 856                 ionic_dev_interrupt_handler,
 857                 adapter);
 858 }
 859
 860 static int
 861 eth_ionic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 862                 struct rte_pci_device *pci_dev)
 863 {
 864         char name[RTE_ETH_NAME_MAX_LEN];
 865         struct rte_mem_resource *resource;
 866         struct ionic_adapter *adapter;
 867         struct ionic_hw *hw;
 868         unsigned long i;
 869         int err;
 870
 871         /* Check structs (trigger error at compilation time) */
 872         ionic_struct_size_checks();
 873
 874         /* Multi-process not supported */
 875         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 876                 err = -EPERM;
 877                 goto err;
 878         }
 879
 880         IONIC_PRINT(DEBUG, "Initializing device %s",
 881                 pci_dev->device.name);
 882
 883         adapter = rte_zmalloc("ionic", sizeof(*adapter), 0);
 884         if (!adapter) {
 885                 IONIC_PRINT(ERR, "OOM");
 886                 err = -ENOMEM;
 887                 goto err;
 888         }
 889
 890         adapter->pci_dev = pci_dev;
 891         hw = &adapter->hw;
 892
 893         hw->device_id = pci_dev->id.device_id;
 894         hw->vendor_id = pci_dev->id.vendor_id;
 895
 896         err = ionic_init_mac(hw);
 897         if (err != 0) {
 898                 IONIC_PRINT(ERR, "Mac init failed: %d", err);
 899                 err = -EIO;
 900                 goto err_free_adapter;
 901         }
 902
 903         adapter->is_mgmt_nic = (pci_dev->id.device_id == IONIC_DEV_ID_ETH_MGMT);
 904
 905         adapter->num_bars = 0;
 906         for (i = 0; i < PCI_MAX_RESOURCE && i < IONIC_BARS_MAX; i++) {
 907                 resource = &pci_dev->mem_resource[i];
 908                 if (resource->phys_addr == 0 || resource->len == 0)
 909                         continue;
 910                 adapter->bars[adapter->num_bars].vaddr = resource->addr;
 911                 adapter->bars[adapter->num_bars].bus_addr = resource->phys_addr;
 912                 adapter->bars[adapter->num_bars].len = resource->len;
 913                 adapter->num_bars++;
 914         }
 915
 916         /* Discover ionic dev resources */
 917
 918         err = ionic_setup(adapter);
 919         if (err) {
 920                 IONIC_PRINT(ERR, "Cannot setup device: %d, aborting", err);
 921                 goto err_free_adapter;
 922         }
 923
 924         err = ionic_identify(adapter);
 925         if (err) {
 926                 IONIC_PRINT(ERR, "Cannot identify device: %d, aborting",
 927                         err);
 928                 goto err_free_adapter;
 929         }
 930
 931         err = ionic_init(adapter);
 932         if (err) {
 933                 IONIC_PRINT(ERR, "Cannot init device: %d, aborting", err);
 934                 goto err_free_adapter;
 935         }
 936
 937         /* Configure the ports */
 938         err = ionic_port_identify(adapter);
 939         if (err) {
 940                 IONIC_PRINT(ERR, "Cannot identify port: %d, aborting",
 941                         err);
 942                 goto err_free_adapter;
 943         }
 944
 945         err = ionic_port_init(adapter);
 946         if (err) {
 947                 IONIC_PRINT(ERR, "Cannot init port: %d, aborting", err);
 948                 goto err_free_adapter;
 949         }
 950
 951         /* Configure LIFs */
 952         err = ionic_lif_identify(adapter);
 953         if (err) {
 954                 IONIC_PRINT(ERR, "Cannot identify lif: %d, aborting", err);
 955                 goto err_free_adapter;
 956         }
 957
 958         /* Allocate and init LIFs */
 959         err = ionic_lifs_size(adapter);
 960         if (err) {
 961                 IONIC_PRINT(ERR, "Cannot size LIFs: %d, aborting", err);
 962                 goto err_free_adapter;
 963         }
 964
 965         adapter->max_mac_addrs = adapter->ident.lif.eth.max_ucast_filters;
 966
 967         adapter->nlifs = 0;
 968         for (i = 0; i < adapter->ident.dev.nlifs; i++) {
 969                 snprintf(name, sizeof(name), "net_%s_lif_%lu",
 970                         pci_dev->device.name, i);
 971
 972                 err = rte_eth_dev_create(&pci_dev->device, name,
 973                         sizeof(struct ionic_lif),
 974                         NULL, NULL,
 975                         eth_ionic_dev_init, adapter);
 976                 if (err) {
 977                         IONIC_PRINT(ERR, "Cannot create eth device for "
 978                                 "ionic lif %s", name);
 979                         break;
 980                 }
 981
 982                 adapter->nlifs++;
 983         }
 984
 985         err = ionic_configure_intr(adapter);
 986
 987         if (err) {
 988                 IONIC_PRINT(ERR, "Failed to configure interrupts");
 989                 goto err_free_adapter;
 990         }
 991
 992         return 0;
 993
 994 err_free_adapter:
 995         rte_free(adapter);
 996 err:
 997         return err;
 998 }
 999
1000 static int
1001 eth_ionic_pci_remove(struct rte_pci_device *pci_dev __rte_unused)
1002 {
1003         char name[RTE_ETH_NAME_MAX_LEN];
1004         struct ionic_adapter *adapter = NULL;
1005         struct rte_eth_dev *eth_dev;
1006         struct ionic_lif *lif;
1007         uint32_t i;
1008
1009         /* Adapter lookup is using (the first) eth_dev name */
1010         snprintf(name, sizeof(name), "net_%s_lif_0",
1011                 pci_dev->device.name);
1012
1013         eth_dev = rte_eth_dev_allocated(name);
1014         if (eth_dev) {
1015                 lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
1016                 adapter = lif->adapter;
1017         }
1018
1019         if (adapter) {
1020                 ionic_unconfigure_intr(adapter);
1021
1022                 for (i = 0; i < adapter->nlifs; i++) {
1023                         lif = adapter->lifs[i];
1024                         rte_eth_dev_destroy(lif->eth_dev, eth_ionic_dev_uninit);
1025                 }
1026
1027                 rte_free(adapter);
1028         }
1029
1030         return 0;
1031 }
1032
1033 static struct rte_pci_driver rte_ionic_pmd = {
1034         .id_table = pci_id_ionic_map,
1035         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1036         .probe = eth_ionic_pci_probe,
1037         .remove = eth_ionic_pci_remove,
1038 };
1039
1040 RTE_PMD_REGISTER_PCI(net_ionic, rte_ionic_pmd);
1041 RTE_PMD_REGISTER_PCI_TABLE(net_ionic, pci_id_ionic_map);
1042 RTE_PMD_REGISTER_KMOD_DEP(net_ionic, "* igb_uio | uio_pci_generic | vfio-pci");
1043
1044 RTE_INIT(ionic_init_log)
1045 {
1046         ionic_logtype = rte_log_register("pmd.net.ionic");
1047         if (ionic_logtype >= 0)
1048                 rte_log_set_level(ionic_logtype, RTE_LOG_NOTICE);
1049 }