+static int
+sfc_mem_bar_init(struct sfc_adapter *sa, const efx_bar_region_t *mem_ebrp)
+{
+ struct rte_eth_dev *eth_dev = sa->eth_dev;
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ efsys_bar_t *ebp = &sa->mem_bar;
+ struct rte_mem_resource *res =
+ &pci_dev->mem_resource[mem_ebrp->ebr_index];
+
+ SFC_BAR_LOCK_INIT(ebp, eth_dev->data->name);
+ ebp->esb_rid = mem_ebrp->ebr_index;
+ ebp->esb_dev = pci_dev;
+ ebp->esb_base = res->addr;
+
+ sa->fcw_offset = mem_ebrp->ebr_offset;
+
+ return 0;
+}
+
+static void
+sfc_mem_bar_fini(struct sfc_adapter *sa)
+{
+ efsys_bar_t *ebp = &sa->mem_bar;
+
+ SFC_BAR_LOCK_DESTROY(ebp);
+ memset(ebp, 0, sizeof(*ebp));
+}
+
+/*
+ * A fixed RSS key which has a property of being symmetric
+ * (symmetrical flows are distributed to the same CPU)
+ * and also known to give a uniform distribution
+ * (a good distribution of traffic between different CPUs)
+ */
+static const uint8_t default_rss_key[EFX_RSS_KEY_SIZE] = {
+ 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
+ 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
+ 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
+ 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
+ 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a, 0x6d, 0x5a,
+};
+
+static int
+sfc_rss_attach(struct sfc_adapter *sa)
+{
+ struct sfc_rss *rss = &sfc_sa2shared(sa)->rss;
+ int rc;
+
+ rc = efx_intr_init(sa->nic, sa->intr.type, NULL);
+ if (rc != 0)
+ goto fail_intr_init;
+
+ rc = efx_ev_init(sa->nic);
+ if (rc != 0)
+ goto fail_ev_init;
+
+ rc = efx_rx_init(sa->nic);
+ if (rc != 0)
+ goto fail_rx_init;
+
+ rc = efx_rx_scale_default_support_get(sa->nic, &rss->context_type);
+ if (rc != 0)
+ goto fail_scale_support_get;
+
+ rc = efx_rx_hash_default_support_get(sa->nic, &rss->hash_support);
+ if (rc != 0)
+ goto fail_hash_support_get;
+
+ rc = sfc_rx_hash_init(sa);
+ if (rc != 0)
+ goto fail_rx_hash_init;
+
+ efx_rx_fini(sa->nic);
+ efx_ev_fini(sa->nic);
+ efx_intr_fini(sa->nic);
+
+ rte_memcpy(rss->key, default_rss_key, sizeof(rss->key));
+ rss->dummy_rss_context = EFX_RSS_CONTEXT_DEFAULT;
+
+ return 0;
+
+fail_rx_hash_init:
+fail_hash_support_get:
+fail_scale_support_get:
+ efx_rx_fini(sa->nic);
+
+fail_rx_init:
+ efx_ev_fini(sa->nic);
+
+fail_ev_init:
+ efx_intr_fini(sa->nic);
+
+fail_intr_init:
+ return rc;
+}
+
+static void
+sfc_rss_detach(struct sfc_adapter *sa)
+{
+ sfc_rx_hash_fini(sa);
+}
+
+int
+sfc_attach(struct sfc_adapter *sa)
+{
+ const efx_nic_cfg_t *encp;
+ efx_nic_t *enp = sa->nic;
+ int rc;
+
+ sfc_log_init(sa, "entry");
+
+ SFC_ASSERT(sfc_adapter_is_locked(sa));
+
+ efx_mcdi_new_epoch(enp);
+
+ sfc_log_init(sa, "reset nic");
+ rc = efx_nic_reset(enp);
+ if (rc != 0)
+ goto fail_nic_reset;
+
+ rc = sfc_sriov_attach(sa);
+ if (rc != 0)
+ goto fail_sriov_attach;
+
+ /*
+ * Probed NIC is sufficient for tunnel init.
+ * Initialize tunnel support to be able to use libefx
+ * efx_tunnel_config_udp_{add,remove}() in any state and
+ * efx_tunnel_reconfigure() on start up.
+ */
+ rc = efx_tunnel_init(enp);
+ if (rc != 0)
+ goto fail_tunnel_init;
+
+ encp = efx_nic_cfg_get(sa->nic);
+
+ /*
+ * Make a copy of supported tunnel encapsulations in shared
+ * memory to be used on supported Rx packet type classes get.
+ */
+ sa->priv.shared->tunnel_encaps =
+ encp->enc_tunnel_encapsulations_supported;
+
+ if (sfc_dp_tx_offload_capa(sa->priv.dp_tx) & RTE_ETH_TX_OFFLOAD_TCP_TSO) {
+ sa->tso = encp->enc_fw_assisted_tso_v2_enabled ||
+ encp->enc_tso_v3_enabled;
+ if (!sa->tso)
+ sfc_info(sa, "TSO support isn't available on this adapter");
+ }
+
+ if (sa->tso &&
+ (sfc_dp_tx_offload_capa(sa->priv.dp_tx) &
+ (RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
+ RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO)) != 0) {
+ sa->tso_encap = encp->enc_fw_assisted_tso_v2_encap_enabled ||
+ encp->enc_tso_v3_enabled;
+ if (!sa->tso_encap)
+ sfc_info(sa, "Encapsulated TSO support isn't available on this adapter");
+ }
+
+ sfc_log_init(sa, "estimate resource limits");
+ rc = sfc_estimate_resource_limits(sa);
+ if (rc != 0)
+ goto fail_estimate_rsrc_limits;
+
+ sa->evq_max_entries = encp->enc_evq_max_nevs;
+ SFC_ASSERT(rte_is_power_of_2(sa->evq_max_entries));
+
+ sa->evq_min_entries = encp->enc_evq_min_nevs;
+ SFC_ASSERT(rte_is_power_of_2(sa->evq_min_entries));
+
+ sa->rxq_max_entries = encp->enc_rxq_max_ndescs;
+ SFC_ASSERT(rte_is_power_of_2(sa->rxq_max_entries));
+
+ sa->rxq_min_entries = encp->enc_rxq_min_ndescs;
+ SFC_ASSERT(rte_is_power_of_2(sa->rxq_min_entries));
+
+ sa->txq_max_entries = encp->enc_txq_max_ndescs;
+ SFC_ASSERT(rte_is_power_of_2(sa->txq_max_entries));
+
+ sa->txq_min_entries = encp->enc_txq_min_ndescs;
+ SFC_ASSERT(rte_is_power_of_2(sa->txq_min_entries));
+
+ rc = sfc_intr_attach(sa);
+ if (rc != 0)
+ goto fail_intr_attach;
+
+ rc = sfc_ev_attach(sa);
+ if (rc != 0)
+ goto fail_ev_attach;
+
+ rc = sfc_port_attach(sa);
+ if (rc != 0)
+ goto fail_port_attach;
+
+ rc = sfc_rss_attach(sa);
+ if (rc != 0)
+ goto fail_rss_attach;
+
+ rc = sfc_filter_attach(sa);
+ if (rc != 0)
+ goto fail_filter_attach;
+
+ rc = sfc_mae_counter_rxq_attach(sa);
+ if (rc != 0)
+ goto fail_mae_counter_rxq_attach;
+
+ rc = sfc_mae_attach(sa);
+ if (rc != 0)
+ goto fail_mae_attach;
+
+ rc = sfc_mae_switchdev_init(sa);
+ if (rc != 0)
+ goto fail_mae_switchdev_init;
+
+ rc = sfc_repr_proxy_attach(sa);
+ if (rc != 0)
+ goto fail_repr_proxy_attach;
+
+ sfc_log_init(sa, "fini nic");
+ efx_nic_fini(enp);
+
+ sfc_flow_init(sa);
+
+ rc = sfc_sw_xstats_init(sa);
+ if (rc != 0)
+ goto fail_sw_xstats_init;
+
+ /*
+ * Create vSwitch to be able to use VFs when PF is not started yet
+ * as DPDK port. VFs should be able to talk to each other even
+ * if PF is down.
+ */
+ rc = sfc_sriov_vswitch_create(sa);
+ if (rc != 0)
+ goto fail_sriov_vswitch_create;
+
+ sa->state = SFC_ETHDEV_INITIALIZED;
+
+ sfc_log_init(sa, "done");
+ return 0;
+
+fail_sriov_vswitch_create:
+ sfc_sw_xstats_close(sa);
+
+fail_sw_xstats_init:
+ sfc_flow_fini(sa);
+ sfc_repr_proxy_detach(sa);
+
+fail_repr_proxy_attach:
+ sfc_mae_switchdev_fini(sa);
+
+fail_mae_switchdev_init:
+ sfc_mae_detach(sa);
+
+fail_mae_attach:
+ sfc_mae_counter_rxq_detach(sa);
+
+fail_mae_counter_rxq_attach:
+ sfc_filter_detach(sa);
+
+fail_filter_attach:
+ sfc_rss_detach(sa);
+
+fail_rss_attach:
+ sfc_port_detach(sa);
+
+fail_port_attach:
+ sfc_ev_detach(sa);
+
+fail_ev_attach:
+ sfc_intr_detach(sa);
+
+fail_intr_attach:
+ efx_nic_fini(sa->nic);
+
+fail_estimate_rsrc_limits:
+fail_tunnel_init:
+ efx_tunnel_fini(sa->nic);
+ sfc_sriov_detach(sa);
+
+fail_sriov_attach:
+fail_nic_reset:
+
+ sfc_log_init(sa, "failed %d", rc);
+ return rc;
+}
+
+void
+sfc_pre_detach(struct sfc_adapter *sa)
+{
+ sfc_log_init(sa, "entry");
+
+ SFC_ASSERT(!sfc_adapter_is_locked(sa));
+
+ sfc_repr_proxy_pre_detach(sa);
+
+ sfc_log_init(sa, "done");
+}
+
+void
+sfc_detach(struct sfc_adapter *sa)
+{
+ sfc_log_init(sa, "entry");
+
+ SFC_ASSERT(sfc_adapter_is_locked(sa));
+
+ sfc_sriov_vswitch_destroy(sa);
+
+ sfc_flow_fini(sa);
+
+ sfc_repr_proxy_detach(sa);
+ sfc_mae_switchdev_fini(sa);
+ sfc_mae_detach(sa);
+ sfc_mae_counter_rxq_detach(sa);
+ sfc_filter_detach(sa);
+ sfc_rss_detach(sa);
+ sfc_port_detach(sa);
+ sfc_ev_detach(sa);
+ sfc_intr_detach(sa);
+ efx_tunnel_fini(sa->nic);
+ sfc_sriov_detach(sa);
+
+ sa->state = SFC_ETHDEV_UNINITIALIZED;
+}
+
+static int
+sfc_kvarg_fv_variant_handler(__rte_unused const char *key,
+ const char *value_str, void *opaque)
+{
+ uint32_t *value = opaque;
+
+ if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DONT_CARE) == 0)
+ *value = EFX_FW_VARIANT_DONT_CARE;
+ else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_FULL_FEATURED) == 0)
+ *value = EFX_FW_VARIANT_FULL_FEATURED;
+ else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_LOW_LATENCY) == 0)
+ *value = EFX_FW_VARIANT_LOW_LATENCY;
+ else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_PACKED_STREAM) == 0)
+ *value = EFX_FW_VARIANT_PACKED_STREAM;
+ else if (strcasecmp(value_str, SFC_KVARG_FW_VARIANT_DPDK) == 0)
+ *value = EFX_FW_VARIANT_DPDK;
+ else
+ return -EINVAL;
+
+ return 0;
+}
+
+static int
+sfc_get_fw_variant(struct sfc_adapter *sa, efx_fw_variant_t *efv)
+{
+ efx_nic_fw_info_t enfi;
+ int rc;
+
+ rc = efx_nic_get_fw_version(sa->nic, &enfi);
+ if (rc != 0)
+ return rc;
+ else if (!enfi.enfi_dpcpu_fw_ids_valid)
+ return ENOTSUP;
+
+ /*
+ * Firmware variant can be uniquely identified by the RxDPCPU
+ * firmware id
+ */
+ switch (enfi.enfi_rx_dpcpu_fw_id) {
+ case EFX_RXDP_FULL_FEATURED_FW_ID:
+ *efv = EFX_FW_VARIANT_FULL_FEATURED;
+ break;
+
+ case EFX_RXDP_LOW_LATENCY_FW_ID:
+ *efv = EFX_FW_VARIANT_LOW_LATENCY;
+ break;
+
+ case EFX_RXDP_PACKED_STREAM_FW_ID:
+ *efv = EFX_FW_VARIANT_PACKED_STREAM;
+ break;
+
+ case EFX_RXDP_DPDK_FW_ID:
+ *efv = EFX_FW_VARIANT_DPDK;
+ break;
+
+ default:
+ /*
+ * Other firmware variants are not considered, since they are
+ * not supported in the device parameters
+ */
+ *efv = EFX_FW_VARIANT_DONT_CARE;
+ break;
+ }
+
+ return 0;
+}
+
+static const char *
+sfc_fw_variant2str(efx_fw_variant_t efv)
+{
+ switch (efv) {
+ case EFX_RXDP_FULL_FEATURED_FW_ID:
+ return SFC_KVARG_FW_VARIANT_FULL_FEATURED;
+ case EFX_RXDP_LOW_LATENCY_FW_ID:
+ return SFC_KVARG_FW_VARIANT_LOW_LATENCY;
+ case EFX_RXDP_PACKED_STREAM_FW_ID:
+ return SFC_KVARG_FW_VARIANT_PACKED_STREAM;
+ case EFX_RXDP_DPDK_FW_ID:
+ return SFC_KVARG_FW_VARIANT_DPDK;
+ default:
+ return "unknown";
+ }
+}
+
+static int
+sfc_kvarg_rxd_wait_timeout_ns(struct sfc_adapter *sa)
+{
+ int rc;
+ long value;
+
+ value = SFC_RXD_WAIT_TIMEOUT_NS_DEF;
+
+ rc = sfc_kvargs_process(sa, SFC_KVARG_RXD_WAIT_TIMEOUT_NS,
+ sfc_kvarg_long_handler, &value);
+ if (rc != 0)
+ return rc;
+
+ if (value < 0 ||
+ (unsigned long)value > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) {
+ sfc_err(sa, "wrong '" SFC_KVARG_RXD_WAIT_TIMEOUT_NS "' "
+ "was set (%ld);", value);
+ sfc_err(sa, "it must not be less than 0 or greater than %u",
+ EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX);
+ return EINVAL;
+ }
+
+ sa->rxd_wait_timeout_ns = value;
+ return 0;
+}
+
+static int
+sfc_nic_probe(struct sfc_adapter *sa)
+{
+ efx_nic_t *enp = sa->nic;
+ efx_fw_variant_t preferred_efv;
+ efx_fw_variant_t efv;
+ int rc;
+
+ preferred_efv = EFX_FW_VARIANT_DONT_CARE;
+ rc = sfc_kvargs_process(sa, SFC_KVARG_FW_VARIANT,
+ sfc_kvarg_fv_variant_handler,
+ &preferred_efv);
+ if (rc != 0) {
+ sfc_err(sa, "invalid %s parameter value", SFC_KVARG_FW_VARIANT);
+ return rc;
+ }
+
+ rc = sfc_kvarg_rxd_wait_timeout_ns(sa);
+ if (rc != 0)
+ return rc;
+
+ rc = efx_nic_probe(enp, preferred_efv);
+ if (rc == EACCES) {
+ /* Unprivileged functions cannot set FW variant */
+ rc = efx_nic_probe(enp, EFX_FW_VARIANT_DONT_CARE);
+ }
+ if (rc != 0)
+ return rc;
+
+ rc = sfc_get_fw_variant(sa, &efv);
+ if (rc == ENOTSUP) {
+ sfc_warn(sa, "FW variant can not be obtained");
+ return 0;
+ }
+ if (rc != 0)
+ return rc;
+
+ /* Check that firmware variant was changed to the requested one */
+ if (preferred_efv != EFX_FW_VARIANT_DONT_CARE && preferred_efv != efv) {
+ sfc_warn(sa, "FW variant has not changed to the requested %s",
+ sfc_fw_variant2str(preferred_efv));
+ }
+
+ sfc_notice(sa, "running FW variant is %s", sfc_fw_variant2str(efv));
+
+ return 0;
+}
+
+int
+sfc_probe(struct sfc_adapter *sa)
+{
+ efx_bar_region_t mem_ebrp;
+ struct rte_eth_dev *eth_dev = sa->eth_dev;
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
+ efx_nic_t *enp;
+ int rc;
+
+ sfc_log_init(sa, "entry");
+
+ SFC_ASSERT(sfc_adapter_is_locked(sa));
+
+ sa->socket_id = rte_socket_id();
+ rte_atomic32_init(&sa->restart_required);
+
+ sfc_log_init(sa, "get family");
+ rc = sfc_efx_family(pci_dev, &mem_ebrp, &sa->family);
+
+ if (rc != 0)
+ goto fail_family;
+ sfc_log_init(sa,
+ "family is %u, membar is %u, function control window offset is %lu",
+ sa->family, mem_ebrp.ebr_index, mem_ebrp.ebr_offset);
+
+ sfc_log_init(sa, "init mem bar");
+ rc = sfc_mem_bar_init(sa, &mem_ebrp);
+ if (rc != 0)
+ goto fail_mem_bar_init;
+
+ sfc_log_init(sa, "create nic");
+ rte_spinlock_init(&sa->nic_lock);
+ rc = efx_nic_create(sa->family, (efsys_identifier_t *)sa,
+ &sa->mem_bar, mem_ebrp.ebr_offset,
+ &sa->nic_lock, &enp);
+ if (rc != 0)
+ goto fail_nic_create;
+ sa->nic = enp;
+
+ rc = sfc_mcdi_init(sa);
+ if (rc != 0)
+ goto fail_mcdi_init;
+
+ sfc_log_init(sa, "probe nic");
+ rc = sfc_nic_probe(sa);
+ if (rc != 0)
+ goto fail_nic_probe;
+
+ sfc_log_init(sa, "done");
+ return 0;
+
+fail_nic_probe:
+ sfc_mcdi_fini(sa);
+
+fail_mcdi_init:
+ sfc_log_init(sa, "destroy nic");
+ sa->nic = NULL;
+ efx_nic_destroy(enp);
+
+fail_nic_create:
+ sfc_mem_bar_fini(sa);
+
+fail_mem_bar_init:
+fail_family:
+ sfc_log_init(sa, "failed %d", rc);
+ return rc;
+}
+
+void
+sfc_unprobe(struct sfc_adapter *sa)
+{
+ efx_nic_t *enp = sa->nic;
+
+ sfc_log_init(sa, "entry");
+
+ SFC_ASSERT(sfc_adapter_is_locked(sa));
+
+ sfc_log_init(sa, "unprobe nic");
+ efx_nic_unprobe(enp);
+
+ sfc_mcdi_fini(sa);
+
+ /*
+ * Make sure there is no pending alarm to restart since we are
+ * going to free device private which is passed as the callback
+ * opaque data. A new alarm cannot be scheduled since MCDI is
+ * shut down.
+ */
+ rte_eal_alarm_cancel(sfc_restart_if_required, sa);
+
+ sfc_mae_clear_switch_port(sa->mae.switch_domain_id,
+ sa->mae.switch_port_id);
+
+ sfc_log_init(sa, "destroy nic");
+ sa->nic = NULL;
+ efx_nic_destroy(enp);
+
+ sfc_mem_bar_fini(sa);
+
+ sfc_flow_fini(sa);
+ sa->state = SFC_ETHDEV_UNINITIALIZED;
+}
+
+uint32_t
+sfc_register_logtype(const struct rte_pci_addr *pci_addr,
+ const char *lt_prefix_str, uint32_t ll_default)
+{
+ size_t lt_prefix_str_size = strlen(lt_prefix_str);
+ size_t lt_str_size_max;
+ char *lt_str = NULL;
+ int ret;
+
+ if (SIZE_MAX - PCI_PRI_STR_SIZE - 1 > lt_prefix_str_size) {
+ ++lt_prefix_str_size; /* Reserve space for prefix separator */
+ lt_str_size_max = lt_prefix_str_size + PCI_PRI_STR_SIZE + 1;
+ } else {
+ return sfc_logtype_driver;
+ }
+
+ lt_str = rte_zmalloc("logtype_str", lt_str_size_max, 0);
+ if (lt_str == NULL)
+ return sfc_logtype_driver;
+
+ strncpy(lt_str, lt_prefix_str, lt_prefix_str_size);
+ lt_str[lt_prefix_str_size - 1] = '.';
+ rte_pci_device_name(pci_addr, lt_str + lt_prefix_str_size,
+ lt_str_size_max - lt_prefix_str_size);
+ lt_str[lt_str_size_max - 1] = '\0';
+
+ ret = rte_log_register_type_and_pick_level(lt_str, ll_default);
+ rte_free(lt_str);
+
+ if (ret < 0)
+ return sfc_logtype_driver;
+
+ return ret;
+}
+
+struct sfc_hw_switch_id {
+ char board_sn[RTE_SIZEOF_FIELD(efx_nic_board_info_t, enbi_serial)];
+};
+
+int
+sfc_hw_switch_id_init(struct sfc_adapter *sa,
+ struct sfc_hw_switch_id **idp)
+{
+ efx_nic_board_info_t board_info;
+ struct sfc_hw_switch_id *id;
+ int rc;
+
+ if (idp == NULL)
+ return EINVAL;
+
+ id = rte_zmalloc("sfc_hw_switch_id", sizeof(*id), 0);
+ if (id == NULL)
+ return ENOMEM;
+
+ rc = efx_nic_get_board_info(sa->nic, &board_info);
+ if (rc != 0)
+ return rc;
+
+ memcpy(id->board_sn, board_info.enbi_serial, sizeof(id->board_sn));
+
+ *idp = id;
+
+ return 0;
+}
+
+void
+sfc_hw_switch_id_fini(__rte_unused struct sfc_adapter *sa,
+ struct sfc_hw_switch_id *id)
+{
+ rte_free(id);
+}
+
+bool
+sfc_hw_switch_ids_equal(const struct sfc_hw_switch_id *left,
+ const struct sfc_hw_switch_id *right)
+{
+ return strncmp(left->board_sn, right->board_sn,
+ sizeof(left->board_sn)) == 0;