-/*-
- * BSD LICENSE
- *
- * Copyright 2012 6WIND S.A.
- * Copyright 2012 Mellanox
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of 6WIND S.A. nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2012 6WIND S.A.
+ * Copyright 2012 Mellanox Technologies, Ltd
*/
#ifndef RTE_PMD_MLX4_H_
#define RTE_PMD_MLX4_H_
+#include <net/if.h>
#include <stdint.h>
+#include <sys/queue.h>
-/*
- * Runtime logging through RTE_LOG() is enabled when not in debugging mode.
- * Intermediate LOG_*() macros add the required end-of-line characters.
- */
-#ifndef NDEBUG
-#define INFO(...) DEBUG(__VA_ARGS__)
-#define WARN(...) DEBUG(__VA_ARGS__)
-#define ERROR(...) DEBUG(__VA_ARGS__)
-#else
-#define LOG__(level, m, ...) \
- RTE_LOG(level, PMD, MLX4_DRIVER_NAME ": " m "%c", __VA_ARGS__)
-#define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n')
-#define INFO(...) LOG_(INFO, __VA_ARGS__)
-#define WARN(...) LOG_(WARNING, __VA_ARGS__)
-#define ERROR(...) LOG_(ERR, __VA_ARGS__)
-#endif
-
-/* Verbs header. */
-/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
+/* Verbs headers do not support -pedantic. */
#ifdef PEDANTIC
#pragma GCC diagnostic ignored "-Wpedantic"
#endif
#pragma GCC diagnostic error "-Wpedantic"
#endif
-/* Request send completion once in every 64 sends, might be less. */
+#include <rte_ethdev_driver.h>
+#include <rte_ether.h>
+#include <rte_interrupts.h>
+#include <rte_mempool.h>
+#include <rte_rwlock.h>
+
+#include "mlx4_mr.h"
+
+#ifndef IBV_RX_HASH_INNER
+/** This is not necessarily defined by supported RDMA core versions. */
+#define IBV_RX_HASH_INNER (1ull << 31)
+#endif /* IBV_RX_HASH_INNER */
+
+/** Maximum number of simultaneous MAC addresses. This value is arbitrary. */
+#define MLX4_MAX_MAC_ADDRESSES 128
+
+/** Request send completion once in every 64 sends, might be less. */
#define MLX4_PMD_TX_PER_COMP_REQ 64
-/* Maximum size for inline data. */
+/** Maximum size for inline data. */
#define MLX4_PMD_MAX_INLINE 0
-/*
- * Maximum number of cached Memory Pools (MPs) per TX queue. Each RTE MP
- * from which buffers are to be transmitted will have to be mapped by this
- * driver to their own Memory Region (MR). This is a slow operation.
- *
- * This value is always 1 for RX queues.
- */
-#ifndef MLX4_PMD_TX_MP_CACHE
-#define MLX4_PMD_TX_MP_CACHE 8
-#endif
+/** Fixed RSS hash key size in bytes. Cannot be modified. */
+#define MLX4_RSS_HASH_KEY_SIZE 40
+
+/** Interrupt alarm timeout value in microseconds. */
+#define MLX4_INTR_ALARM_TIMEOUT 100000
-/* Alarm timeout. */
-#define MLX4_ALARM_TIMEOUT_US 100000
+/* Maximum packet headers size (L2+L3+L4) for TSO. */
+#define MLX4_MAX_TSO_HEADER 192
-/* Port parameter. */
+/** Port parameter. */
#define MLX4_PMD_PORT_KVARG "port"
+/** Enable extending memsegs when creating a MR. */
+#define MLX4_MR_EXT_MEMSEG_EN_KVARG "mr_ext_memseg_en"
+
enum {
PCI_VENDOR_ID_MELLANOX = 0x15b3,
};
PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO = 0x1007,
};
+/* Request types for IPC. */
+enum mlx4_mp_req_type {
+ MLX4_MP_REQ_VERBS_CMD_FD = 1,
+ MLX4_MP_REQ_CREATE_MR,
+ MLX4_MP_REQ_START_RXTX,
+ MLX4_MP_REQ_STOP_RXTX,
+};
+
+/* Pameters for IPC. */
+struct mlx4_mp_param {
+ enum mlx4_mp_req_type type;
+ int port_id;
+ int result;
+ RTE_STD_C11
+ union {
+ uintptr_t addr; /* MLX4_MP_REQ_CREATE_MR */
+ } args;
+};
+
+/** Request timeout for IPC. */
+#define MLX4_MP_REQ_TIMEOUT_SEC 5
+
+/** Key string for IPC. */
+#define MLX4_MP_NAME "net_mlx4_mp"
+
+/** Driver name reported to lower layers and used in log output. */
#define MLX4_DRIVER_NAME "net_mlx4"
-/* Number of elements in array. */
-#define elemof(a) (sizeof(a) / sizeof((a)[0]))
-
-/* Debugging */
-#ifndef NDEBUG
-#include <stdio.h>
-#define DEBUG__(m, ...) \
- (fprintf(stderr, "%s:%d: %s(): " m "%c", \
- __FILE__, __LINE__, __func__, __VA_ARGS__), \
- fflush(stderr), \
- (void)0)
-/*
- * Save/restore errno around DEBUG__().
- * XXX somewhat undefined behavior, but works.
+struct mlx4_drop;
+struct mlx4_rss;
+struct rxq;
+struct txq;
+struct rte_flow;
+
+/**
+ * Type of objet being allocated.
*/
-#define DEBUG_(...) \
- (errno = ((int []){ \
- *(volatile int *)&errno, \
- (DEBUG__(__VA_ARGS__), 0) \
- })[0])
-#define DEBUG(...) DEBUG_(__VA_ARGS__, '\n')
-#ifndef MLX4_PMD_DEBUG_BROKEN_VERBS
-#define claim_zero(...) assert((__VA_ARGS__) == 0)
-#else /* MLX4_PMD_DEBUG_BROKEN_VERBS */
-#define claim_zero(...) \
- (void)(((__VA_ARGS__) == 0) || \
- DEBUG("Assertion `(" # __VA_ARGS__ ") == 0' failed (IGNORED)."))
-#endif /* MLX4_PMD_DEBUG_BROKEN_VERBS */
-#define claim_nonzero(...) assert((__VA_ARGS__) != 0)
-#define claim_positive(...) assert((__VA_ARGS__) >= 0)
-#else /* NDEBUG */
-/* No-ops. */
-#define DEBUG(...) (void)0
-#define claim_zero(...) (__VA_ARGS__)
-#define claim_nonzero(...) (__VA_ARGS__)
-#define claim_positive(...) (__VA_ARGS__)
-#endif /* NDEBUG */
-
-struct mlx4_rxq_stats {
- unsigned int idx; /**< Mapping index. */
- uint64_t ipackets; /**< Total of successfully received packets. */
- uint64_t ibytes; /**< Total of successfully received bytes. */
- uint64_t idropped; /**< Total of packets dropped when RX ring full. */
- uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */
+enum mlx4_verbs_alloc_type {
+ MLX4_VERBS_ALLOC_TYPE_NONE,
+ MLX4_VERBS_ALLOC_TYPE_TX_QUEUE,
+ MLX4_VERBS_ALLOC_TYPE_RX_QUEUE,
};
-/* RX element. */
-struct rxq_elt {
- struct ibv_recv_wr wr; /* Work Request. */
- struct ibv_sge sge; /* Scatter/Gather Element. */
- /* mbuf pointer is derived from WR_ID(wr.wr_id).offset. */
+/**
+ * Verbs allocator needs a context to know in the callback which kind of
+ * resources it is allocating.
+ */
+struct mlx4_verbs_alloc_ctx {
+ int enabled;
+ enum mlx4_verbs_alloc_type type; /* Kind of object being allocated. */
+ const void *obj; /* Pointer to the DPDK object. */
};
-/* RX queue descriptor. */
-struct rxq {
- struct priv *priv; /* Back pointer to private data. */
- struct rte_mempool *mp; /* Memory Pool for allocations. */
- struct ibv_mr *mr; /* Memory Region (for mp). */
- struct ibv_cq *cq; /* Completion Queue. */
- struct ibv_qp *qp; /* Queue Pair. */
- struct ibv_comp_channel *channel;
- unsigned int port_id; /* Port ID for incoming packets. */
- unsigned int elts_n; /* (*elts)[] length. */
- unsigned int elts_head; /* Current index in (*elts)[]. */
- struct rxq_elt (*elts)[]; /* Rx elements. */
- struct mlx4_rxq_stats stats; /* RX queue counters. */
- unsigned int socket; /* CPU socket ID for allocations. */
+LIST_HEAD(mlx4_dev_list, mlx4_priv);
+LIST_HEAD(mlx4_mr_list, mlx4_mr);
+
+/* Shared data between primary and secondary processes. */
+struct mlx4_shared_data {
+ rte_spinlock_t lock;
+ /* Global spinlock for primary and secondary processes. */
+ int init_done; /* Whether primary has done initialization. */
+ unsigned int secondary_cnt; /* Number of secondary processes init'd. */
+ struct mlx4_dev_list mem_event_cb_list;
+ rte_rwlock_t mem_event_rwlock;
};
-/* TX element. */
-struct txq_elt {
- struct ibv_send_wr wr; /* Work request. */
- struct ibv_sge sge; /* Scatter/gather element. */
- struct rte_mbuf *buf;
+/* Per-process data structure, not visible to other processes. */
+struct mlx4_local_data {
+ int init_done; /* Whether a secondary has done initialization. */
};
-struct mlx4_txq_stats {
- unsigned int idx; /**< Mapping index. */
- uint64_t opackets; /**< Total of successfully sent packets. */
- uint64_t obytes; /**< Total of successfully sent bytes. */
- uint64_t odropped; /**< Total of packets not sent when TX ring full. */
-};
+extern struct mlx4_shared_data *mlx4_shared_data;
-/* TX queue descriptor. */
-struct txq {
- struct priv *priv; /* Back pointer to private data. */
- struct {
- const struct rte_mempool *mp; /* Cached Memory Pool. */
- struct ibv_mr *mr; /* Memory Region (for mp). */
- uint32_t lkey; /* mr->lkey */
- } mp2mr[MLX4_PMD_TX_MP_CACHE]; /* MP to MR translation table. */
- struct ibv_cq *cq; /* Completion Queue. */
- struct ibv_qp *qp; /* Queue Pair. */
- uint32_t max_inline; /* Max inline send size <= MLX4_PMD_MAX_INLINE. */
- unsigned int elts_n; /* (*elts)[] length. */
- struct txq_elt (*elts)[]; /* TX elements. */
- unsigned int elts_head; /* Current index in (*elts)[]. */
- unsigned int elts_tail; /* First element awaiting completion. */
- unsigned int elts_comp; /* Number of completion requests. */
- unsigned int elts_comp_cd; /* Countdown for next completion request. */
- unsigned int elts_comp_cd_init; /* Initial value for countdown. */
- struct mlx4_txq_stats stats; /* TX queue counters. */
- unsigned int socket; /* CPU socket ID for allocations. */
+/* Per-process private structure. */
+struct mlx4_proc_priv {
+ size_t uar_table_sz;
+ /* Size of UAR register table. */
+ void *uar_table[];
+ /* Table of UAR registers for each process. */
};
-struct rte_flow;
-
-struct priv {
- struct rte_eth_dev *dev; /* Ethernet device. */
- struct ibv_context *ctx; /* Verbs context. */
- struct ibv_device_attr device_attr; /* Device properties. */
- struct ibv_pd *pd; /* Protection Domain. */
- struct ether_addr mac; /* MAC address. */
- struct ibv_flow *mac_flow; /* Flow associated with MAC address. */
+#define MLX4_PROC_PRIV(port_id) \
+ ((struct mlx4_proc_priv *)rte_eth_devices[port_id].process_private)
+
+/** Private data structure. */
+struct mlx4_priv {
+ LIST_ENTRY(mlx4_priv) mem_event_cb;
+ /**< Called by memory event callback. */
+ struct rte_eth_dev_data *dev_data; /* Pointer to device data. */
+ struct ibv_context *ctx; /**< Verbs context. */
+ struct ibv_device_attr device_attr; /**< Device properties. */
+ struct ibv_pd *pd; /**< Protection Domain. */
/* Device properties. */
- uint16_t mtu; /* Configured MTU. */
- uint8_t port; /* Physical port number. */
- unsigned int started:1; /* Device started, flows enabled. */
- unsigned int vf:1; /* This is a VF device. */
- unsigned int pending_alarm:1; /* An alarm is pending. */
- unsigned int isolated:1; /* Toggle isolated mode. */
- /* RX/TX queues. */
- unsigned int rxqs_n; /* RX queues array size. */
- unsigned int txqs_n; /* TX queues array size. */
- struct rxq *(*rxqs)[]; /* RX queues. */
- struct txq *(*txqs)[]; /* TX queues. */
- struct rte_intr_handle intr_handle_dev; /* Device interrupt handler. */
- struct rte_intr_handle intr_handle; /* Interrupt handler. */
- struct rte_flow_drop *flow_drop_queue; /* Flow drop queue. */
- LIST_HEAD(mlx4_flows, rte_flow) flows;
- struct rte_intr_conf intr_conf; /* Active interrupt configuration. */
- rte_spinlock_t lock; /* Lock for control functions. */
+ uint16_t mtu; /**< Configured MTU. */
+ uint8_t port; /**< Physical port number. */
+ uint32_t started:1; /**< Device started, flows enabled. */
+ uint32_t vf:1; /**< This is a VF device. */
+ uint32_t intr_alarm:1; /**< An interrupt alarm is scheduled. */
+ uint32_t isolated:1; /**< Toggle isolated mode. */
+ uint32_t rss_init:1; /**< Common RSS context is initialized. */
+ uint32_t hw_csum:1; /**< Checksum offload is supported. */
+ uint32_t hw_csum_l2tun:1; /**< Checksum support for L2 tunnels. */
+ uint32_t hw_fcs_strip:1; /**< FCS stripping toggling is supported. */
+ uint32_t tso:1; /**< Transmit segmentation offload is supported. */
+ uint32_t mr_ext_memseg_en:1;
+ /** Whether memseg should be extended for MR creation. */
+ uint32_t tso_max_payload_sz; /**< Max supported TSO payload size. */
+ uint32_t hw_rss_max_qps; /**< Max Rx Queues supported by RSS. */
+ uint64_t hw_rss_sup; /**< Supported RSS hash fields (Verbs format). */
+ struct rte_intr_handle intr_handle; /**< Port interrupt handle. */
+ struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */
+ struct {
+ uint32_t dev_gen; /* Generation number to flush local caches. */
+ rte_rwlock_t rwlock; /* MR Lock. */
+ struct mlx4_mr_btree cache; /* Global MR cache table. */
+ struct mlx4_mr_list mr_list; /* Registered MR list. */
+ struct mlx4_mr_list mr_free_list; /* Freed MR list. */
+ } mr;
+ LIST_HEAD(, mlx4_rss) rss; /**< Shared targets for Rx flow rules. */
+ LIST_HEAD(, rte_flow) flows; /**< Configured flow rule handles. */
+ struct rte_ether_addr mac[MLX4_MAX_MAC_ADDRESSES];
+ /**< Configured MAC addresses. Unused entries are zeroed. */
+ uint32_t mac_mc; /**< Number of trailing multicast entries in mac[]. */
+ struct mlx4_verbs_alloc_ctx verbs_alloc_ctx;
+ /**< Context for Verbs allocator. */
};
-void priv_lock(struct priv *priv);
-void priv_unlock(struct priv *priv);
+#define PORT_ID(priv) ((priv)->dev_data->port_id)
+#define ETH_DEV(priv) (&rte_eth_devices[PORT_ID(priv)])
+
+/* mlx4_ethdev.c */
+
+int mlx4_get_ifname(const struct mlx4_priv *priv, char (*ifname)[IF_NAMESIZE]);
+int mlx4_get_mac(struct mlx4_priv *priv, uint8_t (*mac)[RTE_ETHER_ADDR_LEN]);
+int mlx4_mtu_get(struct mlx4_priv *priv, uint16_t *mtu);
+int mlx4_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
+int mlx4_dev_set_link_down(struct rte_eth_dev *dev);
+int mlx4_dev_set_link_up(struct rte_eth_dev *dev);
+void mlx4_promiscuous_enable(struct rte_eth_dev *dev);
+void mlx4_promiscuous_disable(struct rte_eth_dev *dev);
+void mlx4_allmulticast_enable(struct rte_eth_dev *dev);
+void mlx4_allmulticast_disable(struct rte_eth_dev *dev);
+void mlx4_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
+int mlx4_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
+ uint32_t index, uint32_t vmdq);
+int mlx4_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr);
+int mlx4_set_mc_addr_list(struct rte_eth_dev *dev, struct rte_ether_addr *list,
+ uint32_t num);
+int mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
+int mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
+void mlx4_stats_reset(struct rte_eth_dev *dev);
+int mlx4_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size);
+void mlx4_dev_infos_get(struct rte_eth_dev *dev,
+ struct rte_eth_dev_info *info);
+int mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete);
+int mlx4_flow_ctrl_get(struct rte_eth_dev *dev,
+ struct rte_eth_fc_conf *fc_conf);
+int mlx4_flow_ctrl_set(struct rte_eth_dev *dev,
+ struct rte_eth_fc_conf *fc_conf);
+const uint32_t *mlx4_dev_supported_ptypes_get(struct rte_eth_dev *dev);
+int mlx4_is_removed(struct rte_eth_dev *dev);
+
+/* mlx4_intr.c */
+
+int mlx4_intr_uninstall(struct mlx4_priv *priv);
+int mlx4_intr_install(struct mlx4_priv *priv);
+int mlx4_rxq_intr_enable(struct mlx4_priv *priv);
+void mlx4_rxq_intr_disable(struct mlx4_priv *priv);
+int mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx);
+int mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx);
+
+/* mlx4_mp.c */
+void mlx4_mp_req_start_rxtx(struct rte_eth_dev *dev);
+void mlx4_mp_req_stop_rxtx(struct rte_eth_dev *dev);
+int mlx4_mp_req_mr_create(struct rte_eth_dev *dev, uintptr_t addr);
+int mlx4_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev);
+void mlx4_mp_init_primary(void);
+void mlx4_mp_uninit_primary(void);
+void mlx4_mp_init_secondary(void);
+void mlx4_mp_uninit_secondary(void);
#endif /* RTE_PMD_MLX4_H_ */
git.droids-corp.org / dpdk.git / blobdiff