X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fvdpa%2Fmlx5%2Fmlx5_vdpa.h;h=2bbb868ec64cb8550b98f26a9c6de372f4635c31;hb=69e07f43a226499cf1ede5629616b251dae27734;hp=0edd68800f6ae978ca32a1868e6bed416a573c8e;hpb=7497873f23e73e4421ae44c1182c85bf98b0f95b;p=dpdk.git diff --git a/drivers/vdpa/mlx5/mlx5_vdpa.h b/drivers/vdpa/mlx5/mlx5_vdpa.h index 0edd68800f..2bbb868ec6 100644 --- a/drivers/vdpa/mlx5/mlx5_vdpa.h +++ b/drivers/vdpa/mlx5/mlx5_vdpa.h @@ -12,6 +12,7 @@ #pragma GCC diagnostic ignored "-Wpedantic" #endif #include +#include #include #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" @@ -21,6 +22,7 @@ #include #include +#include #include @@ -35,54 +37,94 @@ #define VIRTIO_F_RING_PACKED 34 #endif +#define MLX5_VDPA_DEFAULT_TIMER_DELAY_US 0u +#define MLX5_VDPA_DEFAULT_TIMER_STEP_US 1u + struct mlx5_vdpa_cq { uint16_t log_desc_n; uint32_t cq_ci:24; uint32_t arm_sn:2; + uint32_t armed:1; int callfd; rte_spinlock_t sl; - struct mlx5_devx_obj *cq; - struct mlx5dv_devx_umem *umem_obj; - union { - volatile void *umem_buf; - volatile struct mlx5_cqe *cqes; - }; - volatile uint32_t *db_rec; + struct mlx5_devx_cq cq_obj; uint64_t errors; }; struct mlx5_vdpa_event_qp { struct mlx5_vdpa_cq cq; struct mlx5_devx_obj *fw_qp; - struct mlx5_devx_obj *sw_qp; - struct mlx5dv_devx_umem *umem_obj; - void *umem_buf; - volatile uint32_t *db_rec; + struct mlx5_devx_qp sw_qp; + uint16_t qp_pi; }; struct mlx5_vdpa_query_mr { SLIST_ENTRY(mlx5_vdpa_query_mr) next; - void *addr; - uint64_t length; - struct mlx5dv_devx_umem *umem; - struct mlx5_devx_obj *mkey; + union { + struct ibv_mr *mr; + struct mlx5_devx_obj *mkey; + }; int is_indirect; }; +enum { + MLX5_VDPA_NOTIFIER_STATE_DISABLED, + MLX5_VDPA_NOTIFIER_STATE_ENABLED, + MLX5_VDPA_NOTIFIER_STATE_ERR +}; + +#define MLX5_VDPA_MAX_C_THRD 256 +#define MLX5_VDPA_MAX_TASKS_PER_THRD 4096 +#define MLX5_VDPA_TASKS_PER_DEV 64 + +/* Generic task information and size must be multiple of 4B. */ +struct mlx5_vdpa_task { + struct mlx5_vdpa_priv *priv; + uint32_t *remaining_cnt; + uint32_t *err_cnt; + uint32_t idx; +} __rte_packed __rte_aligned(4); + +/* Generic mlx5_vdpa_c_thread information. */ +struct mlx5_vdpa_c_thread { + pthread_t tid; + struct rte_ring *rng; + pthread_cond_t c_cond; +}; + +struct mlx5_vdpa_conf_thread_mng { + void *initializer_priv; + uint32_t refcnt; + uint32_t max_thrds; + pthread_mutex_t cthrd_lock; + struct mlx5_vdpa_c_thread cthrd[MLX5_VDPA_MAX_C_THRD]; +}; +extern struct mlx5_vdpa_conf_thread_mng conf_thread_mng; + struct mlx5_vdpa_virtq { SLIST_ENTRY(mlx5_vdpa_virtq) next; uint8_t enable; uint16_t index; uint16_t vq_size; + uint8_t notifier_state; + bool stopped; + uint32_t configured:1; + uint32_t version; + pthread_mutex_t virtq_lock; struct mlx5_vdpa_priv *priv; struct mlx5_devx_obj *virtq; + struct mlx5_devx_obj *counters; struct mlx5_vdpa_event_qp eqp; struct { struct mlx5dv_devx_umem *obj; void *buf; uint32_t size; } umems[3]; - struct rte_intr_handle intr_handle; + struct rte_intr_handle *intr_handle; + uint64_t err_time[3]; /* RDTSC time of recent errors. */ + uint32_t n_retry; + struct mlx5_devx_virtio_q_couners_attr stats; + struct mlx5_devx_virtio_q_couners_attr reset; }; struct mlx5_vdpa_steer { @@ -97,36 +139,90 @@ struct mlx5_vdpa_steer { } rss[7]; }; +enum { + MLX5_VDPA_EVENT_MODE_DYNAMIC_TIMER, + MLX5_VDPA_EVENT_MODE_FIXED_TIMER, + MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT +}; + +enum mlx5_dev_state { + MLX5_VDPA_STATE_PROBED = 0, + MLX5_VDPA_STATE_CONFIGURED, + MLX5_VDPA_STATE_IN_PROGRESS /* Shutting down. */ +}; + struct mlx5_vdpa_priv { TAILQ_ENTRY(mlx5_vdpa_priv) next; - uint8_t configured; - uint8_t direct_notifier; /* Whether direct notifier is on or off. */ - int id; /* vDPA device id. */ + bool connected; + bool use_c_thread; + enum mlx5_dev_state state; + rte_spinlock_t db_lock; + pthread_mutex_t steer_update_lock; + uint64_t no_traffic_counter; + pthread_t timer_tid; + int event_mode; + int event_core; /* Event thread cpu affinity core. */ + uint32_t event_us; + uint32_t timer_delay_us; + uint32_t no_traffic_max; + uint8_t hw_latency_mode; /* Hardware CQ moderation mode. */ + uint16_t hw_max_latency_us; /* Hardware CQ moderation period in usec. */ + uint16_t hw_max_pending_comp; /* Hardware CQ moderation counter. */ + uint16_t queue_size; /* virtq depth for pre-creating virtq resource */ + uint16_t queues; /* Max virtq pair for pre-creating virtq resource */ + struct rte_vdpa_device *vdev; /* vDPA device. */ + struct mlx5_common_device *cdev; /* Backend mlx5 device. */ int vid; /* vhost device id. */ - struct ibv_context *ctx; /* Device context. */ - struct rte_vdpa_dev_addr dev_addr; struct mlx5_hca_vdpa_attr caps; - uint32_t pdn; /* Protection Domain number. */ - struct ibv_pd *pd; uint32_t gpa_mkey_index; struct ibv_mr *null_mr; struct rte_vhost_memory *vmem; - uint32_t eqn; struct mlx5dv_devx_event_channel *eventc; - struct mlx5dv_devx_uar *uar; - struct rte_intr_handle intr_handle; + struct mlx5dv_devx_event_channel *err_chnl; + struct mlx5_uar uar; + struct rte_intr_handle *err_intr_handle; struct mlx5_devx_obj *td; - struct mlx5_devx_obj *tis; + struct mlx5_devx_obj *tiss[16]; /* TIS list for each LAG port. */ uint16_t nr_virtqs; + uint8_t num_lag_ports; uint64_t features; /* Negotiated features. */ uint16_t log_max_rqt_size; struct mlx5_vdpa_steer steer; struct mlx5dv_var *var; void *virtq_db_addr; + struct mlx5_pmd_wrapped_mr lm_mr; SLIST_HEAD(mr_list, mlx5_vdpa_query_mr) mr_list; struct mlx5_vdpa_virtq virtqs[]; }; +enum { + MLX5_VDPA_STATS_RECEIVED_DESCRIPTORS, + MLX5_VDPA_STATS_COMPLETED_DESCRIPTORS, + MLX5_VDPA_STATS_BAD_DESCRIPTOR_ERRORS, + MLX5_VDPA_STATS_EXCEED_MAX_CHAIN, + MLX5_VDPA_STATS_INVALID_BUFFER, + MLX5_VDPA_STATS_COMPLETION_ERRORS, + MLX5_VDPA_STATS_MAX +}; + +/* + * Check whether virtq is for traffic receive. + * According to VIRTIO_NET Spec the virtqueues index identity its type by: + * 0 receiveq1 + * 1 transmitq1 + * ... + * 2(N-1) receiveqN + * 2(N-1)+1 transmitqN + * 2N controlq + */ +static inline uint8_t +is_virtq_recvq(int virtq_index, int nr_vring) +{ + if (virtq_index % 2 == 0 && virtq_index != nr_vring - 1) + return 1; + return 0; +} + /** * Release all the prepared memory regions and all their related resources. * @@ -157,14 +253,15 @@ int mlx5_vdpa_mem_register(struct mlx5_vdpa_priv *priv); * Number of descriptors. * @param[in] callfd * The guest notification file descriptor. - * @param[in/out] eqp - * Pointer to the event QP structure. + * @param[in/out] virtq + * Pointer to the virt-queue structure. * * @return * 0 on success, -1 otherwise and rte_errno is set. */ -int mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n, - int callfd, struct mlx5_vdpa_event_qp *eqp); +int +mlx5_vdpa_event_qp_prepare(struct mlx5_vdpa_priv *priv, uint16_t desc_n, + int callfd, struct mlx5_vdpa_virtq *virtq); /** * Destroy an event QP and all its related resources. @@ -174,6 +271,15 @@ int mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n, */ void mlx5_vdpa_event_qp_destroy(struct mlx5_vdpa_event_qp *eqp); +/** + * Create all the event global resources. + * + * @param[in] priv + * The vdpa driver private structure. + */ +int +mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv); + /** * Release all the event global resources. * @@ -202,13 +308,40 @@ int mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv); void mlx5_vdpa_cqe_event_unset(struct mlx5_vdpa_priv *priv); /** - * Release a virtq and all its related resources. + * Setup error interrupt handler. + * + * @param[in] priv + * The vdpa driver private structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int mlx5_vdpa_err_event_setup(struct mlx5_vdpa_priv *priv); + +/** + * Unset error event handler. + * + * @param[in] priv + * The vdpa driver private structure. + */ +void mlx5_vdpa_err_event_unset(struct mlx5_vdpa_priv *priv); + +/** + * Release virtqs and resources except that to be reused. * * @param[in] priv * The vdpa driver private structure. */ void mlx5_vdpa_virtqs_release(struct mlx5_vdpa_priv *priv); +/** + * Cleanup cached resources of all virtqs. + * + * @param[in] priv + * The vdpa driver private structure. + */ +void mlx5_vdpa_virtqs_cleanup(struct mlx5_vdpa_priv *priv); + /** * Create all the HW virtqs resources and all their related resources. * @@ -223,23 +356,36 @@ int mlx5_vdpa_virtqs_prepare(struct mlx5_vdpa_priv *priv); /** * Enable\Disable virtq.. * - * @param[in] virtq - * The vdpa driver private virtq structure. + * @param[in] priv + * The vdpa driver private structure. + * @param[in] index + * The virtq index. * @param[in] enable * Set to enable, otherwise disable. * * @return * 0 on success, a negative value otherwise. */ -int mlx5_vdpa_virtq_enable(struct mlx5_vdpa_virtq *virtq, int enable); +int mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable); /** - * Unset steering and release all its related resources- stop traffic. + * Unset steering - stop traffic. * * @param[in] priv * The vdpa driver private structure. */ -int mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv); +void mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv); + +/** + * Update steering according to the received queues status. + * + * @param[in] priv + * The vdpa driver private structure. + * + * @return + * 0 on success, a negative value otherwise. + */ +int mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv); /** * Setup steering and all its related resources to enable RSS traffic from the @@ -321,4 +467,86 @@ int mlx5_vdpa_virtq_modify(struct mlx5_vdpa_virtq *virtq, int state); */ int mlx5_vdpa_virtq_stop(struct mlx5_vdpa_priv *priv, int index); +/** + * Query virtq information. + * + * @param[in] priv + * The vdpa driver private structure. + * @param[in] index + * The virtq index. + * + * @return + * 0 on success, a negative value otherwise. + */ +int mlx5_vdpa_virtq_query(struct mlx5_vdpa_priv *priv, int index); + +/** + * Get virtq statistics. + * + * @param[in] priv + * The vdpa driver private structure. + * @param[in] qid + * The virtq index. + * @param stats + * The virtq statistics array to fill. + * @param n + * The number of elements in @p stats array. + * + * @return + * A negative value on error, otherwise the number of entries filled in the + * @p stats array. + */ +int +mlx5_vdpa_virtq_stats_get(struct mlx5_vdpa_priv *priv, int qid, + struct rte_vdpa_stat *stats, unsigned int n); + +/** + * Reset virtq statistics. + * + * @param[in] priv + * The vdpa driver private structure. + * @param[in] qid + * The virtq index. + * + * @return + * A negative value on error, otherwise 0. + */ +int +mlx5_vdpa_virtq_stats_reset(struct mlx5_vdpa_priv *priv, int qid); + +/** + * Drain virtq CQ CQE. + * + * @param[in] priv + * The vdpa driver private structure. + */ +void +mlx5_vdpa_drain_cq(struct mlx5_vdpa_priv *priv); + +bool +mlx5_vdpa_is_modify_virtq_supported(struct mlx5_vdpa_priv *priv); + +/** + * Create configuration multi-threads resource + * + * @param[in] cpu_core + * CPU core number to set configuration threads affinity to. + * + * @return + * 0 on success, a negative value otherwise. + */ +int +mlx5_vdpa_mult_threads_create(int cpu_core); + +/** + * Destroy configuration multi-threads resource + * + */ +void +mlx5_vdpa_mult_threads_destroy(bool need_unlock); + +bool +mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv, + uint32_t thrd_idx, + uint32_t num); #endif /* RTE_PMD_MLX5_VDPA_H_ */