drivers/vdpa/mlx5/mlx5_vdpa_steer.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <netinet/in.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_errno.h>
   8 #include <rte_common.h>
   9
  10 #include <mlx5_common.h>
  11
  12 #include "mlx5_vdpa_utils.h"
  13 #include "mlx5_vdpa.h"
  14
  15 int
  16 mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv)
  17 {
  18         int ret __rte_unused;
  19         unsigned i;
  20
  21         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
  22                 if (priv->steer.rss[i].flow) {
  23                         claim_zero(mlx5_glue->dv_destroy_flow
  24                                                      (priv->steer.rss[i].flow));
  25                         priv->steer.rss[i].flow = NULL;
  26                 }
  27                 if (priv->steer.rss[i].tir_action) {
  28                         claim_zero(mlx5_glue->destroy_flow_action
  29                                                (priv->steer.rss[i].tir_action));
  30                         priv->steer.rss[i].tir_action = NULL;
  31                 }
  32                 if (priv->steer.rss[i].tir) {
  33                         claim_zero(mlx5_devx_cmd_destroy
  34                                                       (priv->steer.rss[i].tir));
  35                         priv->steer.rss[i].tir = NULL;
  36                 }
  37                 if (priv->steer.rss[i].matcher) {
  38                         claim_zero(mlx5_glue->dv_destroy_flow_matcher
  39                                                   (priv->steer.rss[i].matcher));
  40                         priv->steer.rss[i].matcher = NULL;
  41                 }
  42         }
  43         if (priv->steer.tbl) {
  44                 claim_zero(mlx5_glue->dr_destroy_flow_tbl(priv->steer.tbl));
  45                 priv->steer.tbl = NULL;
  46         }
  47         if (priv->steer.domain) {
  48                 claim_zero(mlx5_glue->dr_destroy_domain(priv->steer.domain));
  49                 priv->steer.domain = NULL;
  50         }
  51         if (priv->steer.rqt) {
  52                 claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt));
  53                 priv->steer.rqt = NULL;
  54         }
  55         return 0;
  56 }
  57
  58 /*
  59  * According to VIRTIO_NET Spec the virtqueues index identity its type by:
  60  * 0 receiveq1
  61  * 1 transmitq1
  62  * ...
  63  * 2(N-1) receiveqN
  64  * 2(N-1)+1 transmitqN
  65  * 2N controlq
  66  */
  67 static uint8_t
  68 is_virtq_recvq(int virtq_index, int nr_vring)
  69 {
  70         if (virtq_index % 2 == 0 && virtq_index != nr_vring - 1)
  71                 return 1;
  72         return 0;
  73 }
  74
  75 #define MLX5_VDPA_DEFAULT_RQT_SIZE 512
  76 static int
  77 mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv)
  78 {
  79         struct mlx5_vdpa_virtq *virtq;
  80         uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE,
  81                                  1 << priv->log_max_rqt_size);
  82         struct mlx5_devx_rqt_attr *attr = rte_zmalloc(__func__, sizeof(*attr)
  83                                                       + rqt_n *
  84                                                       sizeof(uint32_t), 0);
  85         uint32_t i = 0, j;
  86         int ret = 0;
  87
  88         if (!attr) {
  89                 DRV_LOG(ERR, "Failed to allocate RQT attributes memory.");
  90                 rte_errno = ENOMEM;
  91                 return -ENOMEM;
  92         }
  93         SLIST_FOREACH(virtq, &priv->virtq_list, next) {
  94                 if (is_virtq_recvq(virtq->index, priv->nr_virtqs) &&
  95                     virtq->enable) {
  96                         attr->rq_list[i] = virtq->virtq->id;
  97                         i++;
  98                 }
  99         }
 100         for (j = 0; i != rqt_n; ++i, ++j)
 101                 attr->rq_list[i] = attr->rq_list[j];
 102         attr->rq_type = MLX5_INLINE_Q_TYPE_VIRTQ;
 103         attr->rqt_max_size = rqt_n;
 104         attr->rqt_actual_size = rqt_n;
 105         if (!priv->steer.rqt) {
 106                 priv->steer.rqt = mlx5_devx_cmd_create_rqt(priv->ctx, attr);
 107                 if (!priv->steer.rqt) {
 108                         DRV_LOG(ERR, "Failed to create RQT.");
 109                         ret = -rte_errno;
 110                 }
 111         } else {
 112                 ret = mlx5_devx_cmd_modify_rqt(priv->steer.rqt, attr);
 113                 if (ret)
 114                         DRV_LOG(ERR, "Failed to modify RQT.");
 115         }
 116         rte_free(attr);
 117         return ret;
 118 }
 119
 120 int
 121 mlx5_vdpa_virtq_enable(struct mlx5_vdpa_virtq *virtq, int enable)
 122 {
 123         struct mlx5_vdpa_priv *priv = virtq->priv;
 124         int ret = 0;
 125
 126         if (virtq->enable == !!enable)
 127                 return 0;
 128         virtq->enable = !!enable;
 129         if (is_virtq_recvq(virtq->index, priv->nr_virtqs)) {
 130                 ret = mlx5_vdpa_rqt_prepare(priv);
 131                 if (ret)
 132                         virtq->enable = !enable;
 133         }
 134         return ret;
 135 }
 136
 137 static int __rte_unused
 138 mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv)
 139 {
 140 #ifdef HAVE_MLX5DV_DR
 141         struct mlx5_devx_tir_attr tir_att = {
 142                 .disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT,
 143                 .rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ,
 144                 .transport_domain = priv->td->id,
 145                 .indirect_table = priv->steer.rqt->id,
 146                 .rx_hash_symmetric = 1,
 147                 .rx_hash_toeplitz_key = { 0x2cc681d1, 0x5bdbf4f7, 0xfca28319,
 148                                           0xdb1a3e94, 0x6b9e38d9, 0x2c9c03d1,
 149                                           0xad9944a7, 0xd9563d59, 0x063c25f3,
 150                                           0xfc1fdc2a },
 151         };
 152         struct {
 153                 size_t size;
 154                 /**< Size of match value. Do NOT split size and key! */
 155                 uint32_t buf[MLX5_ST_SZ_DW(fte_match_param)];
 156                 /**< Matcher value. This value is used as the mask or a key. */
 157         } matcher_mask = {
 158                                 .size = sizeof(matcher_mask.buf),
 159                         },
 160           matcher_value = {
 161                                 .size = sizeof(matcher_value.buf),
 162                         };
 163         struct mlx5dv_flow_matcher_attr dv_attr = {
 164                 .type = IBV_FLOW_ATTR_NORMAL,
 165                 .match_mask = (void *)&matcher_mask,
 166         };
 167         void *match_m = matcher_mask.buf;
 168         void *match_v = matcher_value.buf;
 169         void *headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers);
 170         void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers);
 171         void *actions[1];
 172         const uint8_t l3_hash =
 173                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) |
 174                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP);
 175         const uint8_t l4_hash =
 176                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT) |
 177                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT);
 178         enum { PRIO, CRITERIA, IP_VER_M, IP_VER_V, IP_PROT_M, IP_PROT_V, L3_BIT,
 179                L4_BIT, HASH, END};
 180         const uint8_t vars[RTE_DIM(priv->steer.rss)][END] = {
 181                 { 7, 0, 0, 0, 0, 0, 0, 0, 0 },
 182                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0, 0,
 183                  MLX5_L3_PROT_TYPE_IPV4, 0, l3_hash },
 184                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0, 0,
 185                  MLX5_L3_PROT_TYPE_IPV6, 0, l3_hash },
 186                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 187                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_UDP,
 188                  l3_hash | l4_hash },
 189                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 190                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_TCP,
 191                  l3_hash | l4_hash },
 192                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 193                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_UDP,
 194                  l3_hash | l4_hash },
 195                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 196                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_TCP,
 197                  l3_hash | l4_hash },
 198         };
 199         unsigned i;
 200
 201         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
 202                 dv_attr.priority = vars[i][PRIO];
 203                 dv_attr.match_criteria_enable = vars[i][CRITERIA];
 204                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
 205                          vars[i][IP_VER_M]);
 206                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
 207                          vars[i][IP_VER_V]);
 208                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 209                          vars[i][IP_PROT_M]);
 210                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 211                          vars[i][IP_PROT_V]);
 212                 tir_att.rx_hash_field_selector_outer.l3_prot_type =
 213                                                                 vars[i][L3_BIT];
 214                 tir_att.rx_hash_field_selector_outer.l4_prot_type =
 215                                                                 vars[i][L4_BIT];
 216                 tir_att.rx_hash_field_selector_outer.selected_fields =
 217                                                                   vars[i][HASH];
 218                 priv->steer.rss[i].matcher = mlx5_glue->dv_create_flow_matcher
 219                                          (priv->ctx, &dv_attr, priv->steer.tbl);
 220                 if (!priv->steer.rss[i].matcher) {
 221                         DRV_LOG(ERR, "Failed to create matcher %d.", i);
 222                         goto error;
 223                 }
 224                 priv->steer.rss[i].tir = mlx5_devx_cmd_create_tir(priv->ctx,
 225                                                                   &tir_att);
 226                 if (!priv->steer.rss[i].tir) {
 227                         DRV_LOG(ERR, "Failed to create TIR %d.", i);
 228                         goto error;
 229                 }
 230                 priv->steer.rss[i].tir_action =
 231                                 mlx5_glue->dv_create_flow_action_dest_devx_tir
 232                                                   (priv->steer.rss[i].tir->obj);
 233                 if (!priv->steer.rss[i].tir_action) {
 234                         DRV_LOG(ERR, "Failed to create TIR action %d.", i);
 235                         goto error;
 236                 }
 237                 actions[0] = priv->steer.rss[i].tir_action;
 238                 priv->steer.rss[i].flow = mlx5_glue->dv_create_flow
 239                                         (priv->steer.rss[i].matcher,
 240                                          (void *)&matcher_value, 1, actions);
 241                 if (!priv->steer.rss[i].flow) {
 242                         DRV_LOG(ERR, "Failed to create flow %d.", i);
 243                         goto error;
 244                 }
 245         }
 246         return 0;
 247 error:
 248         /* Resources will be freed by the caller. */
 249         return -1;
 250 #else
 251         (void)priv;
 252         return -ENOTSUP;
 253 #endif /* HAVE_MLX5DV_DR */
 254 }
 255
 256 int
 257 mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv)
 258 {
 259 #ifdef HAVE_MLX5DV_DR
 260         if (mlx5_vdpa_rqt_prepare(priv))
 261                 return -1;
 262         priv->steer.domain = mlx5_glue->dr_create_domain(priv->ctx,
 263                                                   MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
 264         if (!priv->steer.domain) {
 265                 DRV_LOG(ERR, "Failed to create Rx domain.");
 266                 goto error;
 267         }
 268         priv->steer.tbl = mlx5_glue->dr_create_flow_tbl(priv->steer.domain, 0);
 269         if (!priv->steer.tbl) {
 270                 DRV_LOG(ERR, "Failed to create table 0 with Rx domain.");
 271                 goto error;
 272         }
 273         if (mlx5_vdpa_rss_flows_create(priv))
 274                 goto error;
 275         return 0;
 276 error:
 277         mlx5_vdpa_steer_unset(priv);
 278         return -1;
 279 #else
 280         (void)priv;
 281         return -ENOTSUP;
 282 #endif /* HAVE_MLX5DV_DR */
 283 }