drivers/vdpa/mlx5/mlx5_vdpa_steer.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <netinet/in.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_errno.h>
   8 #include <rte_common.h>
   9
  10 #include <mlx5_common.h>
  11
  12 #include "mlx5_vdpa_utils.h"
  13 #include "mlx5_vdpa.h"
  14
  15 static void
  16 mlx5_vdpa_rss_flows_destroy(struct mlx5_vdpa_priv *priv)
  17 {
  18         unsigned i;
  19
  20         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
  21                 if (priv->steer.rss[i].flow) {
  22                         claim_zero(mlx5_glue->dv_destroy_flow
  23                                                      (priv->steer.rss[i].flow));
  24                         priv->steer.rss[i].flow = NULL;
  25                 }
  26                 if (priv->steer.rss[i].tir_action) {
  27                         claim_zero(mlx5_glue->destroy_flow_action
  28                                                (priv->steer.rss[i].tir_action));
  29                         priv->steer.rss[i].tir_action = NULL;
  30                 }
  31                 if (priv->steer.rss[i].tir) {
  32                         claim_zero(mlx5_devx_cmd_destroy
  33                                                       (priv->steer.rss[i].tir));
  34                         priv->steer.rss[i].tir = NULL;
  35                 }
  36                 if (priv->steer.rss[i].matcher) {
  37                         claim_zero(mlx5_glue->dv_destroy_flow_matcher
  38                                                   (priv->steer.rss[i].matcher));
  39                         priv->steer.rss[i].matcher = NULL;
  40                 }
  41         }
  42 }
  43
  44 void
  45 mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv)
  46 {
  47         mlx5_vdpa_rss_flows_destroy(priv);
  48         if (priv->steer.tbl) {
  49                 claim_zero(mlx5_glue->dr_destroy_flow_tbl(priv->steer.tbl));
  50                 priv->steer.tbl = NULL;
  51         }
  52         if (priv->steer.domain) {
  53                 claim_zero(mlx5_glue->dr_destroy_domain(priv->steer.domain));
  54                 priv->steer.domain = NULL;
  55         }
  56         if (priv->steer.rqt) {
  57                 claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt));
  58                 priv->steer.rqt = NULL;
  59         }
  60 }
  61
  62 #define MLX5_VDPA_DEFAULT_RQT_SIZE 512
  63 /*
  64  * Return the number of queues configured to the table on success, otherwise
  65  * -1 on error.
  66  */
  67 static int
  68 mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv)
  69 {
  70         int i;
  71         uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE,
  72                                  1 << priv->log_max_rqt_size);
  73         struct mlx5_devx_rqt_attr *attr = rte_zmalloc(__func__, sizeof(*attr)
  74                                                       + rqt_n *
  75                                                       sizeof(uint32_t), 0);
  76         uint32_t k = 0, j;
  77         int ret = 0, num;
  78
  79         if (!attr) {
  80                 DRV_LOG(ERR, "Failed to allocate RQT attributes memory.");
  81                 rte_errno = ENOMEM;
  82                 return -ENOMEM;
  83         }
  84         for (i = 0; i < priv->nr_virtqs; i++) {
  85                 if (is_virtq_recvq(i, priv->nr_virtqs) &&
  86                     priv->virtqs[i].enable && priv->virtqs[i].virtq) {
  87                         attr->rq_list[k] = priv->virtqs[i].virtq->id;
  88                         k++;
  89                 }
  90         }
  91         if (k == 0)
  92                 /* No enabled RQ to configure for RSS. */
  93                 return 0;
  94         num = (int)k;
  95         for (j = 0; k != rqt_n; ++k, ++j)
  96                 attr->rq_list[k] = attr->rq_list[j];
  97         attr->rq_type = MLX5_INLINE_Q_TYPE_VIRTQ;
  98         attr->rqt_max_size = rqt_n;
  99         attr->rqt_actual_size = rqt_n;
 100         if (!priv->steer.rqt) {
 101                 priv->steer.rqt = mlx5_devx_cmd_create_rqt(priv->cdev->ctx,
 102                                                            attr);
 103                 if (!priv->steer.rqt) {
 104                         DRV_LOG(ERR, "Failed to create RQT.");
 105                         ret = -rte_errno;
 106                 }
 107         } else {
 108                 ret = mlx5_devx_cmd_modify_rqt(priv->steer.rqt, attr);
 109                 if (ret)
 110                         DRV_LOG(ERR, "Failed to modify RQT.");
 111         }
 112         rte_free(attr);
 113         return ret ? -1 : num;
 114 }
 115
 116 static int __rte_unused
 117 mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv)
 118 {
 119 #ifdef HAVE_MLX5DV_DR
 120         struct mlx5_devx_tir_attr tir_att = {
 121                 .disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT,
 122                 .rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ,
 123                 .transport_domain = priv->td->id,
 124                 .indirect_table = priv->steer.rqt->id,
 125                 .rx_hash_symmetric = 1,
 126                 .rx_hash_toeplitz_key = { 0x2c, 0xc6, 0x81, 0xd1,
 127                                           0x5b, 0xdb, 0xf4, 0xf7,
 128                                           0xfc, 0xa2, 0x83, 0x19,
 129                                           0xdb, 0x1a, 0x3e, 0x94,
 130                                           0x6b, 0x9e, 0x38, 0xd9,
 131                                           0x2c, 0x9c, 0x03, 0xd1,
 132                                           0xad, 0x99, 0x44, 0xa7,
 133                                           0xd9, 0x56, 0x3d, 0x59,
 134                                           0x06, 0x3c, 0x25, 0xf3,
 135                                           0xfc, 0x1f, 0xdc, 0x2a },
 136         };
 137         struct {
 138                 size_t size;
 139                 /**< Size of match value. Do NOT split size and key! */
 140                 uint32_t buf[MLX5_ST_SZ_DW(fte_match_param)];
 141                 /**< Matcher value. This value is used as the mask or a key. */
 142         } matcher_mask = {
 143                                 .size = sizeof(matcher_mask.buf) -
 144                                         MLX5_ST_SZ_BYTES(fte_match_set_misc4) -
 145                                         MLX5_ST_SZ_BYTES(fte_match_set_misc5),
 146                         },
 147           matcher_value = {
 148                                 .size = sizeof(matcher_value.buf) -
 149                                         MLX5_ST_SZ_BYTES(fte_match_set_misc4) -
 150                                         MLX5_ST_SZ_BYTES(fte_match_set_misc5),
 151                         };
 152         struct mlx5dv_flow_matcher_attr dv_attr = {
 153                 .type = IBV_FLOW_ATTR_NORMAL,
 154                 .match_mask = (void *)&matcher_mask,
 155         };
 156         void *match_m = matcher_mask.buf;
 157         void *match_v = matcher_value.buf;
 158         void *headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers);
 159         void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers);
 160         void *actions[1];
 161         const uint8_t l3_hash =
 162                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) |
 163                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP);
 164         const uint8_t l4_hash =
 165                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT) |
 166                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT);
 167         enum { PRIO, CRITERIA, IP_VER_M, IP_VER_V, IP_PROT_M, IP_PROT_V, L3_BIT,
 168                L4_BIT, HASH, END};
 169         const uint8_t vars[RTE_DIM(priv->steer.rss)][END] = {
 170                 { 7, 0, 0, 0, 0, 0, 0, 0, 0 },
 171                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0, 0,
 172                  MLX5_L3_PROT_TYPE_IPV4, 0, l3_hash },
 173                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0, 0,
 174                  MLX5_L3_PROT_TYPE_IPV6, 0, l3_hash },
 175                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 176                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_UDP,
 177                  l3_hash | l4_hash },
 178                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 179                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_TCP,
 180                  l3_hash | l4_hash },
 181                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 182                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_UDP,
 183                  l3_hash | l4_hash },
 184                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 185                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_TCP,
 186                  l3_hash | l4_hash },
 187         };
 188         unsigned i;
 189
 190         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
 191                 dv_attr.priority = vars[i][PRIO];
 192                 dv_attr.match_criteria_enable = vars[i][CRITERIA];
 193                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
 194                          vars[i][IP_VER_M]);
 195                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
 196                          vars[i][IP_VER_V]);
 197                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 198                          vars[i][IP_PROT_M]);
 199                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 200                          vars[i][IP_PROT_V]);
 201                 tir_att.rx_hash_field_selector_outer.l3_prot_type =
 202                                                                 vars[i][L3_BIT];
 203                 tir_att.rx_hash_field_selector_outer.l4_prot_type =
 204                                                                 vars[i][L4_BIT];
 205                 tir_att.rx_hash_field_selector_outer.selected_fields =
 206                                                                   vars[i][HASH];
 207                 priv->steer.rss[i].matcher = mlx5_glue->dv_create_flow_matcher
 208                                    (priv->cdev->ctx, &dv_attr, priv->steer.tbl);
 209                 if (!priv->steer.rss[i].matcher) {
 210                         DRV_LOG(ERR, "Failed to create matcher %d.", i);
 211                         goto error;
 212                 }
 213                 priv->steer.rss[i].tir = mlx5_devx_cmd_create_tir
 214                                                     (priv->cdev->ctx, &tir_att);
 215                 if (!priv->steer.rss[i].tir) {
 216                         DRV_LOG(ERR, "Failed to create TIR %d.", i);
 217                         goto error;
 218                 }
 219                 priv->steer.rss[i].tir_action =
 220                                 mlx5_glue->dv_create_flow_action_dest_devx_tir
 221                                                   (priv->steer.rss[i].tir->obj);
 222                 if (!priv->steer.rss[i].tir_action) {
 223                         DRV_LOG(ERR, "Failed to create TIR action %d.", i);
 224                         goto error;
 225                 }
 226                 actions[0] = priv->steer.rss[i].tir_action;
 227                 priv->steer.rss[i].flow = mlx5_glue->dv_create_flow
 228                                         (priv->steer.rss[i].matcher,
 229                                          (void *)&matcher_value, 1, actions);
 230                 if (!priv->steer.rss[i].flow) {
 231                         DRV_LOG(ERR, "Failed to create flow %d.", i);
 232                         goto error;
 233                 }
 234         }
 235         return 0;
 236 error:
 237         /* Resources will be freed by the caller. */
 238         return -1;
 239 #else
 240         (void)priv;
 241         return -ENOTSUP;
 242 #endif /* HAVE_MLX5DV_DR */
 243 }
 244
 245 int
 246 mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv)
 247 {
 248         int ret = mlx5_vdpa_rqt_prepare(priv);
 249
 250         if (ret == 0) {
 251                 mlx5_vdpa_rss_flows_destroy(priv);
 252                 if (priv->steer.rqt) {
 253                         claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt));
 254                         priv->steer.rqt = NULL;
 255                 }
 256         } else if (ret < 0) {
 257                 return ret;
 258         } else if (!priv->steer.rss[0].flow) {
 259                 ret = mlx5_vdpa_rss_flows_create(priv);
 260                 if (ret) {
 261                         DRV_LOG(ERR, "Cannot create RSS flows.");
 262                         return -1;
 263                 }
 264         }
 265         return 0;
 266 }
 267
 268 int
 269 mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv)
 270 {
 271 #ifdef HAVE_MLX5DV_DR
 272         priv->steer.domain = mlx5_glue->dr_create_domain(priv->cdev->ctx,
 273                                                   MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
 274         if (!priv->steer.domain) {
 275                 DRV_LOG(ERR, "Failed to create Rx domain.");
 276                 goto error;
 277         }
 278         priv->steer.tbl = mlx5_glue->dr_create_flow_tbl(priv->steer.domain, 0);
 279         if (!priv->steer.tbl) {
 280                 DRV_LOG(ERR, "Failed to create table 0 with Rx domain.");
 281                 goto error;
 282         }
 283         if (mlx5_vdpa_steer_update(priv))
 284                 goto error;
 285         return 0;
 286 error:
 287         mlx5_vdpa_steer_unset(priv);
 288         return -1;
 289 #else
 290         (void)priv;
 291         return -ENOTSUP;
 292 #endif /* HAVE_MLX5DV_DR */
 293 }