drivers/vdpa/mlx5/mlx5_vdpa_steer.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <netinet/in.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_errno.h>
   8 #include <rte_common.h>
   9
  10 #include <mlx5_common.h>
  11
  12 #include "mlx5_vdpa_utils.h"
  13 #include "mlx5_vdpa.h"
  14
  15 static void
  16 mlx5_vdpa_rss_flows_destroy(struct mlx5_vdpa_priv *priv)
  17 {
  18         unsigned i;
  19
  20         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
  21                 if (priv->steer.rss[i].flow) {
  22                         claim_zero(mlx5_glue->dv_destroy_flow
  23                                                      (priv->steer.rss[i].flow));
  24                         priv->steer.rss[i].flow = NULL;
  25                 }
  26                 if (priv->steer.rss[i].tir_action) {
  27                         claim_zero(mlx5_glue->destroy_flow_action
  28                                                (priv->steer.rss[i].tir_action));
  29                         priv->steer.rss[i].tir_action = NULL;
  30                 }
  31                 if (priv->steer.rss[i].tir) {
  32                         claim_zero(mlx5_devx_cmd_destroy
  33                                                       (priv->steer.rss[i].tir));
  34                         priv->steer.rss[i].tir = NULL;
  35                 }
  36                 if (priv->steer.rss[i].matcher) {
  37                         claim_zero(mlx5_glue->dv_destroy_flow_matcher
  38                                                   (priv->steer.rss[i].matcher));
  39                         priv->steer.rss[i].matcher = NULL;
  40                 }
  41         }
  42 }
  43
  44 void
  45 mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv)
  46 {
  47         mlx5_vdpa_rss_flows_destroy(priv);
  48         if (priv->steer.rqt) {
  49                 claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt));
  50                 priv->steer.rqt = NULL;
  51         }
  52 }
  53
  54 #define MLX5_VDPA_DEFAULT_RQT_SIZE 512
  55 /*
  56  * Return the number of queues configured to the table on success, otherwise
  57  * -1 on error.
  58  */
  59 static int
  60 mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv)
  61 {
  62         int i;
  63         uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE,
  64                                  1 << priv->log_max_rqt_size);
  65         struct mlx5_devx_rqt_attr *attr = rte_zmalloc(__func__, sizeof(*attr)
  66                                                       + rqt_n *
  67                                                       sizeof(uint32_t), 0);
  68         uint32_t k = 0, j;
  69         int ret = 0, num;
  70
  71         if (!attr) {
  72                 DRV_LOG(ERR, "Failed to allocate RQT attributes memory.");
  73                 rte_errno = ENOMEM;
  74                 return -ENOMEM;
  75         }
  76         for (i = 0; i < priv->nr_virtqs; i++) {
  77                 if (is_virtq_recvq(i, priv->nr_virtqs) &&
  78                     priv->virtqs[i].enable && priv->virtqs[i].virtq) {
  79                         attr->rq_list[k] = priv->virtqs[i].virtq->id;
  80                         k++;
  81                 }
  82         }
  83         if (k == 0)
  84                 /* No enabled RQ to configure for RSS. */
  85                 return 0;
  86         num = (int)k;
  87         for (j = 0; k != rqt_n; ++k, ++j)
  88                 attr->rq_list[k] = attr->rq_list[j];
  89         attr->rq_type = MLX5_INLINE_Q_TYPE_VIRTQ;
  90         attr->rqt_max_size = rqt_n;
  91         attr->rqt_actual_size = rqt_n;
  92         if (!priv->steer.rqt) {
  93                 priv->steer.rqt = mlx5_devx_cmd_create_rqt(priv->cdev->ctx,
  94                                                            attr);
  95                 if (!priv->steer.rqt) {
  96                         DRV_LOG(ERR, "Failed to create RQT.");
  97                         ret = -rte_errno;
  98                 }
  99         } else {
 100                 ret = mlx5_devx_cmd_modify_rqt(priv->steer.rqt, attr);
 101                 if (ret)
 102                         DRV_LOG(ERR, "Failed to modify RQT.");
 103         }
 104         rte_free(attr);
 105         return ret ? -1 : num;
 106 }
 107
 108 static int __rte_unused
 109 mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv)
 110 {
 111 #ifdef HAVE_MLX5DV_DR
 112         struct mlx5_devx_tir_attr tir_att = {
 113                 .disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT,
 114                 .rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ,
 115                 .transport_domain = priv->td->id,
 116                 .indirect_table = priv->steer.rqt->id,
 117                 .rx_hash_symmetric = 1,
 118                 .rx_hash_toeplitz_key = { 0x2c, 0xc6, 0x81, 0xd1,
 119                                           0x5b, 0xdb, 0xf4, 0xf7,
 120                                           0xfc, 0xa2, 0x83, 0x19,
 121                                           0xdb, 0x1a, 0x3e, 0x94,
 122                                           0x6b, 0x9e, 0x38, 0xd9,
 123                                           0x2c, 0x9c, 0x03, 0xd1,
 124                                           0xad, 0x99, 0x44, 0xa7,
 125                                           0xd9, 0x56, 0x3d, 0x59,
 126                                           0x06, 0x3c, 0x25, 0xf3,
 127                                           0xfc, 0x1f, 0xdc, 0x2a },
 128         };
 129         struct {
 130                 size_t size;
 131                 /**< Size of match value. Do NOT split size and key! */
 132                 uint32_t buf[MLX5_ST_SZ_DW(fte_match_param)];
 133                 /**< Matcher value. This value is used as the mask or a key. */
 134         } matcher_mask = {
 135                                 .size = sizeof(matcher_mask.buf) -
 136                                         MLX5_ST_SZ_BYTES(fte_match_set_misc4) -
 137                                         MLX5_ST_SZ_BYTES(fte_match_set_misc5),
 138                         },
 139           matcher_value = {
 140                                 .size = sizeof(matcher_value.buf) -
 141                                         MLX5_ST_SZ_BYTES(fte_match_set_misc4) -
 142                                         MLX5_ST_SZ_BYTES(fte_match_set_misc5),
 143                         };
 144         struct mlx5dv_flow_matcher_attr dv_attr = {
 145                 .type = IBV_FLOW_ATTR_NORMAL,
 146                 .match_mask = (void *)&matcher_mask,
 147         };
 148         void *match_m = matcher_mask.buf;
 149         void *match_v = matcher_value.buf;
 150         void *headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers);
 151         void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers);
 152         void *actions[1];
 153         const uint8_t l3_hash =
 154                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) |
 155                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP);
 156         const uint8_t l4_hash =
 157                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT) |
 158                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT);
 159         enum { PRIO, CRITERIA, IP_VER_M, IP_VER_V, IP_PROT_M, IP_PROT_V, L3_BIT,
 160                L4_BIT, HASH, END};
 161         const uint8_t vars[RTE_DIM(priv->steer.rss)][END] = {
 162                 { 7, 0, 0, 0, 0, 0, 0, 0, 0 },
 163                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0, 0,
 164                  MLX5_L3_PROT_TYPE_IPV4, 0, l3_hash },
 165                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0, 0,
 166                  MLX5_L3_PROT_TYPE_IPV6, 0, l3_hash },
 167                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 168                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_UDP,
 169                  l3_hash | l4_hash },
 170                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 171                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_TCP,
 172                  l3_hash | l4_hash },
 173                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 174                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_UDP,
 175                  l3_hash | l4_hash },
 176                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 177                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_TCP,
 178                  l3_hash | l4_hash },
 179         };
 180         unsigned i;
 181
 182         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
 183                 dv_attr.priority = vars[i][PRIO];
 184                 dv_attr.match_criteria_enable = vars[i][CRITERIA];
 185                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
 186                          vars[i][IP_VER_M]);
 187                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
 188                          vars[i][IP_VER_V]);
 189                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 190                          vars[i][IP_PROT_M]);
 191                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 192                          vars[i][IP_PROT_V]);
 193                 tir_att.rx_hash_field_selector_outer.l3_prot_type =
 194                                                                 vars[i][L3_BIT];
 195                 tir_att.rx_hash_field_selector_outer.l4_prot_type =
 196                                                                 vars[i][L4_BIT];
 197                 tir_att.rx_hash_field_selector_outer.selected_fields =
 198                                                                   vars[i][HASH];
 199                 priv->steer.rss[i].matcher = mlx5_glue->dv_create_flow_matcher
 200                                    (priv->cdev->ctx, &dv_attr, priv->steer.tbl);
 201                 if (!priv->steer.rss[i].matcher) {
 202                         DRV_LOG(ERR, "Failed to create matcher %d.", i);
 203                         goto error;
 204                 }
 205                 priv->steer.rss[i].tir = mlx5_devx_cmd_create_tir
 206                                                     (priv->cdev->ctx, &tir_att);
 207                 if (!priv->steer.rss[i].tir) {
 208                         DRV_LOG(ERR, "Failed to create TIR %d.", i);
 209                         goto error;
 210                 }
 211                 priv->steer.rss[i].tir_action =
 212                                 mlx5_glue->dv_create_flow_action_dest_devx_tir
 213                                                   (priv->steer.rss[i].tir->obj);
 214                 if (!priv->steer.rss[i].tir_action) {
 215                         DRV_LOG(ERR, "Failed to create TIR action %d.", i);
 216                         goto error;
 217                 }
 218                 actions[0] = priv->steer.rss[i].tir_action;
 219                 priv->steer.rss[i].flow = mlx5_glue->dv_create_flow
 220                                         (priv->steer.rss[i].matcher,
 221                                          (void *)&matcher_value, 1, actions);
 222                 if (!priv->steer.rss[i].flow) {
 223                         DRV_LOG(ERR, "Failed to create flow %d.", i);
 224                         goto error;
 225                 }
 226         }
 227         return 0;
 228 error:
 229         /* Resources will be freed by the caller. */
 230         return -1;
 231 #else
 232         (void)priv;
 233         return -ENOTSUP;
 234 #endif /* HAVE_MLX5DV_DR */
 235 }
 236
 237 int
 238 mlx5_vdpa_steer_update(struct mlx5_vdpa_priv *priv)
 239 {
 240         int ret;
 241
 242         pthread_mutex_lock(&priv->steer_update_lock);
 243         ret = mlx5_vdpa_rqt_prepare(priv);
 244         if (ret == 0) {
 245                 mlx5_vdpa_steer_unset(priv);
 246         } else if (ret < 0) {
 247                 pthread_mutex_unlock(&priv->steer_update_lock);
 248                 return ret;
 249         } else if (!priv->steer.rss[0].flow) {
 250                 ret = mlx5_vdpa_rss_flows_create(priv);
 251                 if (ret) {
 252                         DRV_LOG(ERR, "Cannot create RSS flows.");
 253                         pthread_mutex_unlock(&priv->steer_update_lock);
 254                         return -1;
 255                 }
 256         }
 257         pthread_mutex_unlock(&priv->steer_update_lock);
 258         return 0;
 259 }
 260
 261 int
 262 mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv)
 263 {
 264         if (mlx5_vdpa_steer_update(priv))
 265                 goto error;
 266         return 0;
 267 error:
 268         mlx5_vdpa_steer_unset(priv);
 269         return -1;
 270 }