drivers/vdpa/mlx5/mlx5_vdpa_steer.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <netinet/in.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_errno.h>
   8 #include <rte_common.h>
   9
  10 #include <mlx5_common.h>
  11
  12 #include "mlx5_vdpa_utils.h"
  13 #include "mlx5_vdpa.h"
  14
  15 int
  16 mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv)
  17 {
  18         int ret __rte_unused;
  19         unsigned i;
  20
  21         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
  22                 if (priv->steer.rss[i].flow) {
  23                         claim_zero(mlx5_glue->dv_destroy_flow
  24                                                      (priv->steer.rss[i].flow));
  25                         priv->steer.rss[i].flow = NULL;
  26                 }
  27                 if (priv->steer.rss[i].tir_action) {
  28                         claim_zero(mlx5_glue->destroy_flow_action
  29                                                (priv->steer.rss[i].tir_action));
  30                         priv->steer.rss[i].tir_action = NULL;
  31                 }
  32                 if (priv->steer.rss[i].tir) {
  33                         claim_zero(mlx5_devx_cmd_destroy
  34                                                       (priv->steer.rss[i].tir));
  35                         priv->steer.rss[i].tir = NULL;
  36                 }
  37                 if (priv->steer.rss[i].matcher) {
  38                         claim_zero(mlx5_glue->dv_destroy_flow_matcher
  39                                                   (priv->steer.rss[i].matcher));
  40                         priv->steer.rss[i].matcher = NULL;
  41                 }
  42         }
  43         if (priv->steer.tbl) {
  44                 claim_zero(mlx5_glue->dr_destroy_flow_tbl(priv->steer.tbl));
  45                 priv->steer.tbl = NULL;
  46         }
  47         if (priv->steer.domain) {
  48                 claim_zero(mlx5_glue->dr_destroy_domain(priv->steer.domain));
  49                 priv->steer.domain = NULL;
  50         }
  51         if (priv->steer.rqt) {
  52                 claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt));
  53                 priv->steer.rqt = NULL;
  54         }
  55         return 0;
  56 }
  57
  58 /*
  59  * According to VIRTIO_NET Spec the virtqueues index identity its type by:
  60  * 0 receiveq1
  61  * 1 transmitq1
  62  * ...
  63  * 2(N-1) receiveqN
  64  * 2(N-1)+1 transmitqN
  65  * 2N controlq
  66  */
  67 static uint8_t
  68 is_virtq_recvq(int virtq_index, int nr_vring)
  69 {
  70         if (virtq_index % 2 == 0 && virtq_index != nr_vring - 1)
  71                 return 1;
  72         return 0;
  73 }
  74
  75 #define MLX5_VDPA_DEFAULT_RQT_SIZE 512
  76 static int
  77 mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv)
  78 {
  79         struct mlx5_vdpa_virtq *virtq;
  80         uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE,
  81                                  1 << priv->log_max_rqt_size);
  82         struct mlx5_devx_rqt_attr *attr = rte_zmalloc(__func__, sizeof(*attr)
  83                                                       + rqt_n *
  84                                                       sizeof(uint32_t), 0);
  85         uint32_t i = 0, j;
  86         int ret = 0;
  87
  88         if (!attr) {
  89                 DRV_LOG(ERR, "Failed to allocate RQT attributes memory.");
  90                 rte_errno = ENOMEM;
  91                 return -ENOMEM;
  92         }
  93         SLIST_FOREACH(virtq, &priv->virtq_list, next) {
  94                 if (is_virtq_recvq(virtq->index, priv->nr_virtqs) &&
  95                     virtq->enable) {
  96                         attr->rq_list[i] = virtq->virtq->id;
  97                         i++;
  98                 }
  99         }
 100         for (j = 0; i != rqt_n; ++i, ++j)
 101                 attr->rq_list[i] = attr->rq_list[j];
 102         attr->rq_type = MLX5_INLINE_Q_TYPE_VIRTQ;
 103         attr->rqt_max_size = rqt_n;
 104         attr->rqt_actual_size = rqt_n;
 105         if (!priv->steer.rqt) {
 106                 priv->steer.rqt = mlx5_devx_cmd_create_rqt(priv->ctx, attr);
 107                 if (!priv->steer.rqt) {
 108                         DRV_LOG(ERR, "Failed to create RQT.");
 109                         ret = -rte_errno;
 110                 }
 111         } else {
 112                 ret = mlx5_devx_cmd_modify_rqt(priv->steer.rqt, attr);
 113                 if (ret)
 114                         DRV_LOG(ERR, "Failed to modify RQT.");
 115         }
 116         rte_free(attr);
 117         return ret;
 118 }
 119
 120 int
 121 mlx5_vdpa_virtq_enable(struct mlx5_vdpa_virtq *virtq, int enable)
 122 {
 123         struct mlx5_vdpa_priv *priv = virtq->priv;
 124         int ret = 0;
 125
 126         if (virtq->enable == !!enable)
 127                 return 0;
 128         virtq->enable = !!enable;
 129         if (is_virtq_recvq(virtq->index, priv->nr_virtqs)) {
 130                 ret = mlx5_vdpa_rqt_prepare(priv);
 131                 if (ret)
 132                         virtq->enable = !enable;
 133         }
 134         return ret;
 135 }
 136
 137 static int __rte_unused
 138 mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv)
 139 {
 140 #ifdef HAVE_MLX5DV_DR
 141         struct mlx5_devx_tir_attr tir_att = {
 142                 .disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT,
 143                 .rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ,
 144                 .transport_domain = priv->td->id,
 145                 .indirect_table = priv->steer.rqt->id,
 146                 .rx_hash_symmetric = 1,
 147                 .rx_hash_toeplitz_key = { 0x2c, 0xc6, 0x81, 0xd1,
 148                                           0x5b, 0xdb, 0xf4, 0xf7,
 149                                           0xfc, 0xa2, 0x83, 0x19,
 150                                           0xdb, 0x1a, 0x3e, 0x94,
 151                                           0x6b, 0x9e, 0x38, 0xd9,
 152                                           0x2c, 0x9c, 0x03, 0xd1,
 153                                           0xad, 0x99, 0x44, 0xa7,
 154                                           0xd9, 0x56, 0x3d, 0x59,
 155                                           0x06, 0x3c, 0x25, 0xf3,
 156                                           0xfc, 0x1f, 0xdc, 0x2a },
 157         };
 158         struct {
 159                 size_t size;
 160                 /**< Size of match value. Do NOT split size and key! */
 161                 uint32_t buf[MLX5_ST_SZ_DW(fte_match_param)];
 162                 /**< Matcher value. This value is used as the mask or a key. */
 163         } matcher_mask = {
 164                                 .size = sizeof(matcher_mask.buf),
 165                         },
 166           matcher_value = {
 167                                 .size = sizeof(matcher_value.buf),
 168                         };
 169         struct mlx5dv_flow_matcher_attr dv_attr = {
 170                 .type = IBV_FLOW_ATTR_NORMAL,
 171                 .match_mask = (void *)&matcher_mask,
 172         };
 173         void *match_m = matcher_mask.buf;
 174         void *match_v = matcher_value.buf;
 175         void *headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers);
 176         void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers);
 177         void *actions[1];
 178         const uint8_t l3_hash =
 179                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) |
 180                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP);
 181         const uint8_t l4_hash =
 182                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT) |
 183                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT);
 184         enum { PRIO, CRITERIA, IP_VER_M, IP_VER_V, IP_PROT_M, IP_PROT_V, L3_BIT,
 185                L4_BIT, HASH, END};
 186         const uint8_t vars[RTE_DIM(priv->steer.rss)][END] = {
 187                 { 7, 0, 0, 0, 0, 0, 0, 0, 0 },
 188                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0, 0,
 189                  MLX5_L3_PROT_TYPE_IPV4, 0, l3_hash },
 190                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0, 0,
 191                  MLX5_L3_PROT_TYPE_IPV6, 0, l3_hash },
 192                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 193                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_UDP,
 194                  l3_hash | l4_hash },
 195                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 196                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_TCP,
 197                  l3_hash | l4_hash },
 198                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 199                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_UDP,
 200                  l3_hash | l4_hash },
 201                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 202                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_TCP,
 203                  l3_hash | l4_hash },
 204         };
 205         unsigned i;
 206
 207         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
 208                 dv_attr.priority = vars[i][PRIO];
 209                 dv_attr.match_criteria_enable = vars[i][CRITERIA];
 210                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
 211                          vars[i][IP_VER_M]);
 212                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
 213                          vars[i][IP_VER_V]);
 214                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 215                          vars[i][IP_PROT_M]);
 216                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 217                          vars[i][IP_PROT_V]);
 218                 tir_att.rx_hash_field_selector_outer.l3_prot_type =
 219                                                                 vars[i][L3_BIT];
 220                 tir_att.rx_hash_field_selector_outer.l4_prot_type =
 221                                                                 vars[i][L4_BIT];
 222                 tir_att.rx_hash_field_selector_outer.selected_fields =
 223                                                                   vars[i][HASH];
 224                 priv->steer.rss[i].matcher = mlx5_glue->dv_create_flow_matcher
 225                                          (priv->ctx, &dv_attr, priv->steer.tbl);
 226                 if (!priv->steer.rss[i].matcher) {
 227                         DRV_LOG(ERR, "Failed to create matcher %d.", i);
 228                         goto error;
 229                 }
 230                 priv->steer.rss[i].tir = mlx5_devx_cmd_create_tir(priv->ctx,
 231                                                                   &tir_att);
 232                 if (!priv->steer.rss[i].tir) {
 233                         DRV_LOG(ERR, "Failed to create TIR %d.", i);
 234                         goto error;
 235                 }
 236                 priv->steer.rss[i].tir_action =
 237                                 mlx5_glue->dv_create_flow_action_dest_devx_tir
 238                                                   (priv->steer.rss[i].tir->obj);
 239                 if (!priv->steer.rss[i].tir_action) {
 240                         DRV_LOG(ERR, "Failed to create TIR action %d.", i);
 241                         goto error;
 242                 }
 243                 actions[0] = priv->steer.rss[i].tir_action;
 244                 priv->steer.rss[i].flow = mlx5_glue->dv_create_flow
 245                                         (priv->steer.rss[i].matcher,
 246                                          (void *)&matcher_value, 1, actions);
 247                 if (!priv->steer.rss[i].flow) {
 248                         DRV_LOG(ERR, "Failed to create flow %d.", i);
 249                         goto error;
 250                 }
 251         }
 252         return 0;
 253 error:
 254         /* Resources will be freed by the caller. */
 255         return -1;
 256 #else
 257         (void)priv;
 258         return -ENOTSUP;
 259 #endif /* HAVE_MLX5DV_DR */
 260 }
 261
 262 int
 263 mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv)
 264 {
 265 #ifdef HAVE_MLX5DV_DR
 266         if (mlx5_vdpa_rqt_prepare(priv))
 267                 return -1;
 268         priv->steer.domain = mlx5_glue->dr_create_domain(priv->ctx,
 269                                                   MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
 270         if (!priv->steer.domain) {
 271                 DRV_LOG(ERR, "Failed to create Rx domain.");
 272                 goto error;
 273         }
 274         priv->steer.tbl = mlx5_glue->dr_create_flow_tbl(priv->steer.domain, 0);
 275         if (!priv->steer.tbl) {
 276                 DRV_LOG(ERR, "Failed to create table 0 with Rx domain.");
 277                 goto error;
 278         }
 279         if (mlx5_vdpa_rss_flows_create(priv))
 280                 goto error;
 281         return 0;
 282 error:
 283         mlx5_vdpa_steer_unset(priv);
 284         return -1;
 285 #else
 286         (void)priv;
 287         return -ENOTSUP;
 288 #endif /* HAVE_MLX5DV_DR */
 289 }