drivers/vdpa/mlx5/mlx5_vdpa_steer.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 Mellanox Technologies, Ltd
   3  */
   4 #include <netinet/in.h>
   5
   6 #include <rte_malloc.h>
   7 #include <rte_errno.h>
   8 #include <rte_common.h>
   9
  10 #include <mlx5_common.h>
  11
  12 #include "mlx5_vdpa_utils.h"
  13 #include "mlx5_vdpa.h"
  14
  15 int
  16 mlx5_vdpa_steer_unset(struct mlx5_vdpa_priv *priv)
  17 {
  18         int ret __rte_unused;
  19         unsigned i;
  20
  21         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
  22                 if (priv->steer.rss[i].flow) {
  23                         claim_zero(mlx5_glue->dv_destroy_flow
  24                                                      (priv->steer.rss[i].flow));
  25                         priv->steer.rss[i].flow = NULL;
  26                 }
  27                 if (priv->steer.rss[i].tir_action) {
  28                         claim_zero(mlx5_glue->destroy_flow_action
  29                                                (priv->steer.rss[i].tir_action));
  30                         priv->steer.rss[i].tir_action = NULL;
  31                 }
  32                 if (priv->steer.rss[i].tir) {
  33                         claim_zero(mlx5_devx_cmd_destroy
  34                                                       (priv->steer.rss[i].tir));
  35                         priv->steer.rss[i].tir = NULL;
  36                 }
  37                 if (priv->steer.rss[i].matcher) {
  38                         claim_zero(mlx5_glue->dv_destroy_flow_matcher
  39                                                   (priv->steer.rss[i].matcher));
  40                         priv->steer.rss[i].matcher = NULL;
  41                 }
  42         }
  43         if (priv->steer.tbl) {
  44                 claim_zero(mlx5_glue->dr_destroy_flow_tbl(priv->steer.tbl));
  45                 priv->steer.tbl = NULL;
  46         }
  47         if (priv->steer.domain) {
  48                 claim_zero(mlx5_glue->dr_destroy_domain(priv->steer.domain));
  49                 priv->steer.domain = NULL;
  50         }
  51         if (priv->steer.rqt) {
  52                 claim_zero(mlx5_devx_cmd_destroy(priv->steer.rqt));
  53                 priv->steer.rqt = NULL;
  54         }
  55         return 0;
  56 }
  57
  58 /*
  59  * According to VIRTIO_NET Spec the virtqueues index identity its type by:
  60  * 0 receiveq1
  61  * 1 transmitq1
  62  * ...
  63  * 2(N-1) receiveqN
  64  * 2(N-1)+1 transmitqN
  65  * 2N controlq
  66  */
  67 static uint8_t
  68 is_virtq_recvq(int virtq_index, int nr_vring)
  69 {
  70         if (virtq_index % 2 == 0 && virtq_index != nr_vring - 1)
  71                 return 1;
  72         return 0;
  73 }
  74
  75 #define MLX5_VDPA_DEFAULT_RQT_SIZE 512
  76 static int
  77 mlx5_vdpa_rqt_prepare(struct mlx5_vdpa_priv *priv)
  78 {
  79         struct mlx5_vdpa_virtq *virtq;
  80         uint32_t rqt_n = RTE_MIN(MLX5_VDPA_DEFAULT_RQT_SIZE,
  81                                  1 << priv->log_max_rqt_size);
  82         struct mlx5_devx_rqt_attr *attr = rte_zmalloc(__func__, sizeof(*attr)
  83                                                       + rqt_n *
  84                                                       sizeof(uint32_t), 0);
  85         uint32_t i = 0, j;
  86         int ret = 0;
  87
  88         if (!attr) {
  89                 DRV_LOG(ERR, "Failed to allocate RQT attributes memory.");
  90                 rte_errno = ENOMEM;
  91                 return -ENOMEM;
  92         }
  93         SLIST_FOREACH(virtq, &priv->virtq_list, next) {
  94                 if (is_virtq_recvq(virtq->index, priv->nr_virtqs) &&
  95                     virtq->enable) {
  96                         attr->rq_list[i] = virtq->virtq->id;
  97                         i++;
  98                 }
  99         }
 100         for (j = 0; i != rqt_n; ++i, ++j)
 101                 attr->rq_list[i] = attr->rq_list[j];
 102         attr->rq_type = MLX5_INLINE_Q_TYPE_VIRTQ;
 103         attr->rqt_max_size = rqt_n;
 104         attr->rqt_actual_size = rqt_n;
 105         if (!priv->steer.rqt) {
 106                 priv->steer.rqt = mlx5_devx_cmd_create_rqt(priv->ctx, attr);
 107                 if (!priv->steer.rqt) {
 108                         DRV_LOG(ERR, "Failed to create RQT.");
 109                         ret = -rte_errno;
 110                 }
 111         } else {
 112                 ret = mlx5_devx_cmd_modify_rqt(priv->steer.rqt, attr);
 113                 if (ret)
 114                         DRV_LOG(ERR, "Failed to modify RQT.");
 115         }
 116         rte_free(attr);
 117         return ret;
 118 }
 119
 120 int
 121 mlx5_vdpa_virtq_enable(struct mlx5_vdpa_virtq *virtq, int enable)
 122 {
 123         struct mlx5_vdpa_priv *priv = virtq->priv;
 124         int ret = 0;
 125
 126         DRV_LOG(INFO, "Update virtq %d status %sable -> %sable.", virtq->index,
 127                 virtq->enable ? "en" : "dis", enable ? "en" : "dis");
 128         if (virtq->enable == !!enable)
 129                 return 0;
 130         virtq->enable = !!enable;
 131         if (is_virtq_recvq(virtq->index, priv->nr_virtqs)) {
 132                 ret = mlx5_vdpa_rqt_prepare(priv);
 133                 if (ret)
 134                         virtq->enable = !enable;
 135         }
 136         return ret;
 137 }
 138
 139 static int __rte_unused
 140 mlx5_vdpa_rss_flows_create(struct mlx5_vdpa_priv *priv)
 141 {
 142 #ifdef HAVE_MLX5DV_DR
 143         struct mlx5_devx_tir_attr tir_att = {
 144                 .disp_type = MLX5_TIRC_DISP_TYPE_INDIRECT,
 145                 .rx_hash_fn = MLX5_RX_HASH_FN_TOEPLITZ,
 146                 .transport_domain = priv->td->id,
 147                 .indirect_table = priv->steer.rqt->id,
 148                 .rx_hash_symmetric = 1,
 149                 .rx_hash_toeplitz_key = { 0x2c, 0xc6, 0x81, 0xd1,
 150                                           0x5b, 0xdb, 0xf4, 0xf7,
 151                                           0xfc, 0xa2, 0x83, 0x19,
 152                                           0xdb, 0x1a, 0x3e, 0x94,
 153                                           0x6b, 0x9e, 0x38, 0xd9,
 154                                           0x2c, 0x9c, 0x03, 0xd1,
 155                                           0xad, 0x99, 0x44, 0xa7,
 156                                           0xd9, 0x56, 0x3d, 0x59,
 157                                           0x06, 0x3c, 0x25, 0xf3,
 158                                           0xfc, 0x1f, 0xdc, 0x2a },
 159         };
 160         struct {
 161                 size_t size;
 162                 /**< Size of match value. Do NOT split size and key! */
 163                 uint32_t buf[MLX5_ST_SZ_DW(fte_match_param)];
 164                 /**< Matcher value. This value is used as the mask or a key. */
 165         } matcher_mask = {
 166                                 .size = sizeof(matcher_mask.buf),
 167                         },
 168           matcher_value = {
 169                                 .size = sizeof(matcher_value.buf),
 170                         };
 171         struct mlx5dv_flow_matcher_attr dv_attr = {
 172                 .type = IBV_FLOW_ATTR_NORMAL,
 173                 .match_mask = (void *)&matcher_mask,
 174         };
 175         void *match_m = matcher_mask.buf;
 176         void *match_v = matcher_value.buf;
 177         void *headers_m = MLX5_ADDR_OF(fte_match_param, match_m, outer_headers);
 178         void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v, outer_headers);
 179         void *actions[1];
 180         const uint8_t l3_hash =
 181                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP) |
 182                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP);
 183         const uint8_t l4_hash =
 184                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT) |
 185                 (1 << MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT);
 186         enum { PRIO, CRITERIA, IP_VER_M, IP_VER_V, IP_PROT_M, IP_PROT_V, L3_BIT,
 187                L4_BIT, HASH, END};
 188         const uint8_t vars[RTE_DIM(priv->steer.rss)][END] = {
 189                 { 7, 0, 0, 0, 0, 0, 0, 0, 0 },
 190                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0, 0,
 191                  MLX5_L3_PROT_TYPE_IPV4, 0, l3_hash },
 192                 { 6, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0, 0,
 193                  MLX5_L3_PROT_TYPE_IPV6, 0, l3_hash },
 194                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 195                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_UDP,
 196                  l3_hash | l4_hash },
 197                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 4, 0xff,
 198                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV4, MLX5_L4_PROT_TYPE_TCP,
 199                  l3_hash | l4_hash },
 200                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 201                  IPPROTO_UDP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_UDP,
 202                  l3_hash | l4_hash },
 203                 { 5, 1 << MLX5_MATCH_CRITERIA_ENABLE_OUTER_BIT, 0xf, 6, 0xff,
 204                  IPPROTO_TCP, MLX5_L3_PROT_TYPE_IPV6, MLX5_L4_PROT_TYPE_TCP,
 205                  l3_hash | l4_hash },
 206         };
 207         unsigned i;
 208
 209         for (i = 0; i < RTE_DIM(priv->steer.rss); ++i) {
 210                 dv_attr.priority = vars[i][PRIO];
 211                 dv_attr.match_criteria_enable = vars[i][CRITERIA];
 212                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_version,
 213                          vars[i][IP_VER_M]);
 214                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_version,
 215                          vars[i][IP_VER_V]);
 216                 MLX5_SET(fte_match_set_lyr_2_4, headers_m, ip_protocol,
 217                          vars[i][IP_PROT_M]);
 218                 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
 219                          vars[i][IP_PROT_V]);
 220                 tir_att.rx_hash_field_selector_outer.l3_prot_type =
 221                                                                 vars[i][L3_BIT];
 222                 tir_att.rx_hash_field_selector_outer.l4_prot_type =
 223                                                                 vars[i][L4_BIT];
 224                 tir_att.rx_hash_field_selector_outer.selected_fields =
 225                                                                   vars[i][HASH];
 226                 priv->steer.rss[i].matcher = mlx5_glue->dv_create_flow_matcher
 227                                          (priv->ctx, &dv_attr, priv->steer.tbl);
 228                 if (!priv->steer.rss[i].matcher) {
 229                         DRV_LOG(ERR, "Failed to create matcher %d.", i);
 230                         goto error;
 231                 }
 232                 priv->steer.rss[i].tir = mlx5_devx_cmd_create_tir(priv->ctx,
 233                                                                   &tir_att);
 234                 if (!priv->steer.rss[i].tir) {
 235                         DRV_LOG(ERR, "Failed to create TIR %d.", i);
 236                         goto error;
 237                 }
 238                 priv->steer.rss[i].tir_action =
 239                                 mlx5_glue->dv_create_flow_action_dest_devx_tir
 240                                                   (priv->steer.rss[i].tir->obj);
 241                 if (!priv->steer.rss[i].tir_action) {
 242                         DRV_LOG(ERR, "Failed to create TIR action %d.", i);
 243                         goto error;
 244                 }
 245                 actions[0] = priv->steer.rss[i].tir_action;
 246                 priv->steer.rss[i].flow = mlx5_glue->dv_create_flow
 247                                         (priv->steer.rss[i].matcher,
 248                                          (void *)&matcher_value, 1, actions);
 249                 if (!priv->steer.rss[i].flow) {
 250                         DRV_LOG(ERR, "Failed to create flow %d.", i);
 251                         goto error;
 252                 }
 253         }
 254         return 0;
 255 error:
 256         /* Resources will be freed by the caller. */
 257         return -1;
 258 #else
 259         (void)priv;
 260         return -ENOTSUP;
 261 #endif /* HAVE_MLX5DV_DR */
 262 }
 263
 264 int
 265 mlx5_vdpa_steer_setup(struct mlx5_vdpa_priv *priv)
 266 {
 267 #ifdef HAVE_MLX5DV_DR
 268         if (mlx5_vdpa_rqt_prepare(priv))
 269                 return -1;
 270         priv->steer.domain = mlx5_glue->dr_create_domain(priv->ctx,
 271                                                   MLX5DV_DR_DOMAIN_TYPE_NIC_RX);
 272         if (!priv->steer.domain) {
 273                 DRV_LOG(ERR, "Failed to create Rx domain.");
 274                 goto error;
 275         }
 276         priv->steer.tbl = mlx5_glue->dr_create_flow_tbl(priv->steer.domain, 0);
 277         if (!priv->steer.tbl) {
 278                 DRV_LOG(ERR, "Failed to create table 0 with Rx domain.");
 279                 goto error;
 280         }
 281         if (mlx5_vdpa_rss_flows_create(priv))
 282                 goto error;
 283         return 0;
 284 error:
 285         mlx5_vdpa_steer_unset(priv);
 286         return -1;
 287 #else
 288         (void)priv;
 289         return -ENOTSUP;
 290 #endif /* HAVE_MLX5DV_DR */
 291 }