drivers/net/mlx5/mlx5_flow_hw.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2022 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_flow.h>
   6
   7 #include <mlx5_malloc.h>
   8 #include "mlx5_defs.h"
   9 #include "mlx5_flow.h"
  10 #include "mlx5_rx.h"
  11
  12 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
  13
  14 /* The maximum actions support in the flow. */
  15 #define MLX5_HW_MAX_ACTS 16
  16
  17 /* Default push burst threshold. */
  18 #define BURST_THR 32u
  19
  20 /* Default queue to flush the flows. */
  21 #define MLX5_DEFAULT_FLUSH_QUEUE 0
  22
  23 const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops;
  24
  25 /* DR action flags with different table. */
  26 static uint32_t mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_MAX]
  27                                 [MLX5DR_TABLE_TYPE_MAX] = {
  28         {
  29                 MLX5DR_ACTION_FLAG_ROOT_RX,
  30                 MLX5DR_ACTION_FLAG_ROOT_TX,
  31                 MLX5DR_ACTION_FLAG_ROOT_FDB,
  32         },
  33         {
  34                 MLX5DR_ACTION_FLAG_HWS_RX,
  35                 MLX5DR_ACTION_FLAG_HWS_TX,
  36                 MLX5DR_ACTION_FLAG_HWS_FDB,
  37         },
  38 };
  39
  40 /**
  41  * Register destination table DR jump action.
  42  *
  43  * @param[in] dev
  44  *   Pointer to the rte_eth_dev structure.
  45  * @param[in] table_attr
  46  *   Pointer to the flow attributes.
  47  * @param[in] dest_group
  48  *   The destination group ID.
  49  * @param[out] error
  50  *   Pointer to error structure.
  51  *
  52  * @return
  53  *    Table on success, NULL otherwise and rte_errno is set.
  54  */
  55 static struct mlx5_hw_jump_action *
  56 flow_hw_jump_action_register(struct rte_eth_dev *dev,
  57                              const struct rte_flow_attr *attr,
  58                              uint32_t dest_group,
  59                              struct rte_flow_error *error)
  60 {
  61         struct mlx5_priv *priv = dev->data->dev_private;
  62         struct rte_flow_attr jattr = *attr;
  63         struct mlx5_flow_group *grp;
  64         struct mlx5_flow_cb_ctx ctx = {
  65                 .dev = dev,
  66                 .error = error,
  67                 .data = &jattr,
  68         };
  69         struct mlx5_list_entry *ge;
  70
  71         jattr.group = dest_group;
  72         ge = mlx5_hlist_register(priv->sh->flow_tbls, dest_group, &ctx);
  73         if (!ge)
  74                 return NULL;
  75         grp = container_of(ge, struct mlx5_flow_group, entry);
  76         return &grp->jump;
  77 }
  78
  79 /**
  80  * Release jump action.
  81  *
  82  * @param[in] dev
  83  *   Pointer to the rte_eth_dev structure.
  84  * @param[in] jump
  85  *   Pointer to the jump action.
  86  */
  87
  88 static void
  89 flow_hw_jump_release(struct rte_eth_dev *dev, struct mlx5_hw_jump_action *jump)
  90 {
  91         struct mlx5_priv *priv = dev->data->dev_private;
  92         struct mlx5_flow_group *grp;
  93
  94         grp = container_of
  95                 (jump, struct mlx5_flow_group, jump);
  96         mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
  97 }
  98
  99 /**
 100  * Register queue/RSS action.
 101  *
 102  * @param[in] dev
 103  *   Pointer to the rte_eth_dev structure.
 104  * @param[in] hws_flags
 105  *   DR action flags.
 106  * @param[in] action
 107  *   rte flow action.
 108  *
 109  * @return
 110  *    Table on success, NULL otherwise and rte_errno is set.
 111  */
 112 static inline struct mlx5_hrxq*
 113 flow_hw_tir_action_register(struct rte_eth_dev *dev,
 114                             uint32_t hws_flags,
 115                             const struct rte_flow_action *action)
 116 {
 117         struct mlx5_flow_rss_desc rss_desc = {
 118                 .hws_flags = hws_flags,
 119         };
 120         struct mlx5_hrxq *hrxq;
 121
 122         if (action->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
 123                 const struct rte_flow_action_queue *queue = action->conf;
 124
 125                 rss_desc.const_q = &queue->index;
 126                 rss_desc.queue_num = 1;
 127         } else {
 128                 const struct rte_flow_action_rss *rss = action->conf;
 129
 130                 rss_desc.queue_num = rss->queue_num;
 131                 rss_desc.const_q = rss->queue;
 132                 memcpy(rss_desc.key,
 133                        !rss->key ? rss_hash_default_key : rss->key,
 134                        MLX5_RSS_HASH_KEY_LEN);
 135                 rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
 136                 rss_desc.types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
 137                 flow_dv_hashfields_set(0, &rss_desc, &rss_desc.hash_fields);
 138                 flow_dv_action_rss_l34_hash_adjust(rss->types,
 139                                                    &rss_desc.hash_fields);
 140                 if (rss->level > 1) {
 141                         rss_desc.hash_fields |= IBV_RX_HASH_INNER;
 142                         rss_desc.tunnel = 1;
 143                 }
 144         }
 145         hrxq = mlx5_hrxq_get(dev, &rss_desc);
 146         return hrxq;
 147 }
 148
 149 /**
 150  * Destroy DR actions created by action template.
 151  *
 152  * For DR actions created during table creation's action translate.
 153  * Need to destroy the DR action when destroying the table.
 154  *
 155  * @param[in] dev
 156  *   Pointer to the rte_eth_dev structure.
 157  * @param[in] acts
 158  *   Pointer to the template HW steering DR actions.
 159  */
 160 static void
 161 __flow_hw_action_template_destroy(struct rte_eth_dev *dev,
 162                                  struct mlx5_hw_actions *acts)
 163 {
 164         struct mlx5_priv *priv = dev->data->dev_private;
 165
 166         if (acts->jump) {
 167                 struct mlx5_flow_group *grp;
 168
 169                 grp = container_of
 170                         (acts->jump, struct mlx5_flow_group, jump);
 171                 mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
 172                 acts->jump = NULL;
 173         }
 174 }
 175
 176 /**
 177  * Append dynamic action to the dynamic action list.
 178  *
 179  * @param[in] priv
 180  *   Pointer to the port private data structure.
 181  * @param[in] acts
 182  *   Pointer to the template HW steering DR actions.
 183  * @param[in] type
 184  *   Action type.
 185  * @param[in] action_src
 186  *   Offset of source rte flow action.
 187  * @param[in] action_dst
 188  *   Offset of destination DR action.
 189  *
 190  * @return
 191  *    0 on success, negative value otherwise and rte_errno is set.
 192  */
 193 static __rte_always_inline struct mlx5_action_construct_data *
 194 __flow_hw_act_data_alloc(struct mlx5_priv *priv,
 195                          enum rte_flow_action_type type,
 196                          uint16_t action_src,
 197                          uint16_t action_dst)
 198 {
 199         struct mlx5_action_construct_data *act_data;
 200         uint32_t idx = 0;
 201
 202         act_data = mlx5_ipool_zmalloc(priv->acts_ipool, &idx);
 203         if (!act_data)
 204                 return NULL;
 205         act_data->idx = idx;
 206         act_data->type = type;
 207         act_data->action_src = action_src;
 208         act_data->action_dst = action_dst;
 209         return act_data;
 210 }
 211
 212 /**
 213  * Append dynamic action to the dynamic action list.
 214  *
 215  * @param[in] priv
 216  *   Pointer to the port private data structure.
 217  * @param[in] acts
 218  *   Pointer to the template HW steering DR actions.
 219  * @param[in] type
 220  *   Action type.
 221  * @param[in] action_src
 222  *   Offset of source rte flow action.
 223  * @param[in] action_dst
 224  *   Offset of destination DR action.
 225  *
 226  * @return
 227  *    0 on success, negative value otherwise and rte_errno is set.
 228  */
 229 static __rte_always_inline int
 230 __flow_hw_act_data_general_append(struct mlx5_priv *priv,
 231                                   struct mlx5_hw_actions *acts,
 232                                   enum rte_flow_action_type type,
 233                                   uint16_t action_src,
 234                                   uint16_t action_dst)
 235 {       struct mlx5_action_construct_data *act_data;
 236
 237         act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
 238         if (!act_data)
 239                 return -1;
 240         LIST_INSERT_HEAD(&acts->act_list, act_data, next);
 241         return 0;
 242 }
 243
 244 /**
 245  * Translate rte_flow actions to DR action.
 246  *
 247  * As the action template has already indicated the actions. Translate
 248  * the rte_flow actions to DR action if possbile. So in flow create
 249  * stage we will save cycles from handing the actions' organizing.
 250  * For the actions with limited information, need to add these to a
 251  * list.
 252  *
 253  * @param[in] dev
 254  *   Pointer to the rte_eth_dev structure.
 255  * @param[in] table_attr
 256  *   Pointer to the table attributes.
 257  * @param[in] item_templates
 258  *   Item template array to be binded to the table.
 259  * @param[in/out] acts
 260  *   Pointer to the template HW steering DR actions.
 261  * @param[in] at
 262  *   Action template.
 263  * @param[out] error
 264  *   Pointer to error structure.
 265  *
 266  * @return
 267  *    Table on success, NULL otherwise and rte_errno is set.
 268  */
 269 static int
 270 flow_hw_actions_translate(struct rte_eth_dev *dev,
 271                           const struct rte_flow_template_table_attr *table_attr,
 272                           struct mlx5_hw_actions *acts,
 273                           struct rte_flow_actions_template *at,
 274                           struct rte_flow_error *error)
 275 {
 276         struct mlx5_priv *priv = dev->data->dev_private;
 277         const struct rte_flow_attr *attr = &table_attr->flow_attr;
 278         struct rte_flow_action *actions = at->actions;
 279         struct rte_flow_action *action_start = actions;
 280         struct rte_flow_action *masks = at->masks;
 281         bool actions_end = false;
 282         uint32_t type, i;
 283         int err;
 284
 285         if (attr->transfer)
 286                 type = MLX5DR_TABLE_TYPE_FDB;
 287         else if (attr->egress)
 288                 type = MLX5DR_TABLE_TYPE_NIC_TX;
 289         else
 290                 type = MLX5DR_TABLE_TYPE_NIC_RX;
 291         for (i = 0; !actions_end; actions++, masks++) {
 292                 switch (actions->type) {
 293                 case RTE_FLOW_ACTION_TYPE_INDIRECT:
 294                         break;
 295                 case RTE_FLOW_ACTION_TYPE_VOID:
 296                         break;
 297                 case RTE_FLOW_ACTION_TYPE_DROP:
 298                         acts->rule_acts[i++].action =
 299                                 priv->hw_drop[!!attr->group][type];
 300                         break;
 301                 case RTE_FLOW_ACTION_TYPE_JUMP:
 302                         if (masks->conf) {
 303                                 uint32_t jump_group =
 304                                         ((const struct rte_flow_action_jump *)
 305                                         actions->conf)->group;
 306                                 acts->jump = flow_hw_jump_action_register
 307                                                 (dev, attr, jump_group, error);
 308                                 if (!acts->jump)
 309                                         goto err;
 310                                 acts->rule_acts[i].action = (!!attr->group) ?
 311                                                 acts->jump->hws_action :
 312                                                 acts->jump->root_action;
 313                         } else if (__flow_hw_act_data_general_append
 314                                         (priv, acts, actions->type,
 315                                          actions - action_start, i)){
 316                                 goto err;
 317                         }
 318                         i++;
 319                         break;
 320                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 321                         if (masks->conf) {
 322                                 acts->tir = flow_hw_tir_action_register
 323                                 (dev,
 324                                  mlx5_hw_act_flag[!!attr->group][type],
 325                                  actions);
 326                                 if (!acts->tir)
 327                                         goto err;
 328                                 acts->rule_acts[i].action =
 329                                         acts->tir->action;
 330                         } else if (__flow_hw_act_data_general_append
 331                                         (priv, acts, actions->type,
 332                                          actions - action_start, i)) {
 333                                 goto err;
 334                         }
 335                         i++;
 336                         break;
 337                 case RTE_FLOW_ACTION_TYPE_RSS:
 338                         if (masks->conf) {
 339                                 acts->tir = flow_hw_tir_action_register
 340                                 (dev,
 341                                  mlx5_hw_act_flag[!!attr->group][type],
 342                                  actions);
 343                                 if (!acts->tir)
 344                                         goto err;
 345                                 acts->rule_acts[i].action =
 346                                         acts->tir->action;
 347                         } else if (__flow_hw_act_data_general_append
 348                                         (priv, acts, actions->type,
 349                                          actions - action_start, i)) {
 350                                 goto err;
 351                         }
 352                         i++;
 353                         break;
 354                 case RTE_FLOW_ACTION_TYPE_END:
 355                         actions_end = true;
 356                         break;
 357                 default:
 358                         break;
 359                 }
 360         }
 361         acts->acts_num = i;
 362         return 0;
 363 err:
 364         err = rte_errno;
 365         __flow_hw_action_template_destroy(dev, acts);
 366         return rte_flow_error_set(error, err,
 367                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 368                                   "fail to create rte table");
 369 }
 370
 371 /**
 372  * Construct flow action array.
 373  *
 374  * For action template contains dynamic actions, these actions need to
 375  * be updated according to the rte_flow action during flow creation.
 376  *
 377  * @param[in] dev
 378  *   Pointer to the rte_eth_dev structure.
 379  * @param[in] job
 380  *   Pointer to job descriptor.
 381  * @param[in] hw_acts
 382  *   Pointer to translated actions from template.
 383  * @param[in] actions
 384  *   Array of rte_flow action need to be checked.
 385  * @param[in] rule_acts
 386  *   Array of DR rule actions to be used during flow creation..
 387  * @param[in] acts_num
 388  *   Pointer to the real acts_num flow has.
 389  *
 390  * @return
 391  *    0 on success, negative value otherwise and rte_errno is set.
 392  */
 393 static __rte_always_inline int
 394 flow_hw_actions_construct(struct rte_eth_dev *dev,
 395                           struct mlx5_hw_q_job *job,
 396                           struct mlx5_hw_actions *hw_acts,
 397                           const struct rte_flow_action actions[],
 398                           struct mlx5dr_rule_action *rule_acts,
 399                           uint32_t *acts_num)
 400 {
 401         struct rte_flow_template_table *table = job->flow->table;
 402         struct mlx5_action_construct_data *act_data;
 403         const struct rte_flow_action *action;
 404         struct rte_flow_attr attr = {
 405                         .ingress = 1,
 406         };
 407         uint32_t ft_flag;
 408
 409         memcpy(rule_acts, hw_acts->rule_acts,
 410                sizeof(*rule_acts) * hw_acts->acts_num);
 411         *acts_num = hw_acts->acts_num;
 412         if (LIST_EMPTY(&hw_acts->act_list))
 413                 return 0;
 414         attr.group = table->grp->group_id;
 415         ft_flag = mlx5_hw_act_flag[!!table->grp->group_id][table->type];
 416         if (table->type == MLX5DR_TABLE_TYPE_FDB) {
 417                 attr.transfer = 1;
 418                 attr.ingress = 1;
 419         } else if (table->type == MLX5DR_TABLE_TYPE_NIC_TX) {
 420                 attr.egress = 1;
 421                 attr.ingress = 0;
 422         } else {
 423                 attr.ingress = 1;
 424         }
 425         LIST_FOREACH(act_data, &hw_acts->act_list, next) {
 426                 uint32_t jump_group;
 427                 struct mlx5_hw_jump_action *jump;
 428                 struct mlx5_hrxq *hrxq;
 429
 430                 action = &actions[act_data->action_src];
 431                 MLX5_ASSERT(action->type == RTE_FLOW_ACTION_TYPE_INDIRECT ||
 432                             (int)action->type == act_data->type);
 433                 switch (action->type) {
 434                 case RTE_FLOW_ACTION_TYPE_INDIRECT:
 435                         break;
 436                 case RTE_FLOW_ACTION_TYPE_VOID:
 437                         break;
 438                 case RTE_FLOW_ACTION_TYPE_JUMP:
 439                         jump_group = ((const struct rte_flow_action_jump *)
 440                                                 action->conf)->group;
 441                         jump = flow_hw_jump_action_register
 442                                 (dev, &attr, jump_group, NULL);
 443                         if (!jump)
 444                                 return -1;
 445                         rule_acts[act_data->action_dst].action =
 446                         (!!attr.group) ? jump->hws_action : jump->root_action;
 447                         job->flow->jump = jump;
 448                         job->flow->fate_type = MLX5_FLOW_FATE_JUMP;
 449                         break;
 450                 case RTE_FLOW_ACTION_TYPE_RSS:
 451                 case RTE_FLOW_ACTION_TYPE_QUEUE:
 452                         hrxq = flow_hw_tir_action_register(dev,
 453                                         ft_flag,
 454                                         action);
 455                         if (!hrxq)
 456                                 return -1;
 457                         rule_acts[act_data->action_dst].action = hrxq->action;
 458                         job->flow->hrxq = hrxq;
 459                         job->flow->fate_type = MLX5_FLOW_FATE_QUEUE;
 460                         break;
 461                 default:
 462                         break;
 463                 }
 464         }
 465         return 0;
 466 }
 467
 468 /**
 469  * Enqueue HW steering flow creation.
 470  *
 471  * The flow will be applied to the HW only if the postpone bit is not set or
 472  * the extra push function is called.
 473  * The flow creation status should be checked from dequeue result.
 474  *
 475  * @param[in] dev
 476  *   Pointer to the rte_eth_dev structure.
 477  * @param[in] queue
 478  *   The queue to create the flow.
 479  * @param[in] attr
 480  *   Pointer to the flow operation attributes.
 481  * @param[in] items
 482  *   Items with flow spec value.
 483  * @param[in] pattern_template_index
 484  *   The item pattern flow follows from the table.
 485  * @param[in] actions
 486  *   Action with flow spec value.
 487  * @param[in] action_template_index
 488  *   The action pattern flow follows from the table.
 489  * @param[in] user_data
 490  *   Pointer to the user_data.
 491  * @param[out] error
 492  *   Pointer to error structure.
 493  *
 494  * @return
 495  *    Flow pointer on success, NULL otherwise and rte_errno is set.
 496  */
 497 static struct rte_flow *
 498 flow_hw_async_flow_create(struct rte_eth_dev *dev,
 499                           uint32_t queue,
 500                           const struct rte_flow_op_attr *attr,
 501                           struct rte_flow_template_table *table,
 502                           const struct rte_flow_item items[],
 503                           uint8_t pattern_template_index,
 504                           const struct rte_flow_action actions[],
 505                           uint8_t action_template_index,
 506                           void *user_data,
 507                           struct rte_flow_error *error)
 508 {
 509         struct mlx5_priv *priv = dev->data->dev_private;
 510         struct mlx5dr_rule_attr rule_attr = {
 511                 .queue_id = queue,
 512                 .user_data = user_data,
 513                 .burst = attr->postpone,
 514         };
 515         struct mlx5dr_rule_action rule_acts[MLX5_HW_MAX_ACTS];
 516         struct mlx5_hw_actions *hw_acts;
 517         struct rte_flow_hw *flow;
 518         struct mlx5_hw_q_job *job;
 519         uint32_t acts_num, flow_idx;
 520         int ret;
 521
 522         if (unlikely(!priv->hw_q[queue].job_idx)) {
 523                 rte_errno = ENOMEM;
 524                 goto error;
 525         }
 526         flow = mlx5_ipool_zmalloc(table->flow, &flow_idx);
 527         if (!flow)
 528                 goto error;
 529         /*
 530          * Set the table here in order to know the destination table
 531          * when free the flow afterwards.
 532          */
 533         flow->table = table;
 534         flow->idx = flow_idx;
 535         job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
 536         /*
 537          * Set the job type here in order to know if the flow memory
 538          * should be freed or not when get the result from dequeue.
 539          */
 540         job->type = MLX5_HW_Q_JOB_TYPE_CREATE;
 541         job->flow = flow;
 542         job->user_data = user_data;
 543         rule_attr.user_data = job;
 544         hw_acts = &table->ats[action_template_index].acts;
 545         /* Construct the flow action array based on the input actions.*/
 546         flow_hw_actions_construct(dev, job, hw_acts, actions,
 547                                   rule_acts, &acts_num);
 548         ret = mlx5dr_rule_create(table->matcher,
 549                                  pattern_template_index, items,
 550                                  rule_acts, acts_num,
 551                                  &rule_attr, &flow->rule);
 552         if (likely(!ret))
 553                 return (struct rte_flow *)flow;
 554         /* Flow created fail, return the descriptor and flow memory. */
 555         mlx5_ipool_free(table->flow, flow_idx);
 556         priv->hw_q[queue].job_idx++;
 557 error:
 558         rte_flow_error_set(error, rte_errno,
 559                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 560                            "fail to create rte flow");
 561         return NULL;
 562 }
 563
 564 /**
 565  * Enqueue HW steering flow destruction.
 566  *
 567  * The flow will be applied to the HW only if the postpone bit is not set or
 568  * the extra push function is called.
 569  * The flow destruction status should be checked from dequeue result.
 570  *
 571  * @param[in] dev
 572  *   Pointer to the rte_eth_dev structure.
 573  * @param[in] queue
 574  *   The queue to destroy the flow.
 575  * @param[in] attr
 576  *   Pointer to the flow operation attributes.
 577  * @param[in] flow
 578  *   Pointer to the flow to be destroyed.
 579  * @param[in] user_data
 580  *   Pointer to the user_data.
 581  * @param[out] error
 582  *   Pointer to error structure.
 583  *
 584  * @return
 585  *    0 on success, negative value otherwise and rte_errno is set.
 586  */
 587 static int
 588 flow_hw_async_flow_destroy(struct rte_eth_dev *dev,
 589                            uint32_t queue,
 590                            const struct rte_flow_op_attr *attr,
 591                            struct rte_flow *flow,
 592                            void *user_data,
 593                            struct rte_flow_error *error)
 594 {
 595         struct mlx5_priv *priv = dev->data->dev_private;
 596         struct mlx5dr_rule_attr rule_attr = {
 597                 .queue_id = queue,
 598                 .user_data = user_data,
 599                 .burst = attr->postpone,
 600         };
 601         struct rte_flow_hw *fh = (struct rte_flow_hw *)flow;
 602         struct mlx5_hw_q_job *job;
 603         int ret;
 604
 605         if (unlikely(!priv->hw_q[queue].job_idx)) {
 606                 rte_errno = ENOMEM;
 607                 goto error;
 608         }
 609         job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
 610         job->type = MLX5_HW_Q_JOB_TYPE_DESTROY;
 611         job->user_data = user_data;
 612         job->flow = fh;
 613         rule_attr.user_data = job;
 614         ret = mlx5dr_rule_destroy(&fh->rule, &rule_attr);
 615         if (likely(!ret))
 616                 return 0;
 617         priv->hw_q[queue].job_idx++;
 618 error:
 619         return rte_flow_error_set(error, rte_errno,
 620                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 621                         "fail to create rte flow");
 622 }
 623
 624 /**
 625  * Pull the enqueued flows.
 626  *
 627  * For flows enqueued from creation/destruction, the status should be
 628  * checked from the dequeue result.
 629  *
 630  * @param[in] dev
 631  *   Pointer to the rte_eth_dev structure.
 632  * @param[in] queue
 633  *   The queue to pull the result.
 634  * @param[in/out] res
 635  *   Array to save the results.
 636  * @param[in] n_res
 637  *   Available result with the array.
 638  * @param[out] error
 639  *   Pointer to error structure.
 640  *
 641  * @return
 642  *    Result number on success, negative value otherwise and rte_errno is set.
 643  */
 644 static int
 645 flow_hw_pull(struct rte_eth_dev *dev,
 646              uint32_t queue,
 647              struct rte_flow_op_result res[],
 648              uint16_t n_res,
 649              struct rte_flow_error *error)
 650 {
 651         struct mlx5_priv *priv = dev->data->dev_private;
 652         struct mlx5_hw_q_job *job;
 653         int ret, i;
 654
 655         ret = mlx5dr_send_queue_poll(priv->dr_ctx, queue, res, n_res);
 656         if (ret < 0)
 657                 return rte_flow_error_set(error, rte_errno,
 658                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 659                                 "fail to query flow queue");
 660         for (i = 0; i <  ret; i++) {
 661                 job = (struct mlx5_hw_q_job *)res[i].user_data;
 662                 /* Restore user data. */
 663                 res[i].user_data = job->user_data;
 664                 if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY) {
 665                         if (job->flow->fate_type == MLX5_FLOW_FATE_JUMP)
 666                                 flow_hw_jump_release(dev, job->flow->jump);
 667                         else if (job->flow->fate_type == MLX5_FLOW_FATE_QUEUE)
 668                                 mlx5_hrxq_obj_release(dev, job->flow->hrxq);
 669                         mlx5_ipool_free(job->flow->table->flow, job->flow->idx);
 670                 }
 671                 priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job;
 672         }
 673         return ret;
 674 }
 675
 676 /**
 677  * Push the enqueued flows to HW.
 678  *
 679  * Force apply all the enqueued flows to the HW.
 680  *
 681  * @param[in] dev
 682  *   Pointer to the rte_eth_dev structure.
 683  * @param[in] queue
 684  *   The queue to push the flow.
 685  * @param[out] error
 686  *   Pointer to error structure.
 687  *
 688  * @return
 689  *    0 on success, negative value otherwise and rte_errno is set.
 690  */
 691 static int
 692 flow_hw_push(struct rte_eth_dev *dev,
 693              uint32_t queue,
 694              struct rte_flow_error *error)
 695 {
 696         struct mlx5_priv *priv = dev->data->dev_private;
 697         int ret;
 698
 699         ret = mlx5dr_send_queue_action(priv->dr_ctx, queue,
 700                                        MLX5DR_SEND_QUEUE_ACTION_DRAIN);
 701         if (ret) {
 702                 rte_flow_error_set(error, rte_errno,
 703                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 704                                    "fail to push flows");
 705                 return ret;
 706         }
 707         return 0;
 708 }
 709
 710 /**
 711  * Drain the enqueued flows' completion.
 712  *
 713  * @param[in] dev
 714  *   Pointer to the rte_eth_dev structure.
 715  * @param[in] queue
 716  *   The queue to pull the flow.
 717  * @param[in] pending_rules
 718  *   The pending flow number.
 719  * @param[out] error
 720  *   Pointer to error structure.
 721  *
 722  * @return
 723  *    0 on success, negative value otherwise and rte_errno is set.
 724  */
 725 static int
 726 __flow_hw_pull_comp(struct rte_eth_dev *dev,
 727                     uint32_t queue,
 728                     uint32_t pending_rules,
 729                     struct rte_flow_error *error)
 730 {
 731         struct rte_flow_op_result comp[BURST_THR];
 732         int ret, i, empty_loop = 0;
 733
 734         flow_hw_push(dev, queue, error);
 735         while (pending_rules) {
 736                 ret = flow_hw_pull(dev, queue, comp, BURST_THR, error);
 737                 if (ret < 0)
 738                         return -1;
 739                 if (!ret) {
 740                         rte_delay_us_sleep(20000);
 741                         if (++empty_loop > 5) {
 742                                 DRV_LOG(WARNING, "No available dequeue, quit.");
 743                                 break;
 744                         }
 745                         continue;
 746                 }
 747                 for (i = 0; i < ret; i++) {
 748                         if (comp[i].status == RTE_FLOW_OP_ERROR)
 749                                 DRV_LOG(WARNING, "Flow flush get error CQE.");
 750                 }
 751                 if ((uint32_t)ret > pending_rules) {
 752                         DRV_LOG(WARNING, "Flow flush get extra CQE.");
 753                         return rte_flow_error_set(error, ERANGE,
 754                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 755                                         "get extra CQE");
 756                 }
 757                 pending_rules -= ret;
 758                 empty_loop = 0;
 759         }
 760         return 0;
 761 }
 762
 763 /**
 764  * Flush created flows.
 765  *
 766  * @param[in] dev
 767  *   Pointer to the rte_eth_dev structure.
 768  * @param[out] error
 769  *   Pointer to error structure.
 770  *
 771  * @return
 772  *    0 on success, negative value otherwise and rte_errno is set.
 773  */
 774 int
 775 flow_hw_q_flow_flush(struct rte_eth_dev *dev,
 776                      struct rte_flow_error *error)
 777 {
 778         struct mlx5_priv *priv = dev->data->dev_private;
 779         struct mlx5_hw_q *hw_q;
 780         struct rte_flow_template_table *tbl;
 781         struct rte_flow_hw *flow;
 782         struct rte_flow_op_attr attr = {
 783                 .postpone = 0,
 784         };
 785         uint32_t pending_rules = 0;
 786         uint32_t queue;
 787         uint32_t fidx;
 788
 789         /*
 790          * Ensure to push and dequeue all the enqueued flow
 791          * creation/destruction jobs in case user forgot to
 792          * dequeue. Or the enqueued created flows will be
 793          * leaked. The forgotten dequeues would also cause
 794          * flow flush get extra CQEs as expected and pending_rules
 795          * be minus value.
 796          */
 797         for (queue = 0; queue < priv->nb_queue; queue++) {
 798                 hw_q = &priv->hw_q[queue];
 799                 if (__flow_hw_pull_comp(dev, queue, hw_q->size - hw_q->job_idx,
 800                                         error))
 801                         return -1;
 802         }
 803         /* Flush flow per-table from MLX5_DEFAULT_FLUSH_QUEUE. */
 804         hw_q = &priv->hw_q[MLX5_DEFAULT_FLUSH_QUEUE];
 805         LIST_FOREACH(tbl, &priv->flow_hw_tbl, next) {
 806                 MLX5_IPOOL_FOREACH(tbl->flow, fidx, flow) {
 807                         if (flow_hw_async_flow_destroy(dev,
 808                                                 MLX5_DEFAULT_FLUSH_QUEUE,
 809                                                 &attr,
 810                                                 (struct rte_flow *)flow,
 811                                                 NULL,
 812                                                 error))
 813                                 return -1;
 814                         pending_rules++;
 815                         /* Drain completion with queue size. */
 816                         if (pending_rules >= hw_q->size) {
 817                                 if (__flow_hw_pull_comp(dev,
 818                                                 MLX5_DEFAULT_FLUSH_QUEUE,
 819                                                 pending_rules, error))
 820                                         return -1;
 821                                 pending_rules = 0;
 822                         }
 823                 }
 824         }
 825         /* Drain left completion. */
 826         if (pending_rules &&
 827             __flow_hw_pull_comp(dev, MLX5_DEFAULT_FLUSH_QUEUE, pending_rules,
 828                                 error))
 829                 return -1;
 830         return 0;
 831 }
 832
 833 /**
 834  * Create flow table.
 835  *
 836  * The input item and action templates will be binded to the table.
 837  * Flow memory will also be allocated. Matcher will be created based
 838  * on the item template. Action will be translated to the dedicated
 839  * DR action if possible.
 840  *
 841  * @param[in] dev
 842  *   Pointer to the rte_eth_dev structure.
 843  * @param[in] attr
 844  *   Pointer to the table attributes.
 845  * @param[in] item_templates
 846  *   Item template array to be binded to the table.
 847  * @param[in] nb_item_templates
 848  *   Number of item template.
 849  * @param[in] action_templates
 850  *   Action template array to be binded to the table.
 851  * @param[in] nb_action_templates
 852  *   Number of action template.
 853  * @param[out] error
 854  *   Pointer to error structure.
 855  *
 856  * @return
 857  *    Table on success, NULL otherwise and rte_errno is set.
 858  */
 859 static struct rte_flow_template_table *
 860 flow_hw_table_create(struct rte_eth_dev *dev,
 861                      const struct rte_flow_template_table_attr *attr,
 862                      struct rte_flow_pattern_template *item_templates[],
 863                      uint8_t nb_item_templates,
 864                      struct rte_flow_actions_template *action_templates[],
 865                      uint8_t nb_action_templates,
 866                      struct rte_flow_error *error)
 867 {
 868         struct mlx5_priv *priv = dev->data->dev_private;
 869         struct mlx5dr_matcher_attr matcher_attr = {0};
 870         struct rte_flow_template_table *tbl = NULL;
 871         struct mlx5_flow_group *grp;
 872         struct mlx5dr_match_template *mt[MLX5_HW_TBL_MAX_ITEM_TEMPLATE];
 873         struct rte_flow_attr flow_attr = attr->flow_attr;
 874         struct mlx5_flow_cb_ctx ctx = {
 875                 .dev = dev,
 876                 .error = error,
 877                 .data = &flow_attr,
 878         };
 879         struct mlx5_indexed_pool_config cfg = {
 880                 .size = sizeof(struct rte_flow_hw),
 881                 .trunk_size = 1 << 12,
 882                 .per_core_cache = 1 << 13,
 883                 .need_lock = 1,
 884                 .release_mem_en = !!priv->sh->config.reclaim_mode,
 885                 .malloc = mlx5_malloc,
 886                 .free = mlx5_free,
 887                 .type = "mlx5_hw_table_flow",
 888         };
 889         struct mlx5_list_entry *ge;
 890         uint32_t i, max_tpl = MLX5_HW_TBL_MAX_ITEM_TEMPLATE;
 891         uint32_t nb_flows = rte_align32pow2(attr->nb_flows);
 892         int err;
 893
 894         /* HWS layer accepts only 1 item template with root table. */
 895         if (!attr->flow_attr.group)
 896                 max_tpl = 1;
 897         cfg.max_idx = nb_flows;
 898         /* For table has very limited flows, disable cache. */
 899         if (nb_flows < cfg.trunk_size) {
 900                 cfg.per_core_cache = 0;
 901                 cfg.trunk_size = nb_flows;
 902         }
 903         /* Check if we requires too many templates. */
 904         if (nb_item_templates > max_tpl ||
 905             nb_action_templates > MLX5_HW_TBL_MAX_ACTION_TEMPLATE) {
 906                 rte_errno = EINVAL;
 907                 goto error;
 908         }
 909         /* Allocate the table memory. */
 910         tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl), 0, rte_socket_id());
 911         if (!tbl)
 912                 goto error;
 913         /* Allocate flow indexed pool. */
 914         tbl->flow = mlx5_ipool_create(&cfg);
 915         if (!tbl->flow)
 916                 goto error;
 917         /* Register the flow group. */
 918         ge = mlx5_hlist_register(priv->sh->groups, attr->flow_attr.group, &ctx);
 919         if (!ge)
 920                 goto error;
 921         grp = container_of(ge, struct mlx5_flow_group, entry);
 922         tbl->grp = grp;
 923         /* Prepare matcher information. */
 924         matcher_attr.priority = attr->flow_attr.priority;
 925         matcher_attr.mode = MLX5DR_MATCHER_RESOURCE_MODE_RULE;
 926         matcher_attr.rule.num_log = rte_log2_u32(nb_flows);
 927         /* Build the item template. */
 928         for (i = 0; i < nb_item_templates; i++) {
 929                 uint32_t ret;
 930
 931                 ret = __atomic_add_fetch(&item_templates[i]->refcnt, 1,
 932                                          __ATOMIC_RELAXED);
 933                 if (ret <= 1) {
 934                         rte_errno = EINVAL;
 935                         goto it_error;
 936                 }
 937                 mt[i] = item_templates[i]->mt;
 938                 tbl->its[i] = item_templates[i];
 939         }
 940         tbl->matcher = mlx5dr_matcher_create
 941                 (tbl->grp->tbl, mt, nb_item_templates, &matcher_attr);
 942         if (!tbl->matcher)
 943                 goto it_error;
 944         tbl->nb_item_templates = nb_item_templates;
 945         /* Build the action template. */
 946         for (i = 0; i < nb_action_templates; i++) {
 947                 uint32_t ret;
 948
 949                 ret = __atomic_add_fetch(&action_templates[i]->refcnt, 1,
 950                                          __ATOMIC_RELAXED);
 951                 if (ret <= 1) {
 952                         rte_errno = EINVAL;
 953                         goto at_error;
 954                 }
 955                 LIST_INIT(&tbl->ats[i].acts.act_list);
 956                 err = flow_hw_actions_translate(dev, attr,
 957                                                 &tbl->ats[i].acts,
 958                                                 action_templates[i], error);
 959                 if (err) {
 960                         i++;
 961                         goto at_error;
 962                 }
 963                 tbl->ats[i].action_template = action_templates[i];
 964         }
 965         tbl->nb_action_templates = nb_action_templates;
 966         tbl->type = attr->flow_attr.transfer ? MLX5DR_TABLE_TYPE_FDB :
 967                     (attr->flow_attr.egress ? MLX5DR_TABLE_TYPE_NIC_TX :
 968                     MLX5DR_TABLE_TYPE_NIC_RX);
 969         LIST_INSERT_HEAD(&priv->flow_hw_tbl, tbl, next);
 970         return tbl;
 971 at_error:
 972         while (i--) {
 973                 __flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
 974                 __atomic_sub_fetch(&action_templates[i]->refcnt,
 975                                    1, __ATOMIC_RELAXED);
 976         }
 977         i = nb_item_templates;
 978 it_error:
 979         while (i--)
 980                 __atomic_sub_fetch(&item_templates[i]->refcnt,
 981                                    1, __ATOMIC_RELAXED);
 982         mlx5dr_matcher_destroy(tbl->matcher);
 983 error:
 984         err = rte_errno;
 985         if (tbl) {
 986                 if (tbl->grp)
 987                         mlx5_hlist_unregister(priv->sh->groups,
 988                                               &tbl->grp->entry);
 989                 if (tbl->flow)
 990                         mlx5_ipool_destroy(tbl->flow);
 991                 mlx5_free(tbl);
 992         }
 993         rte_flow_error_set(error, err,
 994                           RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
 995                           "fail to create rte table");
 996         return NULL;
 997 }
 998
 999 /**
1000  * Destroy flow table.
1001  *
1002  * @param[in] dev
1003  *   Pointer to the rte_eth_dev structure.
1004  * @param[in] table
1005  *   Pointer to the table to be destroyed.
1006  * @param[out] error
1007  *   Pointer to error structure.
1008  *
1009  * @return
1010  *   0 on success, a negative errno value otherwise and rte_errno is set.
1011  */
1012 static int
1013 flow_hw_table_destroy(struct rte_eth_dev *dev,
1014                       struct rte_flow_template_table *table,
1015                       struct rte_flow_error *error)
1016 {
1017         struct mlx5_priv *priv = dev->data->dev_private;
1018         int i;
1019
1020         if (table->refcnt) {
1021                 DRV_LOG(WARNING, "Table %p is still in using.", (void *)table);
1022                 return rte_flow_error_set(error, EBUSY,
1023                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1024                                    NULL,
1025                                    "table in using");
1026         }
1027         LIST_REMOVE(table, next);
1028         for (i = 0; i < table->nb_item_templates; i++)
1029                 __atomic_sub_fetch(&table->its[i]->refcnt,
1030                                    1, __ATOMIC_RELAXED);
1031         for (i = 0; i < table->nb_action_templates; i++) {
1032                 __flow_hw_action_template_destroy(dev, &table->ats[i].acts);
1033                 __atomic_sub_fetch(&table->ats[i].action_template->refcnt,
1034                                    1, __ATOMIC_RELAXED);
1035         }
1036         mlx5dr_matcher_destroy(table->matcher);
1037         mlx5_hlist_unregister(priv->sh->groups, &table->grp->entry);
1038         mlx5_ipool_destroy(table->flow);
1039         mlx5_free(table);
1040         return 0;
1041 }
1042
1043 /**
1044  * Create flow action template.
1045  *
1046  * @param[in] dev
1047  *   Pointer to the rte_eth_dev structure.
1048  * @param[in] attr
1049  *   Pointer to the action template attributes.
1050  * @param[in] actions
1051  *   Associated actions (list terminated by the END action).
1052  * @param[in] masks
1053  *   List of actions that marks which of the action's member is constant.
1054  * @param[out] error
1055  *   Pointer to error structure.
1056  *
1057  * @return
1058  *   Action template pointer on success, NULL otherwise and rte_errno is set.
1059  */
1060 static struct rte_flow_actions_template *
1061 flow_hw_actions_template_create(struct rte_eth_dev *dev,
1062                         const struct rte_flow_actions_template_attr *attr,
1063                         const struct rte_flow_action actions[],
1064                         const struct rte_flow_action masks[],
1065                         struct rte_flow_error *error)
1066 {
1067         struct mlx5_priv *priv = dev->data->dev_private;
1068         int len, act_len, mask_len, i;
1069         struct rte_flow_actions_template *at;
1070
1071         act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS,
1072                                 NULL, 0, actions, error);
1073         if (act_len <= 0)
1074                 return NULL;
1075         len = RTE_ALIGN(act_len, 16);
1076         mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS,
1077                                  NULL, 0, masks, error);
1078         if (mask_len <= 0)
1079                 return NULL;
1080         len += RTE_ALIGN(mask_len, 16);
1081         at = mlx5_malloc(MLX5_MEM_ZERO, len + sizeof(*at), 64, rte_socket_id());
1082         if (!at) {
1083                 rte_flow_error_set(error, ENOMEM,
1084                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1085                                    NULL,
1086                                    "cannot allocate action template");
1087                 return NULL;
1088         }
1089         at->attr = *attr;
1090         at->actions = (struct rte_flow_action *)(at + 1);
1091         act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->actions, len,
1092                                 actions, error);
1093         if (act_len <= 0)
1094                 goto error;
1095         at->masks = (struct rte_flow_action *)
1096                     (((uint8_t *)at->actions) + act_len);
1097         mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->masks,
1098                                  len - act_len, masks, error);
1099         if (mask_len <= 0)
1100                 goto error;
1101         /*
1102          * mlx5 PMD hacks indirect action index directly to the action conf.
1103          * The rte_flow_conv() function copies the content from conf pointer.
1104          * Need to restore the indirect action index from action conf here.
1105          */
1106         for (i = 0; actions->type != RTE_FLOW_ACTION_TYPE_END;
1107              actions++, masks++, i++) {
1108                 if (actions->type == RTE_FLOW_ACTION_TYPE_INDIRECT) {
1109                         at->actions[i].conf = actions->conf;
1110                         at->masks[i].conf = masks->conf;
1111                 }
1112         }
1113         __atomic_fetch_add(&at->refcnt, 1, __ATOMIC_RELAXED);
1114         LIST_INSERT_HEAD(&priv->flow_hw_at, at, next);
1115         return at;
1116 error:
1117         mlx5_free(at);
1118         return NULL;
1119 }
1120
1121 /**
1122  * Destroy flow action template.
1123  *
1124  * @param[in] dev
1125  *   Pointer to the rte_eth_dev structure.
1126  * @param[in] template
1127  *   Pointer to the action template to be destroyed.
1128  * @param[out] error
1129  *   Pointer to error structure.
1130  *
1131  * @return
1132  *   0 on success, a negative errno value otherwise and rte_errno is set.
1133  */
1134 static int
1135 flow_hw_actions_template_destroy(struct rte_eth_dev *dev __rte_unused,
1136                                  struct rte_flow_actions_template *template,
1137                                  struct rte_flow_error *error __rte_unused)
1138 {
1139         if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
1140                 DRV_LOG(WARNING, "Action template %p is still in use.",
1141                         (void *)template);
1142                 return rte_flow_error_set(error, EBUSY,
1143                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1144                                    NULL,
1145                                    "action template in using");
1146         }
1147         LIST_REMOVE(template, next);
1148         mlx5_free(template);
1149         return 0;
1150 }
1151
1152 /**
1153  * Create flow item template.
1154  *
1155  * @param[in] dev
1156  *   Pointer to the rte_eth_dev structure.
1157  * @param[in] attr
1158  *   Pointer to the item template attributes.
1159  * @param[in] items
1160  *   The template item pattern.
1161  * @param[out] error
1162  *   Pointer to error structure.
1163  *
1164  * @return
1165  *  Item template pointer on success, NULL otherwise and rte_errno is set.
1166  */
1167 static struct rte_flow_pattern_template *
1168 flow_hw_pattern_template_create(struct rte_eth_dev *dev,
1169                              const struct rte_flow_pattern_template_attr *attr,
1170                              const struct rte_flow_item items[],
1171                              struct rte_flow_error *error)
1172 {
1173         struct mlx5_priv *priv = dev->data->dev_private;
1174         struct rte_flow_pattern_template *it;
1175
1176         it = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*it), 0, rte_socket_id());
1177         if (!it) {
1178                 rte_flow_error_set(error, ENOMEM,
1179                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1180                                    NULL,
1181                                    "cannot allocate item template");
1182                 return NULL;
1183         }
1184         it->attr = *attr;
1185         it->mt = mlx5dr_match_template_create(items, attr->relaxed_matching);
1186         if (!it->mt) {
1187                 mlx5_free(it);
1188                 rte_flow_error_set(error, rte_errno,
1189                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1190                                    NULL,
1191                                    "cannot create match template");
1192                 return NULL;
1193         }
1194         __atomic_fetch_add(&it->refcnt, 1, __ATOMIC_RELAXED);
1195         LIST_INSERT_HEAD(&priv->flow_hw_itt, it, next);
1196         return it;
1197 }
1198
1199 /**
1200  * Destroy flow item template.
1201  *
1202  * @param[in] dev
1203  *   Pointer to the rte_eth_dev structure.
1204  * @param[in] template
1205  *   Pointer to the item template to be destroyed.
1206  * @param[out] error
1207  *   Pointer to error structure.
1208  *
1209  * @return
1210  *   0 on success, a negative errno value otherwise and rte_errno is set.
1211  */
1212 static int
1213 flow_hw_pattern_template_destroy(struct rte_eth_dev *dev __rte_unused,
1214                               struct rte_flow_pattern_template *template,
1215                               struct rte_flow_error *error __rte_unused)
1216 {
1217         if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
1218                 DRV_LOG(WARNING, "Item template %p is still in use.",
1219                         (void *)template);
1220                 return rte_flow_error_set(error, EBUSY,
1221                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1222                                    NULL,
1223                                    "item template in using");
1224         }
1225         LIST_REMOVE(template, next);
1226         claim_zero(mlx5dr_match_template_destroy(template->mt));
1227         mlx5_free(template);
1228         return 0;
1229 }
1230
1231 /*
1232  * Get information about HWS pre-configurable resources.
1233  *
1234  * @param[in] dev
1235  *   Pointer to the rte_eth_dev structure.
1236  * @param[out] port_info
1237  *   Pointer to port information.
1238  * @param[out] queue_info
1239  *   Pointer to queue information.
1240  * @param[out] error
1241  *   Pointer to error structure.
1242  *
1243  * @return
1244  *   0 on success, a negative errno value otherwise and rte_errno is set.
1245  */
1246 static int
1247 flow_hw_info_get(struct rte_eth_dev *dev __rte_unused,
1248                  struct rte_flow_port_info *port_info __rte_unused,
1249                  struct rte_flow_queue_info *queue_info __rte_unused,
1250                  struct rte_flow_error *error __rte_unused)
1251 {
1252         /* Nothing to be updated currently. */
1253         memset(port_info, 0, sizeof(*port_info));
1254         /* Queue size is unlimited from low-level. */
1255         queue_info->max_size = UINT32_MAX;
1256         return 0;
1257 }
1258
1259 /**
1260  * Create group callback.
1261  *
1262  * @param[in] tool_ctx
1263  *   Pointer to the hash list related context.
1264  * @param[in] cb_ctx
1265  *   Pointer to the group creation context.
1266  *
1267  * @return
1268  *   Group entry on success, NULL otherwise and rte_errno is set.
1269  */
1270 struct mlx5_list_entry *
1271 flow_hw_grp_create_cb(void *tool_ctx, void *cb_ctx)
1272 {
1273         struct mlx5_dev_ctx_shared *sh = tool_ctx;
1274         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
1275         struct rte_eth_dev *dev = ctx->dev;
1276         struct rte_flow_attr *attr = (struct rte_flow_attr *)ctx->data;
1277         struct mlx5_priv *priv = dev->data->dev_private;
1278         struct mlx5dr_table_attr dr_tbl_attr = {0};
1279         struct rte_flow_error *error = ctx->error;
1280         struct mlx5_flow_group *grp_data;
1281         struct mlx5dr_table *tbl = NULL;
1282         struct mlx5dr_action *jump;
1283         uint32_t idx = 0;
1284
1285         grp_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
1286         if (!grp_data) {
1287                 rte_flow_error_set(error, ENOMEM,
1288                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1289                                    NULL,
1290                                    "cannot allocate flow table data entry");
1291                 return NULL;
1292         }
1293         dr_tbl_attr.level = attr->group;
1294         if (attr->transfer)
1295                 dr_tbl_attr.type = MLX5DR_TABLE_TYPE_FDB;
1296         else if (attr->egress)
1297                 dr_tbl_attr.type = MLX5DR_TABLE_TYPE_NIC_TX;
1298         else
1299                 dr_tbl_attr.type = MLX5DR_TABLE_TYPE_NIC_RX;
1300         tbl = mlx5dr_table_create(priv->dr_ctx, &dr_tbl_attr);
1301         if (!tbl)
1302                 goto error;
1303         grp_data->tbl = tbl;
1304         if (attr->group) {
1305                 /* Jump action be used by non-root table. */
1306                 jump = mlx5dr_action_create_dest_table
1307                         (priv->dr_ctx, tbl,
1308                          mlx5_hw_act_flag[!!attr->group][dr_tbl_attr.type]);
1309                 if (!jump)
1310                         goto error;
1311                 grp_data->jump.hws_action = jump;
1312                 /* Jump action be used by root table.  */
1313                 jump = mlx5dr_action_create_dest_table
1314                         (priv->dr_ctx, tbl,
1315                          mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_ROOT]
1316                                          [dr_tbl_attr.type]);
1317                 if (!jump)
1318                         goto error;
1319                 grp_data->jump.root_action = jump;
1320         }
1321         grp_data->idx = idx;
1322         grp_data->group_id = attr->group;
1323         grp_data->type = dr_tbl_attr.type;
1324         return &grp_data->entry;
1325 error:
1326         if (grp_data->jump.root_action)
1327                 mlx5dr_action_destroy(grp_data->jump.root_action);
1328         if (grp_data->jump.hws_action)
1329                 mlx5dr_action_destroy(grp_data->jump.hws_action);
1330         if (tbl)
1331                 mlx5dr_table_destroy(tbl);
1332         if (idx)
1333                 mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], idx);
1334         rte_flow_error_set(error, ENOMEM,
1335                            RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1336                            NULL,
1337                            "cannot allocate flow dr table");
1338         return NULL;
1339 }
1340
1341 /**
1342  * Remove group callback.
1343  *
1344  * @param[in] tool_ctx
1345  *   Pointer to the hash list related context.
1346  * @param[in] entry
1347  *   Pointer to the entry to be removed.
1348  */
1349 void
1350 flow_hw_grp_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
1351 {
1352         struct mlx5_dev_ctx_shared *sh = tool_ctx;
1353         struct mlx5_flow_group *grp_data =
1354                     container_of(entry, struct mlx5_flow_group, entry);
1355
1356         MLX5_ASSERT(entry && sh);
1357         /* To use the wrapper glue functions instead. */
1358         if (grp_data->jump.hws_action)
1359                 mlx5dr_action_destroy(grp_data->jump.hws_action);
1360         if (grp_data->jump.root_action)
1361                 mlx5dr_action_destroy(grp_data->jump.root_action);
1362         mlx5dr_table_destroy(grp_data->tbl);
1363         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
1364 }
1365
1366 /**
1367  * Match group callback.
1368  *
1369  * @param[in] tool_ctx
1370  *   Pointer to the hash list related context.
1371  * @param[in] entry
1372  *   Pointer to the group to be matched.
1373  * @param[in] cb_ctx
1374  *   Pointer to the group matching context.
1375  *
1376  * @return
1377  *   0 on matched, 1 on miss matched.
1378  */
1379 int
1380 flow_hw_grp_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
1381                      void *cb_ctx)
1382 {
1383         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
1384         struct mlx5_flow_group *grp_data =
1385                 container_of(entry, struct mlx5_flow_group, entry);
1386         struct rte_flow_attr *attr =
1387                         (struct rte_flow_attr *)ctx->data;
1388
1389         return (grp_data->group_id != attr->group) ||
1390                 ((grp_data->type != MLX5DR_TABLE_TYPE_FDB) &&
1391                 attr->transfer) ||
1392                 ((grp_data->type != MLX5DR_TABLE_TYPE_NIC_TX) &&
1393                 attr->egress) ||
1394                 ((grp_data->type != MLX5DR_TABLE_TYPE_NIC_RX) &&
1395                 attr->ingress);
1396 }
1397
1398 /**
1399  * Clone group entry callback.
1400  *
1401  * @param[in] tool_ctx
1402  *   Pointer to the hash list related context.
1403  * @param[in] entry
1404  *   Pointer to the group to be matched.
1405  * @param[in] cb_ctx
1406  *   Pointer to the group matching context.
1407  *
1408  * @return
1409  *   0 on matched, 1 on miss matched.
1410  */
1411 struct mlx5_list_entry *
1412 flow_hw_grp_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
1413                      void *cb_ctx)
1414 {
1415         struct mlx5_dev_ctx_shared *sh = tool_ctx;
1416         struct mlx5_flow_cb_ctx *ctx = cb_ctx;
1417         struct mlx5_flow_group *grp_data;
1418         struct rte_flow_error *error = ctx->error;
1419         uint32_t idx = 0;
1420
1421         grp_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
1422         if (!grp_data) {
1423                 rte_flow_error_set(error, ENOMEM,
1424                                    RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
1425                                    NULL,
1426                                    "cannot allocate flow table data entry");
1427                 return NULL;
1428         }
1429         memcpy(grp_data, oentry, sizeof(*grp_data));
1430         grp_data->idx = idx;
1431         return &grp_data->entry;
1432 }
1433
1434 /**
1435  * Free cloned group entry callback.
1436  *
1437  * @param[in] tool_ctx
1438  *   Pointer to the hash list related context.
1439  * @param[in] entry
1440  *   Pointer to the group to be freed.
1441  */
1442 void
1443 flow_hw_grp_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
1444 {
1445         struct mlx5_dev_ctx_shared *sh = tool_ctx;
1446         struct mlx5_flow_group *grp_data =
1447                     container_of(entry, struct mlx5_flow_group, entry);
1448
1449         mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
1450 }
1451
1452 /**
1453  * Configure port HWS resources.
1454  *
1455  * @param[in] dev
1456  *   Pointer to the rte_eth_dev structure.
1457  * @param[in] port_attr
1458  *   Port configuration attributes.
1459  * @param[in] nb_queue
1460  *   Number of queue.
1461  * @param[in] queue_attr
1462  *   Array that holds attributes for each flow queue.
1463  * @param[out] error
1464  *   Pointer to error structure.
1465  *
1466  * @return
1467  *   0 on success, a negative errno value otherwise and rte_errno is set.
1468  */
1469
1470 static int
1471 flow_hw_configure(struct rte_eth_dev *dev,
1472                   const struct rte_flow_port_attr *port_attr,
1473                   uint16_t nb_queue,
1474                   const struct rte_flow_queue_attr *queue_attr[],
1475                   struct rte_flow_error *error)
1476 {
1477         struct mlx5_priv *priv = dev->data->dev_private;
1478         struct mlx5dr_context *dr_ctx = NULL;
1479         struct mlx5dr_context_attr dr_ctx_attr = {0};
1480         struct mlx5_hw_q *hw_q;
1481         struct mlx5_hw_q_job *job = NULL;
1482         uint32_t mem_size, i, j;
1483         struct mlx5_indexed_pool_config cfg = {
1484                 .size = sizeof(struct rte_flow_hw),
1485                 .trunk_size = 4096,
1486                 .need_lock = 1,
1487                 .release_mem_en = !!priv->sh->config.reclaim_mode,
1488                 .malloc = mlx5_malloc,
1489                 .free = mlx5_free,
1490                 .type = "mlx5_hw_action_construct_data",
1491         };
1492
1493         if (!port_attr || !nb_queue || !queue_attr) {
1494                 rte_errno = EINVAL;
1495                 goto err;
1496         }
1497         /* In case re-configuring, release existing context at first. */
1498         if (priv->dr_ctx) {
1499                 /* */
1500                 for (i = 0; i < nb_queue; i++) {
1501                         hw_q = &priv->hw_q[i];
1502                         /* Make sure all queues are empty. */
1503                         if (hw_q->size != hw_q->job_idx) {
1504                                 rte_errno = EBUSY;
1505                                 goto err;
1506                         }
1507                 }
1508                 flow_hw_resource_release(dev);
1509         }
1510         priv->acts_ipool = mlx5_ipool_create(&cfg);
1511         if (!priv->acts_ipool)
1512                 goto err;
1513         /* Allocate the queue job descriptor LIFO. */
1514         mem_size = sizeof(priv->hw_q[0]) * nb_queue;
1515         for (i = 0; i < nb_queue; i++) {
1516                 /*
1517                  * Check if the queues' size are all the same as the
1518                  * limitation from HWS layer.
1519                  */
1520                 if (queue_attr[i]->size != queue_attr[0]->size) {
1521                         rte_errno = EINVAL;
1522                         goto err;
1523                 }
1524                 mem_size += (sizeof(struct mlx5_hw_q_job *) +
1525                             sizeof(struct mlx5_hw_q_job)) *
1526                             queue_attr[0]->size;
1527         }
1528         priv->hw_q = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
1529                                  64, SOCKET_ID_ANY);
1530         if (!priv->hw_q) {
1531                 rte_errno = ENOMEM;
1532                 goto err;
1533         }
1534         for (i = 0; i < nb_queue; i++) {
1535                 priv->hw_q[i].job_idx = queue_attr[i]->size;
1536                 priv->hw_q[i].size = queue_attr[i]->size;
1537                 if (i == 0)
1538                         priv->hw_q[i].job = (struct mlx5_hw_q_job **)
1539                                             &priv->hw_q[nb_queue];
1540                 else
1541                         priv->hw_q[i].job = (struct mlx5_hw_q_job **)
1542                                             &job[queue_attr[i - 1]->size];
1543                 job = (struct mlx5_hw_q_job *)
1544                       &priv->hw_q[i].job[queue_attr[i]->size];
1545                 for (j = 0; j < queue_attr[i]->size; j++)
1546                         priv->hw_q[i].job[j] = &job[j];
1547         }
1548         dr_ctx_attr.pd = priv->sh->cdev->pd;
1549         dr_ctx_attr.queues = nb_queue;
1550         /* Queue size should all be the same. Take the first one. */
1551         dr_ctx_attr.queue_size = queue_attr[0]->size;
1552         dr_ctx = mlx5dr_context_open(priv->sh->cdev->ctx, &dr_ctx_attr);
1553         /* rte_errno has been updated by HWS layer. */
1554         if (!dr_ctx)
1555                 goto err;
1556         priv->dr_ctx = dr_ctx;
1557         priv->nb_queue = nb_queue;
1558         /* Add global actions. */
1559         for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
1560                 for (j = 0; j < MLX5DR_TABLE_TYPE_MAX; j++) {
1561                         priv->hw_drop[i][j] = mlx5dr_action_create_dest_drop
1562                                 (priv->dr_ctx, mlx5_hw_act_flag[i][j]);
1563                         if (!priv->hw_drop[i][j])
1564                                 goto err;
1565                 }
1566         }
1567         return 0;
1568 err:
1569         for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
1570                 for (j = 0; j < MLX5DR_TABLE_TYPE_MAX; j++) {
1571                         if (!priv->hw_drop[i][j])
1572                                 continue;
1573                         mlx5dr_action_destroy(priv->hw_drop[i][j]);
1574                 }
1575         }
1576         if (dr_ctx)
1577                 claim_zero(mlx5dr_context_close(dr_ctx));
1578         mlx5_free(priv->hw_q);
1579         priv->hw_q = NULL;
1580         if (priv->acts_ipool) {
1581                 mlx5_ipool_destroy(priv->acts_ipool);
1582                 priv->acts_ipool = NULL;
1583         }
1584         return rte_flow_error_set(error, rte_errno,
1585                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1586                                   "fail to configure port");
1587 }
1588
1589 /**
1590  * Release HWS resources.
1591  *
1592  * @param[in] dev
1593  *   Pointer to the rte_eth_dev structure.
1594  */
1595 void
1596 flow_hw_resource_release(struct rte_eth_dev *dev)
1597 {
1598         struct mlx5_priv *priv = dev->data->dev_private;
1599         struct rte_flow_template_table *tbl;
1600         struct rte_flow_pattern_template *it;
1601         struct rte_flow_actions_template *at;
1602         int i, j;
1603
1604         if (!priv->dr_ctx)
1605                 return;
1606         while (!LIST_EMPTY(&priv->flow_hw_tbl)) {
1607                 tbl = LIST_FIRST(&priv->flow_hw_tbl);
1608                 flow_hw_table_destroy(dev, tbl, NULL);
1609         }
1610         while (!LIST_EMPTY(&priv->flow_hw_itt)) {
1611                 it = LIST_FIRST(&priv->flow_hw_itt);
1612                 flow_hw_pattern_template_destroy(dev, it, NULL);
1613         }
1614         while (!LIST_EMPTY(&priv->flow_hw_at)) {
1615                 at = LIST_FIRST(&priv->flow_hw_at);
1616                 flow_hw_actions_template_destroy(dev, at, NULL);
1617         }
1618         for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
1619                 for (j = 0; j < MLX5DR_TABLE_TYPE_MAX; j++) {
1620                         if (!priv->hw_drop[i][j])
1621                                 continue;
1622                         mlx5dr_action_destroy(priv->hw_drop[i][j]);
1623                 }
1624         }
1625         if (priv->acts_ipool) {
1626                 mlx5_ipool_destroy(priv->acts_ipool);
1627                 priv->acts_ipool = NULL;
1628         }
1629         mlx5_free(priv->hw_q);
1630         priv->hw_q = NULL;
1631         claim_zero(mlx5dr_context_close(priv->dr_ctx));
1632         priv->dr_ctx = NULL;
1633         priv->nb_queue = 0;
1634 }
1635
1636 const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
1637         .info_get = flow_hw_info_get,
1638         .configure = flow_hw_configure,
1639         .pattern_template_create = flow_hw_pattern_template_create,
1640         .pattern_template_destroy = flow_hw_pattern_template_destroy,
1641         .actions_template_create = flow_hw_actions_template_create,
1642         .actions_template_destroy = flow_hw_actions_template_destroy,
1643         .template_table_create = flow_hw_table_create,
1644         .template_table_destroy = flow_hw_table_destroy,
1645         .async_flow_create = flow_hw_async_flow_create,
1646         .async_flow_destroy = flow_hw_async_flow_destroy,
1647         .pull = flow_hw_pull,
1648         .push = flow_hw_push,
1649 };
1650
1651 #endif