drivers/common/mlx5/mlx5_devx_cmds.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 #include <unistd.h>
   6
   7 #include <rte_errno.h>
   8 #include <rte_malloc.h>
   9 #include <rte_eal_paging.h>
  10
  11 #include "mlx5_prm.h"
  12 #include "mlx5_devx_cmds.h"
  13 #include "mlx5_common_log.h"
  14 #include "mlx5_malloc.h"
  15
  16 static void *
  17 mlx5_devx_get_hca_cap(void *ctx, uint32_t *in, uint32_t *out,
  18                       int *err, uint32_t flags)
  19 {
  20         const size_t size_in = MLX5_ST_SZ_DW(query_hca_cap_in) * sizeof(int);
  21         const size_t size_out = MLX5_ST_SZ_DW(query_hca_cap_out) * sizeof(int);
  22         int status, syndrome, rc;
  23
  24         if (err)
  25                 *err = 0;
  26         memset(in, 0, size_in);
  27         memset(out, 0, size_out);
  28         MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
  29         MLX5_SET(query_hca_cap_in, in, op_mod, flags);
  30         rc = mlx5_glue->devx_general_cmd(ctx, in, size_in, out, size_out);
  31         if (rc) {
  32                 DRV_LOG(ERR,
  33                         "Failed to query devx HCA capabilities func %#02x",
  34                         flags >> 1);
  35                 if (err)
  36                         *err = rc > 0 ? -rc : rc;
  37                 return NULL;
  38         }
  39         status = MLX5_GET(query_hca_cap_out, out, status);
  40         syndrome = MLX5_GET(query_hca_cap_out, out, syndrome);
  41         if (status) {
  42                 DRV_LOG(ERR,
  43                         "Failed to query devx HCA capabilities func %#02x status %x, syndrome = %x",
  44                         flags >> 1, status, syndrome);
  45                 if (err)
  46                         *err = -1;
  47                 return NULL;
  48         }
  49         return MLX5_ADDR_OF(query_hca_cap_out, out, capability);
  50 }
  51
  52 /**
  53  * Perform read access to the registers. Reads data from register
  54  * and writes ones to the specified buffer.
  55  *
  56  * @param[in] ctx
  57  *   Context returned from mlx5 open_device() glue function.
  58  * @param[in] reg_id
  59  *   Register identifier according to the PRM.
  60  * @param[in] arg
  61  *   Register access auxiliary parameter according to the PRM.
  62  * @param[out] data
  63  *   Pointer to the buffer to store read data.
  64  * @param[in] dw_cnt
  65  *   Buffer size in double words.
  66  *
  67  * @return
  68  *   0 on success, a negative value otherwise.
  69  */
  70 int
  71 mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id, uint32_t arg,
  72                             uint32_t *data, uint32_t dw_cnt)
  73 {
  74         uint32_t in[MLX5_ST_SZ_DW(access_register_in)]   = {0};
  75         uint32_t out[MLX5_ST_SZ_DW(access_register_out) +
  76                      MLX5_ACCESS_REGISTER_DATA_DWORD_MAX] = {0};
  77         int status, rc;
  78
  79         MLX5_ASSERT(data && dw_cnt);
  80         MLX5_ASSERT(dw_cnt <= MLX5_ACCESS_REGISTER_DATA_DWORD_MAX);
  81         if (dw_cnt  > MLX5_ACCESS_REGISTER_DATA_DWORD_MAX) {
  82                 DRV_LOG(ERR, "Not enough  buffer for register read data");
  83                 return -1;
  84         }
  85         MLX5_SET(access_register_in, in, opcode,
  86                  MLX5_CMD_OP_ACCESS_REGISTER_USER);
  87         MLX5_SET(access_register_in, in, op_mod,
  88                                         MLX5_ACCESS_REGISTER_IN_OP_MOD_READ);
  89         MLX5_SET(access_register_in, in, register_id, reg_id);
  90         MLX5_SET(access_register_in, in, argument, arg);
  91         rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out,
  92                                          MLX5_ST_SZ_BYTES(access_register_out) +
  93                                          sizeof(uint32_t) * dw_cnt);
  94         if (rc)
  95                 goto error;
  96         status = MLX5_GET(access_register_out, out, status);
  97         if (status) {
  98                 int syndrome = MLX5_GET(access_register_out, out, syndrome);
  99
 100                 DRV_LOG(DEBUG, "Failed to read access NIC register 0x%X, "
 101                                "status %x, syndrome = %x",
 102                                reg_id, status, syndrome);
 103                 return -1;
 104         }
 105         memcpy(data, &out[MLX5_ST_SZ_DW(access_register_out)],
 106                dw_cnt * sizeof(uint32_t));
 107         return 0;
 108 error:
 109         rc = (rc > 0) ? -rc : rc;
 110         return rc;
 111 }
 112
 113 /**
 114  * Perform write access to the registers.
 115  *
 116  * @param[in] ctx
 117  *   Context returned from mlx5 open_device() glue function.
 118  * @param[in] reg_id
 119  *   Register identifier according to the PRM.
 120  * @param[in] arg
 121  *   Register access auxiliary parameter according to the PRM.
 122  * @param[out] data
 123  *   Pointer to the buffer containing data to write.
 124  * @param[in] dw_cnt
 125  *   Buffer size in double words (32bit units).
 126  *
 127  * @return
 128  *   0 on success, a negative value otherwise.
 129  */
 130 int
 131 mlx5_devx_cmd_register_write(void *ctx, uint16_t reg_id, uint32_t arg,
 132                              uint32_t *data, uint32_t dw_cnt)
 133 {
 134         uint32_t in[MLX5_ST_SZ_DW(access_register_in) +
 135                     MLX5_ACCESS_REGISTER_DATA_DWORD_MAX] = {0};
 136         uint32_t out[MLX5_ST_SZ_DW(access_register_out)] = {0};
 137         int status, rc;
 138         void *ptr;
 139
 140         MLX5_ASSERT(data && dw_cnt);
 141         MLX5_ASSERT(dw_cnt <= MLX5_ACCESS_REGISTER_DATA_DWORD_MAX);
 142         if (dw_cnt > MLX5_ACCESS_REGISTER_DATA_DWORD_MAX) {
 143                 DRV_LOG(ERR, "Data to write exceeds max size");
 144                 return -1;
 145         }
 146         MLX5_SET(access_register_in, in, opcode,
 147                  MLX5_CMD_OP_ACCESS_REGISTER_USER);
 148         MLX5_SET(access_register_in, in, op_mod,
 149                  MLX5_ACCESS_REGISTER_IN_OP_MOD_WRITE);
 150         MLX5_SET(access_register_in, in, register_id, reg_id);
 151         MLX5_SET(access_register_in, in, argument, arg);
 152         ptr = MLX5_ADDR_OF(access_register_in, in, register_data);
 153         memcpy(ptr, data, dw_cnt * sizeof(uint32_t));
 154         rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
 155
 156         rc = mlx5_glue->devx_general_cmd(ctx, in,
 157                                          MLX5_ST_SZ_BYTES(access_register_in) +
 158                                          dw_cnt * sizeof(uint32_t),
 159                                          out, sizeof(out));
 160         if (rc)
 161                 goto error;
 162         status = MLX5_GET(access_register_out, out, status);
 163         if (status) {
 164                 int syndrome = MLX5_GET(access_register_out, out, syndrome);
 165
 166                 DRV_LOG(DEBUG, "Failed to write access NIC register 0x%X, "
 167                                "status %x, syndrome = %x",
 168                                reg_id, status, syndrome);
 169                 return -1;
 170         }
 171         return 0;
 172 error:
 173         rc = (rc > 0) ? -rc : rc;
 174         return rc;
 175 }
 176
 177 /**
 178  * Allocate flow counters via devx interface.
 179  *
 180  * @param[in] ctx
 181  *   Context returned from mlx5 open_device() glue function.
 182  * @param dcs
 183  *   Pointer to counters properties structure to be filled by the routine.
 184  * @param bulk_n_128
 185  *   Bulk counter numbers in 128 counters units.
 186  *
 187  * @return
 188  *   Pointer to counter object on success, a negative value otherwise and
 189  *   rte_errno is set.
 190  */
 191 struct mlx5_devx_obj *
 192 mlx5_devx_cmd_flow_counter_alloc(void *ctx, uint32_t bulk_n_128)
 193 {
 194         struct mlx5_devx_obj *dcs = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*dcs),
 195                                                 0, SOCKET_ID_ANY);
 196         uint32_t in[MLX5_ST_SZ_DW(alloc_flow_counter_in)]   = {0};
 197         uint32_t out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {0};
 198
 199         if (!dcs) {
 200                 rte_errno = ENOMEM;
 201                 return NULL;
 202         }
 203         MLX5_SET(alloc_flow_counter_in, in, opcode,
 204                  MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
 205         MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, bulk_n_128);
 206         dcs->obj = mlx5_glue->devx_obj_create(ctx, in,
 207                                               sizeof(in), out, sizeof(out));
 208         if (!dcs->obj) {
 209                 DRV_LOG(ERR, "Can't allocate counters - error %d", errno);
 210                 rte_errno = errno;
 211                 mlx5_free(dcs);
 212                 return NULL;
 213         }
 214         dcs->id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
 215         return dcs;
 216 }
 217
 218 /**
 219  * Query flow counters values.
 220  *
 221  * @param[in] dcs
 222  *   devx object that was obtained from mlx5_devx_cmd_fc_alloc.
 223  * @param[in] clear
 224  *   Whether hardware should clear the counters after the query or not.
 225  * @param[in] n_counters
 226  *   0 in case of 1 counter to read, otherwise the counter number to read.
 227  *  @param pkts
 228  *   The number of packets that matched the flow.
 229  *  @param bytes
 230  *    The number of bytes that matched the flow.
 231  *  @param mkey
 232  *   The mkey key for batch query.
 233  *  @param addr
 234  *    The address in the mkey range for batch query.
 235  *  @param cmd_comp
 236  *   The completion object for asynchronous batch query.
 237  *  @param async_id
 238  *    The ID to be returned in the asynchronous batch query response.
 239  *
 240  * @return
 241  *   0 on success, a negative value otherwise.
 242  */
 243 int
 244 mlx5_devx_cmd_flow_counter_query(struct mlx5_devx_obj *dcs,
 245                                  int clear, uint32_t n_counters,
 246                                  uint64_t *pkts, uint64_t *bytes,
 247                                  uint32_t mkey, void *addr,
 248                                  void *cmd_comp,
 249                                  uint64_t async_id)
 250 {
 251         int out_len = MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 252                         MLX5_ST_SZ_BYTES(traffic_counter);
 253         uint32_t out[out_len];
 254         uint32_t in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {0};
 255         void *stats;
 256         int rc;
 257
 258         MLX5_SET(query_flow_counter_in, in, opcode,
 259                  MLX5_CMD_OP_QUERY_FLOW_COUNTER);
 260         MLX5_SET(query_flow_counter_in, in, op_mod, 0);
 261         MLX5_SET(query_flow_counter_in, in, flow_counter_id, dcs->id);
 262         MLX5_SET(query_flow_counter_in, in, clear, !!clear);
 263
 264         if (n_counters) {
 265                 MLX5_SET(query_flow_counter_in, in, num_of_counters,
 266                          n_counters);
 267                 MLX5_SET(query_flow_counter_in, in, dump_to_memory, 1);
 268                 MLX5_SET(query_flow_counter_in, in, mkey, mkey);
 269                 MLX5_SET64(query_flow_counter_in, in, address,
 270                            (uint64_t)(uintptr_t)addr);
 271         }
 272         if (!cmd_comp)
 273                 rc = mlx5_glue->devx_obj_query(dcs->obj, in, sizeof(in), out,
 274                                                out_len);
 275         else
 276                 rc = mlx5_glue->devx_obj_query_async(dcs->obj, in, sizeof(in),
 277                                                      out_len, async_id,
 278                                                      cmd_comp);
 279         if (rc) {
 280                 DRV_LOG(ERR, "Failed to query devx counters with rc %d", rc);
 281                 rte_errno = rc;
 282                 return -rc;
 283         }
 284         if (!n_counters) {
 285                 stats = MLX5_ADDR_OF(query_flow_counter_out,
 286                                      out, flow_statistics);
 287                 *pkts = MLX5_GET64(traffic_counter, stats, packets);
 288                 *bytes = MLX5_GET64(traffic_counter, stats, octets);
 289         }
 290         return 0;
 291 }
 292
 293 /**
 294  * Create a new mkey.
 295  *
 296  * @param[in] ctx
 297  *   Context returned from mlx5 open_device() glue function.
 298  * @param[in] attr
 299  *   Attributes of the requested mkey.
 300  *
 301  * @return
 302  *   Pointer to Devx mkey on success, a negative value otherwise and rte_errno
 303  *   is set.
 304  */
 305 struct mlx5_devx_obj *
 306 mlx5_devx_cmd_mkey_create(void *ctx,
 307                           struct mlx5_devx_mkey_attr *attr)
 308 {
 309         struct mlx5_klm *klm_array = attr->klm_array;
 310         int klm_num = attr->klm_num;
 311         int in_size_dw = MLX5_ST_SZ_DW(create_mkey_in) +
 312                      (klm_num ? RTE_ALIGN(klm_num, 4) : 0) * MLX5_ST_SZ_DW(klm);
 313         uint32_t in[in_size_dw];
 314         uint32_t out[MLX5_ST_SZ_DW(create_mkey_out)] = {0};
 315         void *mkc;
 316         struct mlx5_devx_obj *mkey = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*mkey),
 317                                                  0, SOCKET_ID_ANY);
 318         size_t pgsize;
 319         uint32_t translation_size;
 320
 321         if (!mkey) {
 322                 rte_errno = ENOMEM;
 323                 return NULL;
 324         }
 325         memset(in, 0, in_size_dw * 4);
 326         pgsize = rte_mem_page_size();
 327         if (pgsize == (size_t)-1) {
 328                 mlx5_free(mkey);
 329                 DRV_LOG(ERR, "Failed to get page size");
 330                 rte_errno = ENOMEM;
 331                 return NULL;
 332         }
 333         MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
 334         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 335         if (klm_num > 0) {
 336                 int i;
 337                 uint8_t *klm = (uint8_t *)MLX5_ADDR_OF(create_mkey_in, in,
 338                                                        klm_pas_mtt);
 339                 translation_size = RTE_ALIGN(klm_num, 4);
 340                 for (i = 0; i < klm_num; i++) {
 341                         MLX5_SET(klm, klm, byte_count, klm_array[i].byte_count);
 342                         MLX5_SET(klm, klm, mkey, klm_array[i].mkey);
 343                         MLX5_SET64(klm, klm, address, klm_array[i].address);
 344                         klm += MLX5_ST_SZ_BYTES(klm);
 345                 }
 346                 for (; i < (int)translation_size; i++) {
 347                         MLX5_SET(klm, klm, mkey, 0x0);
 348                         MLX5_SET64(klm, klm, address, 0x0);
 349                         klm += MLX5_ST_SZ_BYTES(klm);
 350                 }
 351                 MLX5_SET(mkc, mkc, access_mode_1_0, attr->log_entity_size ?
 352                          MLX5_MKC_ACCESS_MODE_KLM_FBS :
 353                          MLX5_MKC_ACCESS_MODE_KLM);
 354                 MLX5_SET(mkc, mkc, log_page_size, attr->log_entity_size);
 355         } else {
 356                 translation_size = (RTE_ALIGN(attr->size, pgsize) * 8) / 16;
 357                 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
 358                 MLX5_SET(mkc, mkc, log_page_size, rte_log2_u32(pgsize));
 359         }
 360         MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
 361                  translation_size);
 362         MLX5_SET(create_mkey_in, in, mkey_umem_id, attr->umem_id);
 363         MLX5_SET(create_mkey_in, in, pg_access, attr->pg_access);
 364         MLX5_SET(mkc, mkc, lw, 0x1);
 365         MLX5_SET(mkc, mkc, lr, 0x1);
 366         if (attr->set_remote_rw) {
 367                 MLX5_SET(mkc, mkc, rw, 0x1);
 368                 MLX5_SET(mkc, mkc, rr, 0x1);
 369         }
 370         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 371         MLX5_SET(mkc, mkc, pd, attr->pd);
 372         MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF);
 373         MLX5_SET(mkc, mkc, umr_en, attr->umr_en);
 374         MLX5_SET(mkc, mkc, translations_octword_size, translation_size);
 375         MLX5_SET(mkc, mkc, relaxed_ordering_write,
 376                  attr->relaxed_ordering_write);
 377         MLX5_SET(mkc, mkc, relaxed_ordering_read, attr->relaxed_ordering_read);
 378         MLX5_SET64(mkc, mkc, start_addr, attr->addr);
 379         MLX5_SET64(mkc, mkc, len, attr->size);
 380         MLX5_SET(mkc, mkc, crypto_en, attr->crypto_en);
 381         if (attr->crypto_en) {
 382                 MLX5_SET(mkc, mkc, bsf_en, attr->crypto_en);
 383                 MLX5_SET(mkc, mkc, bsf_octword_size, 4);
 384         }
 385         mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out,
 386                                                sizeof(out));
 387         if (!mkey->obj) {
 388                 DRV_LOG(ERR, "Can't create %sdirect mkey - error %d",
 389                         klm_num ? "an in" : "a ", errno);
 390                 rte_errno = errno;
 391                 mlx5_free(mkey);
 392                 return NULL;
 393         }
 394         mkey->id = MLX5_GET(create_mkey_out, out, mkey_index);
 395         mkey->id = (mkey->id << 8) | (attr->umem_id & 0xFF);
 396         return mkey;
 397 }
 398
 399 /**
 400  * Get status of devx command response.
 401  * Mainly used for asynchronous commands.
 402  *
 403  * @param[in] out
 404  *   The out response buffer.
 405  *
 406  * @return
 407  *   0 on success, non-zero value otherwise.
 408  */
 409 int
 410 mlx5_devx_get_out_command_status(void *out)
 411 {
 412         int status;
 413
 414         if (!out)
 415                 return -EINVAL;
 416         status = MLX5_GET(query_flow_counter_out, out, status);
 417         if (status) {
 418                 int syndrome = MLX5_GET(query_flow_counter_out, out, syndrome);
 419
 420                 DRV_LOG(ERR, "Bad DevX status %x, syndrome = %x", status,
 421                         syndrome);
 422         }
 423         return status;
 424 }
 425
 426 /**
 427  * Destroy any object allocated by a Devx API.
 428  *
 429  * @param[in] obj
 430  *   Pointer to a general object.
 431  *
 432  * @return
 433  *   0 on success, a negative value otherwise.
 434  */
 435 int
 436 mlx5_devx_cmd_destroy(struct mlx5_devx_obj *obj)
 437 {
 438         int ret;
 439
 440         if (!obj)
 441                 return 0;
 442         ret =  mlx5_glue->devx_obj_destroy(obj->obj);
 443         mlx5_free(obj);
 444         return ret;
 445 }
 446
 447 /**
 448  * Query NIC vport context.
 449  * Fills minimal inline attribute.
 450  *
 451  * @param[in] ctx
 452  *   ibv contexts returned from mlx5dv_open_device.
 453  * @param[in] vport
 454  *   vport index
 455  * @param[out] attr
 456  *   Attributes device values.
 457  *
 458  * @return
 459  *   0 on success, a negative value otherwise.
 460  */
 461 static int
 462 mlx5_devx_cmd_query_nic_vport_context(void *ctx,
 463                                       unsigned int vport,
 464                                       struct mlx5_hca_attr *attr)
 465 {
 466         uint32_t in[MLX5_ST_SZ_DW(query_nic_vport_context_in)] = {0};
 467         uint32_t out[MLX5_ST_SZ_DW(query_nic_vport_context_out)] = {0};
 468         void *vctx;
 469         int status, syndrome, rc;
 470
 471         /* Query NIC vport context to determine inline mode. */
 472         MLX5_SET(query_nic_vport_context_in, in, opcode,
 473                  MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT);
 474         MLX5_SET(query_nic_vport_context_in, in, vport_number, vport);
 475         if (vport)
 476                 MLX5_SET(query_nic_vport_context_in, in, other_vport, 1);
 477         rc = mlx5_glue->devx_general_cmd(ctx,
 478                                          in, sizeof(in),
 479                                          out, sizeof(out));
 480         if (rc)
 481                 goto error;
 482         status = MLX5_GET(query_nic_vport_context_out, out, status);
 483         syndrome = MLX5_GET(query_nic_vport_context_out, out, syndrome);
 484         if (status) {
 485                 DRV_LOG(DEBUG, "Failed to query NIC vport context, "
 486                         "status %x, syndrome = %x", status, syndrome);
 487                 return -1;
 488         }
 489         vctx = MLX5_ADDR_OF(query_nic_vport_context_out, out,
 490                             nic_vport_context);
 491         attr->vport_inline_mode = MLX5_GET(nic_vport_context, vctx,
 492                                            min_wqe_inline_mode);
 493         return 0;
 494 error:
 495         rc = (rc > 0) ? -rc : rc;
 496         return rc;
 497 }
 498
 499 /**
 500  * Query NIC vDPA attributes.
 501  *
 502  * @param[in] ctx
 503  *   Context returned from mlx5 open_device() glue function.
 504  * @param[out] vdpa_attr
 505  *   vDPA Attributes structure to fill.
 506  */
 507 static void
 508 mlx5_devx_cmd_query_hca_vdpa_attr(void *ctx,
 509                                   struct mlx5_hca_vdpa_attr *vdpa_attr)
 510 {
 511         uint32_t in[MLX5_ST_SZ_DW(query_hca_cap_in)];
 512         uint32_t out[MLX5_ST_SZ_DW(query_hca_cap_out)];
 513         void *hcattr;
 514
 515         hcattr = mlx5_devx_get_hca_cap(ctx, in, out, NULL,
 516                         MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION |
 517                         MLX5_HCA_CAP_OPMOD_GET_CUR);
 518         if (!hcattr) {
 519                 RTE_LOG(DEBUG, PMD, "Failed to query devx VDPA capabilities");
 520                 vdpa_attr->valid = 0;
 521         } else {
 522                 vdpa_attr->valid = 1;
 523                 vdpa_attr->desc_tunnel_offload_type =
 524                         MLX5_GET(virtio_emulation_cap, hcattr,
 525                                  desc_tunnel_offload_type);
 526                 vdpa_attr->eth_frame_offload_type =
 527                         MLX5_GET(virtio_emulation_cap, hcattr,
 528                                  eth_frame_offload_type);
 529                 vdpa_attr->virtio_version_1_0 =
 530                         MLX5_GET(virtio_emulation_cap, hcattr,
 531                                  virtio_version_1_0);
 532                 vdpa_attr->tso_ipv4 = MLX5_GET(virtio_emulation_cap, hcattr,
 533                                                tso_ipv4);
 534                 vdpa_attr->tso_ipv6 = MLX5_GET(virtio_emulation_cap, hcattr,
 535                                                tso_ipv6);
 536                 vdpa_attr->tx_csum = MLX5_GET(virtio_emulation_cap, hcattr,
 537                                               tx_csum);
 538                 vdpa_attr->rx_csum = MLX5_GET(virtio_emulation_cap, hcattr,
 539                                               rx_csum);
 540                 vdpa_attr->event_mode = MLX5_GET(virtio_emulation_cap, hcattr,
 541                                                  event_mode);
 542                 vdpa_attr->virtio_queue_type =
 543                         MLX5_GET(virtio_emulation_cap, hcattr,
 544                                  virtio_queue_type);
 545                 vdpa_attr->log_doorbell_stride =
 546                         MLX5_GET(virtio_emulation_cap, hcattr,
 547                                  log_doorbell_stride);
 548                 vdpa_attr->log_doorbell_bar_size =
 549                         MLX5_GET(virtio_emulation_cap, hcattr,
 550                                  log_doorbell_bar_size);
 551                 vdpa_attr->doorbell_bar_offset =
 552                         MLX5_GET64(virtio_emulation_cap, hcattr,
 553                                    doorbell_bar_offset);
 554                 vdpa_attr->max_num_virtio_queues =
 555                         MLX5_GET(virtio_emulation_cap, hcattr,
 556                                  max_num_virtio_queues);
 557                 vdpa_attr->umems[0].a = MLX5_GET(virtio_emulation_cap, hcattr,
 558                                                  umem_1_buffer_param_a);
 559                 vdpa_attr->umems[0].b = MLX5_GET(virtio_emulation_cap, hcattr,
 560                                                  umem_1_buffer_param_b);
 561                 vdpa_attr->umems[1].a = MLX5_GET(virtio_emulation_cap, hcattr,
 562                                                  umem_2_buffer_param_a);
 563                 vdpa_attr->umems[1].b = MLX5_GET(virtio_emulation_cap, hcattr,
 564                                                  umem_2_buffer_param_b);
 565                 vdpa_attr->umems[2].a = MLX5_GET(virtio_emulation_cap, hcattr,
 566                                                  umem_3_buffer_param_a);
 567                 vdpa_attr->umems[2].b = MLX5_GET(virtio_emulation_cap, hcattr,
 568                                                  umem_3_buffer_param_b);
 569         }
 570 }
 571
 572 int
 573 mlx5_devx_cmd_query_parse_samples(struct mlx5_devx_obj *flex_obj,
 574                                   uint32_t ids[], uint32_t num)
 575 {
 576         uint32_t in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {0};
 577         uint32_t out[MLX5_ST_SZ_DW(create_flex_parser_out)] = {0};
 578         void *hdr = MLX5_ADDR_OF(create_flex_parser_out, in, hdr);
 579         void *flex = MLX5_ADDR_OF(create_flex_parser_out, out, flex);
 580         void *sample = MLX5_ADDR_OF(parse_graph_flex, flex, sample_table);
 581         int ret;
 582         uint32_t idx = 0;
 583         uint32_t i;
 584
 585         if (num > MLX5_GRAPH_NODE_SAMPLE_NUM) {
 586                 rte_errno = EINVAL;
 587                 DRV_LOG(ERR, "Too many sample IDs to be fetched.");
 588                 return -rte_errno;
 589         }
 590         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
 591                  MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
 592         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
 593                  MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH);
 594         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, flex_obj->id);
 595         ret = mlx5_glue->devx_obj_query(flex_obj->obj, in, sizeof(in),
 596                                         out, sizeof(out));
 597         if (ret) {
 598                 rte_errno = ret;
 599                 DRV_LOG(ERR, "Failed to query sample IDs with object %p.",
 600                         (void *)flex_obj);
 601                 return -rte_errno;
 602         }
 603         for (i = 0; i < MLX5_GRAPH_NODE_SAMPLE_NUM; i++) {
 604                 void *s_off = (void *)((char *)sample + i *
 605                               MLX5_ST_SZ_BYTES(parse_graph_flow_match_sample));
 606                 uint32_t en;
 607
 608                 en = MLX5_GET(parse_graph_flow_match_sample, s_off,
 609                               flow_match_sample_en);
 610                 if (!en)
 611                         continue;
 612                 ids[idx++] = MLX5_GET(parse_graph_flow_match_sample, s_off,
 613                                   flow_match_sample_field_id);
 614         }
 615         if (num != idx) {
 616                 rte_errno = EINVAL;
 617                 DRV_LOG(ERR, "Number of sample IDs are not as expected.");
 618                 return -rte_errno;
 619         }
 620         return ret;
 621 }
 622
 623 struct mlx5_devx_obj *
 624 mlx5_devx_cmd_create_flex_parser(void *ctx,
 625                                  struct mlx5_devx_graph_node_attr *data)
 626 {
 627         uint32_t in[MLX5_ST_SZ_DW(create_flex_parser_in)] = {0};
 628         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
 629         void *hdr = MLX5_ADDR_OF(create_flex_parser_in, in, hdr);
 630         void *flex = MLX5_ADDR_OF(create_flex_parser_in, in, flex);
 631         void *sample = MLX5_ADDR_OF(parse_graph_flex, flex, sample_table);
 632         void *in_arc = MLX5_ADDR_OF(parse_graph_flex, flex, input_arc);
 633         void *out_arc = MLX5_ADDR_OF(parse_graph_flex, flex, output_arc);
 634         struct mlx5_devx_obj *parse_flex_obj = mlx5_malloc
 635                      (MLX5_MEM_ZERO, sizeof(*parse_flex_obj), 0, SOCKET_ID_ANY);
 636         uint32_t i;
 637
 638         if (!parse_flex_obj) {
 639                 DRV_LOG(ERR, "Failed to allocate flex parser data.");
 640                 rte_errno = ENOMEM;
 641                 return NULL;
 642         }
 643         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
 644                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
 645         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
 646                  MLX5_GENERAL_OBJ_TYPE_FLEX_PARSE_GRAPH);
 647         MLX5_SET(parse_graph_flex, flex, header_length_mode,
 648                  data->header_length_mode);
 649         MLX5_SET64(parse_graph_flex, flex, modify_field_select,
 650                    data->modify_field_select);
 651         MLX5_SET(parse_graph_flex, flex, header_length_base_value,
 652                  data->header_length_base_value);
 653         MLX5_SET(parse_graph_flex, flex, header_length_field_offset,
 654                  data->header_length_field_offset);
 655         MLX5_SET(parse_graph_flex, flex, header_length_field_shift,
 656                  data->header_length_field_shift);
 657         MLX5_SET(parse_graph_flex, flex, next_header_field_offset,
 658                  data->next_header_field_offset);
 659         MLX5_SET(parse_graph_flex, flex, next_header_field_size,
 660                  data->next_header_field_size);
 661         MLX5_SET(parse_graph_flex, flex, header_length_field_mask,
 662                  data->header_length_field_mask);
 663         for (i = 0; i < MLX5_GRAPH_NODE_SAMPLE_NUM; i++) {
 664                 struct mlx5_devx_match_sample_attr *s = &data->sample[i];
 665                 void *s_off = (void *)((char *)sample + i *
 666                               MLX5_ST_SZ_BYTES(parse_graph_flow_match_sample));
 667
 668                 if (!s->flow_match_sample_en)
 669                         continue;
 670                 MLX5_SET(parse_graph_flow_match_sample, s_off,
 671                          flow_match_sample_en, !!s->flow_match_sample_en);
 672                 MLX5_SET(parse_graph_flow_match_sample, s_off,
 673                          flow_match_sample_field_offset,
 674                          s->flow_match_sample_field_offset);
 675                 MLX5_SET(parse_graph_flow_match_sample, s_off,
 676                          flow_match_sample_offset_mode,
 677                          s->flow_match_sample_offset_mode);
 678                 MLX5_SET(parse_graph_flow_match_sample, s_off,
 679                          flow_match_sample_field_offset_mask,
 680                          s->flow_match_sample_field_offset_mask);
 681                 MLX5_SET(parse_graph_flow_match_sample, s_off,
 682                          flow_match_sample_field_offset_shift,
 683                          s->flow_match_sample_field_offset_shift);
 684                 MLX5_SET(parse_graph_flow_match_sample, s_off,
 685                          flow_match_sample_field_base_offset,
 686                          s->flow_match_sample_field_base_offset);
 687                 MLX5_SET(parse_graph_flow_match_sample, s_off,
 688                          flow_match_sample_tunnel_mode,
 689                          s->flow_match_sample_tunnel_mode);
 690         }
 691         for (i = 0; i < MLX5_GRAPH_NODE_ARC_NUM; i++) {
 692                 struct mlx5_devx_graph_arc_attr *ia = &data->in[i];
 693                 struct mlx5_devx_graph_arc_attr *oa = &data->out[i];
 694                 void *in_off = (void *)((char *)in_arc + i *
 695                               MLX5_ST_SZ_BYTES(parse_graph_arc));
 696                 void *out_off = (void *)((char *)out_arc + i *
 697                               MLX5_ST_SZ_BYTES(parse_graph_arc));
 698
 699                 if (ia->arc_parse_graph_node != 0) {
 700                         MLX5_SET(parse_graph_arc, in_off,
 701                                  compare_condition_value,
 702                                  ia->compare_condition_value);
 703                         MLX5_SET(parse_graph_arc, in_off, start_inner_tunnel,
 704                                  ia->start_inner_tunnel);
 705                         MLX5_SET(parse_graph_arc, in_off, arc_parse_graph_node,
 706                                  ia->arc_parse_graph_node);
 707                         MLX5_SET(parse_graph_arc, in_off,
 708                                  parse_graph_node_handle,
 709                                  ia->parse_graph_node_handle);
 710                 }
 711                 if (oa->arc_parse_graph_node != 0) {
 712                         MLX5_SET(parse_graph_arc, out_off,
 713                                  compare_condition_value,
 714                                  oa->compare_condition_value);
 715                         MLX5_SET(parse_graph_arc, out_off, start_inner_tunnel,
 716                                  oa->start_inner_tunnel);
 717                         MLX5_SET(parse_graph_arc, out_off, arc_parse_graph_node,
 718                                  oa->arc_parse_graph_node);
 719                         MLX5_SET(parse_graph_arc, out_off,
 720                                  parse_graph_node_handle,
 721                                  oa->parse_graph_node_handle);
 722                 }
 723         }
 724         parse_flex_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
 725                                                          out, sizeof(out));
 726         if (!parse_flex_obj->obj) {
 727                 rte_errno = errno;
 728                 DRV_LOG(ERR, "Failed to create FLEX PARSE GRAPH object "
 729                         "by using DevX.");
 730                 mlx5_free(parse_flex_obj);
 731                 return NULL;
 732         }
 733         parse_flex_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 734         return parse_flex_obj;
 735 }
 736
 737 static int
 738 mlx5_devx_cmd_query_hca_parse_graph_node_cap
 739         (void *ctx, struct mlx5_hca_flex_attr *attr)
 740 {
 741         uint32_t in[MLX5_ST_SZ_DW(query_hca_cap_in)];
 742         uint32_t out[MLX5_ST_SZ_DW(query_hca_cap_out)];
 743         void *hcattr;
 744         int rc;
 745
 746         hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
 747                         MLX5_GET_HCA_CAP_OP_MOD_PARSE_GRAPH_NODE_CAP |
 748                         MLX5_HCA_CAP_OPMOD_GET_CUR);
 749         if (!hcattr)
 750                 return rc;
 751         attr->node_in = MLX5_GET(parse_graph_node_cap, hcattr, node_in);
 752         attr->node_out = MLX5_GET(parse_graph_node_cap, hcattr, node_out);
 753         attr->header_length_mode = MLX5_GET(parse_graph_node_cap, hcattr,
 754                                             header_length_mode);
 755         attr->sample_offset_mode = MLX5_GET(parse_graph_node_cap, hcattr,
 756                                             sample_offset_mode);
 757         attr->max_num_arc_in = MLX5_GET(parse_graph_node_cap, hcattr,
 758                                         max_num_arc_in);
 759         attr->max_num_arc_out = MLX5_GET(parse_graph_node_cap, hcattr,
 760                                          max_num_arc_out);
 761         attr->max_num_sample = MLX5_GET(parse_graph_node_cap, hcattr,
 762                                         max_num_sample);
 763         attr->sample_id_in_out = MLX5_GET(parse_graph_node_cap, hcattr,
 764                                           sample_id_in_out);
 765         attr->max_base_header_length = MLX5_GET(parse_graph_node_cap, hcattr,
 766                                                 max_base_header_length);
 767         attr->max_sample_base_offset = MLX5_GET(parse_graph_node_cap, hcattr,
 768                                                 max_sample_base_offset);
 769         attr->max_next_header_offset = MLX5_GET(parse_graph_node_cap, hcattr,
 770                                                 max_next_header_offset);
 771         attr->header_length_mask_width = MLX5_GET(parse_graph_node_cap, hcattr,
 772                                                   header_length_mask_width);
 773         /* Get the max supported samples from HCA CAP 2 */
 774         hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
 775                         MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE_2 |
 776                         MLX5_HCA_CAP_OPMOD_GET_CUR);
 777         if (!hcattr)
 778                 return rc;
 779         attr->max_num_prog_sample =
 780                 MLX5_GET(cmd_hca_cap_2, hcattr, max_num_prog_sample_field);
 781         return 0;
 782 }
 783
 784 static int
 785 mlx5_devx_query_pkt_integrity_match(void *hcattr)
 786 {
 787         return MLX5_GET(flow_table_nic_cap, hcattr,
 788                         ft_field_support_2_nic_receive.inner_l3_ok) &&
 789                MLX5_GET(flow_table_nic_cap, hcattr,
 790                         ft_field_support_2_nic_receive.inner_l4_ok) &&
 791                MLX5_GET(flow_table_nic_cap, hcattr,
 792                         ft_field_support_2_nic_receive.outer_l3_ok) &&
 793                MLX5_GET(flow_table_nic_cap, hcattr,
 794                         ft_field_support_2_nic_receive.outer_l4_ok) &&
 795                MLX5_GET(flow_table_nic_cap, hcattr,
 796                         ft_field_support_2_nic_receive
 797                                 .inner_ipv4_checksum_ok) &&
 798                MLX5_GET(flow_table_nic_cap, hcattr,
 799                         ft_field_support_2_nic_receive.inner_l4_checksum_ok) &&
 800                MLX5_GET(flow_table_nic_cap, hcattr,
 801                         ft_field_support_2_nic_receive
 802                                 .outer_ipv4_checksum_ok) &&
 803                MLX5_GET(flow_table_nic_cap, hcattr,
 804                         ft_field_support_2_nic_receive.outer_l4_checksum_ok);
 805 }
 806
 807 /**
 808  * Query HCA attributes.
 809  * Using those attributes we can check on run time if the device
 810  * is having the required capabilities.
 811  *
 812  * @param[in] ctx
 813  *   Context returned from mlx5 open_device() glue function.
 814  * @param[out] attr
 815  *   Attributes device values.
 816  *
 817  * @return
 818  *   0 on success, a negative value otherwise.
 819  */
 820 int
 821 mlx5_devx_cmd_query_hca_attr(void *ctx,
 822                              struct mlx5_hca_attr *attr)
 823 {
 824         uint32_t in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {0};
 825         uint32_t out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {0};
 826         bool hca_cap_2_sup;
 827         uint64_t general_obj_types_supported = 0;
 828         void *hcattr;
 829         int rc, i;
 830
 831         hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
 832                         MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE |
 833                         MLX5_HCA_CAP_OPMOD_GET_CUR);
 834         if (!hcattr)
 835                 return rc;
 836         hca_cap_2_sup = MLX5_GET(cmd_hca_cap, hcattr, hca_cap_2);
 837         attr->max_wqe_sz_sq = MLX5_GET(cmd_hca_cap, hcattr, max_wqe_sz_sq);
 838         attr->flow_counter_bulk_alloc_bitmap =
 839                         MLX5_GET(cmd_hca_cap, hcattr, flow_counter_bulk_alloc);
 840         attr->flow_counters_dump = MLX5_GET(cmd_hca_cap, hcattr,
 841                                             flow_counters_dump);
 842         attr->log_max_rmp = MLX5_GET(cmd_hca_cap, hcattr, log_max_rmp);
 843         attr->mem_rq_rmp = MLX5_GET(cmd_hca_cap, hcattr, mem_rq_rmp);
 844         attr->log_max_rqt_size = MLX5_GET(cmd_hca_cap, hcattr,
 845                                           log_max_rqt_size);
 846         attr->eswitch_manager = MLX5_GET(cmd_hca_cap, hcattr, eswitch_manager);
 847         attr->hairpin = MLX5_GET(cmd_hca_cap, hcattr, hairpin);
 848         attr->log_max_hairpin_queues = MLX5_GET(cmd_hca_cap, hcattr,
 849                                                 log_max_hairpin_queues);
 850         attr->log_max_hairpin_wq_data_sz = MLX5_GET(cmd_hca_cap, hcattr,
 851                                                     log_max_hairpin_wq_data_sz);
 852         attr->log_max_hairpin_num_packets = MLX5_GET
 853                 (cmd_hca_cap, hcattr, log_min_hairpin_wq_data_sz);
 854         attr->vhca_id = MLX5_GET(cmd_hca_cap, hcattr, vhca_id);
 855         attr->relaxed_ordering_write = MLX5_GET(cmd_hca_cap, hcattr,
 856                                                 relaxed_ordering_write);
 857         attr->relaxed_ordering_read = MLX5_GET(cmd_hca_cap, hcattr,
 858                                                relaxed_ordering_read);
 859         attr->access_register_user = MLX5_GET(cmd_hca_cap, hcattr,
 860                                               access_register_user);
 861         attr->eth_net_offloads = MLX5_GET(cmd_hca_cap, hcattr,
 862                                           eth_net_offloads);
 863         attr->eth_virt = MLX5_GET(cmd_hca_cap, hcattr, eth_virt);
 864         attr->flex_parser_protocols = MLX5_GET(cmd_hca_cap, hcattr,
 865                                                flex_parser_protocols);
 866         attr->max_geneve_tlv_options = MLX5_GET(cmd_hca_cap, hcattr,
 867                         max_geneve_tlv_options);
 868         attr->max_geneve_tlv_option_data_len = MLX5_GET(cmd_hca_cap, hcattr,
 869                         max_geneve_tlv_option_data_len);
 870         attr->qos.sup = MLX5_GET(cmd_hca_cap, hcattr, qos);
 871         attr->qos.flow_meter_aso_sup = !!(MLX5_GET64(cmd_hca_cap, hcattr,
 872                                          general_obj_types) &
 873                               MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_METER_ASO);
 874         attr->vdpa.valid = !!(MLX5_GET64(cmd_hca_cap, hcattr,
 875                                          general_obj_types) &
 876                               MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q);
 877         attr->vdpa.queue_counters_valid = !!(MLX5_GET64(cmd_hca_cap, hcattr,
 878                                                         general_obj_types) &
 879                                   MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS);
 880         attr->parse_graph_flex_node = !!(MLX5_GET64(cmd_hca_cap, hcattr,
 881                                          general_obj_types) &
 882                               MLX5_GENERAL_OBJ_TYPES_CAP_PARSE_GRAPH_FLEX_NODE);
 883         attr->wqe_index_ignore = MLX5_GET(cmd_hca_cap, hcattr,
 884                                           wqe_index_ignore_cap);
 885         attr->cross_channel = MLX5_GET(cmd_hca_cap, hcattr, cd);
 886         attr->non_wire_sq = MLX5_GET(cmd_hca_cap, hcattr, non_wire_sq);
 887         attr->log_max_static_sq_wq = MLX5_GET(cmd_hca_cap, hcattr,
 888                                               log_max_static_sq_wq);
 889         attr->num_lag_ports = MLX5_GET(cmd_hca_cap, hcattr, num_lag_ports);
 890         attr->dev_freq_khz = MLX5_GET(cmd_hca_cap, hcattr,
 891                                       device_frequency_khz);
 892         attr->scatter_fcs_w_decap_disable =
 893                 MLX5_GET(cmd_hca_cap, hcattr, scatter_fcs_w_decap_disable);
 894         attr->roce = MLX5_GET(cmd_hca_cap, hcattr, roce);
 895         attr->rq_ts_format = MLX5_GET(cmd_hca_cap, hcattr, rq_ts_format);
 896         attr->sq_ts_format = MLX5_GET(cmd_hca_cap, hcattr, sq_ts_format);
 897         attr->steering_format_version =
 898                 MLX5_GET(cmd_hca_cap, hcattr, steering_format_version);
 899         attr->regexp_params = MLX5_GET(cmd_hca_cap, hcattr, regexp_params);
 900         attr->regexp_version = MLX5_GET(cmd_hca_cap, hcattr, regexp_version);
 901         attr->regexp_num_of_engines = MLX5_GET(cmd_hca_cap, hcattr,
 902                                                regexp_num_of_engines);
 903         /* Read the general_obj_types bitmap and extract the relevant bits. */
 904         general_obj_types_supported = MLX5_GET64(cmd_hca_cap, hcattr,
 905                                                  general_obj_types);
 906         attr->vdpa.valid = !!(general_obj_types_supported &
 907                               MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q);
 908         attr->vdpa.queue_counters_valid =
 909                         !!(general_obj_types_supported &
 910                            MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS);
 911         attr->parse_graph_flex_node =
 912                         !!(general_obj_types_supported &
 913                            MLX5_GENERAL_OBJ_TYPES_CAP_PARSE_GRAPH_FLEX_NODE);
 914         attr->flow_hit_aso = !!(general_obj_types_supported &
 915                                 MLX5_GENERAL_OBJ_TYPES_CAP_FLOW_HIT_ASO);
 916         attr->geneve_tlv_opt = !!(general_obj_types_supported &
 917                                   MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT);
 918         attr->dek = !!(general_obj_types_supported &
 919                        MLX5_GENERAL_OBJ_TYPES_CAP_DEK);
 920         attr->import_kek = !!(general_obj_types_supported &
 921                               MLX5_GENERAL_OBJ_TYPES_CAP_IMPORT_KEK);
 922         attr->credential = !!(general_obj_types_supported &
 923                               MLX5_GENERAL_OBJ_TYPES_CAP_CREDENTIAL);
 924         attr->crypto_login = !!(general_obj_types_supported &
 925                                 MLX5_GENERAL_OBJ_TYPES_CAP_CRYPTO_LOGIN);
 926         /* Add reading of other GENERAL_OBJ_TYPES_CAP bits above this line. */
 927         attr->log_max_cq = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq);
 928         attr->log_max_qp = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp);
 929         attr->log_max_cq_sz = MLX5_GET(cmd_hca_cap, hcattr, log_max_cq_sz);
 930         attr->log_max_qp_sz = MLX5_GET(cmd_hca_cap, hcattr, log_max_qp_sz);
 931         attr->log_max_mrw_sz = MLX5_GET(cmd_hca_cap, hcattr, log_max_mrw_sz);
 932         attr->log_max_pd = MLX5_GET(cmd_hca_cap, hcattr, log_max_pd);
 933         attr->log_max_srq = MLX5_GET(cmd_hca_cap, hcattr, log_max_srq);
 934         attr->log_max_srq_sz = MLX5_GET(cmd_hca_cap, hcattr, log_max_srq_sz);
 935         attr->reg_c_preserve =
 936                 MLX5_GET(cmd_hca_cap, hcattr, reg_c_preserve);
 937         attr->mmo_regex_qp_en = MLX5_GET(cmd_hca_cap, hcattr, regexp_mmo_qp);
 938         attr->mmo_regex_sq_en = MLX5_GET(cmd_hca_cap, hcattr, regexp_mmo_sq);
 939         attr->mmo_dma_sq_en = MLX5_GET(cmd_hca_cap, hcattr, dma_mmo_sq);
 940         attr->mmo_compress_sq_en = MLX5_GET(cmd_hca_cap, hcattr,
 941                         compress_mmo_sq);
 942         attr->mmo_decompress_sq_en = MLX5_GET(cmd_hca_cap, hcattr,
 943                         decompress_mmo_sq);
 944         attr->mmo_dma_qp_en = MLX5_GET(cmd_hca_cap, hcattr, dma_mmo_qp);
 945         attr->mmo_compress_qp_en = MLX5_GET(cmd_hca_cap, hcattr,
 946                         compress_mmo_qp);
 947         attr->mmo_decompress_qp_en = MLX5_GET(cmd_hca_cap, hcattr,
 948                         decompress_mmo_qp);
 949         attr->compress_min_block_size = MLX5_GET(cmd_hca_cap, hcattr,
 950                                                  compress_min_block_size);
 951         attr->log_max_mmo_dma = MLX5_GET(cmd_hca_cap, hcattr, log_dma_mmo_size);
 952         attr->log_max_mmo_compress = MLX5_GET(cmd_hca_cap, hcattr,
 953                                               log_compress_mmo_size);
 954         attr->log_max_mmo_decompress = MLX5_GET(cmd_hca_cap, hcattr,
 955                                                 log_decompress_mmo_size);
 956         attr->cqe_compression = MLX5_GET(cmd_hca_cap, hcattr, cqe_compression);
 957         attr->mini_cqe_resp_flow_tag = MLX5_GET(cmd_hca_cap, hcattr,
 958                                                 mini_cqe_resp_flow_tag);
 959         attr->mini_cqe_resp_l3_l4_tag = MLX5_GET(cmd_hca_cap, hcattr,
 960                                                  mini_cqe_resp_l3_l4_tag);
 961         attr->umr_indirect_mkey_disabled =
 962                 MLX5_GET(cmd_hca_cap, hcattr, umr_indirect_mkey_disabled);
 963         attr->umr_modify_entity_size_disabled =
 964                 MLX5_GET(cmd_hca_cap, hcattr, umr_modify_entity_size_disabled);
 965         attr->wait_on_time = MLX5_GET(cmd_hca_cap, hcattr, wait_on_time);
 966         attr->crypto = MLX5_GET(cmd_hca_cap, hcattr, crypto);
 967         if (attr->crypto)
 968                 attr->aes_xts = MLX5_GET(cmd_hca_cap, hcattr, aes_xts);
 969         attr->ct_offload = !!(MLX5_GET64(cmd_hca_cap, hcattr,
 970                                          general_obj_types) &
 971                               MLX5_GENERAL_OBJ_TYPES_CAP_CONN_TRACK_OFFLOAD);
 972         attr->rq_delay_drop = MLX5_GET(cmd_hca_cap, hcattr, rq_delay_drop);
 973         if (hca_cap_2_sup) {
 974                 hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
 975                                 MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE_2 |
 976                                 MLX5_HCA_CAP_OPMOD_GET_CUR);
 977                 if (!hcattr) {
 978                         DRV_LOG(DEBUG,
 979                                 "Failed to query DevX HCA capabilities 2.");
 980                         return rc;
 981                 }
 982                 attr->log_min_stride_wqe_sz = MLX5_GET(cmd_hca_cap_2, hcattr,
 983                                                        log_min_stride_wqe_sz);
 984         }
 985         if (attr->log_min_stride_wqe_sz == 0)
 986                 attr->log_min_stride_wqe_sz = MLX5_MPRQ_LOG_MIN_STRIDE_WQE_SIZE;
 987         if (attr->qos.sup) {
 988                 hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
 989                                 MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP |
 990                                 MLX5_HCA_CAP_OPMOD_GET_CUR);
 991                 if (!hcattr) {
 992                         DRV_LOG(DEBUG, "Failed to query devx QOS capabilities");
 993                         return rc;
 994                 }
 995                 attr->qos.flow_meter_old =
 996                                 MLX5_GET(qos_cap, hcattr, flow_meter_old);
 997                 attr->qos.log_max_flow_meter =
 998                                 MLX5_GET(qos_cap, hcattr, log_max_flow_meter);
 999                 attr->qos.flow_meter_reg_c_ids =
1000                                 MLX5_GET(qos_cap, hcattr, flow_meter_reg_id);
1001                 attr->qos.flow_meter =
1002                                 MLX5_GET(qos_cap, hcattr, flow_meter);
1003                 attr->qos.packet_pacing =
1004                                 MLX5_GET(qos_cap, hcattr, packet_pacing);
1005                 attr->qos.wqe_rate_pp =
1006                                 MLX5_GET(qos_cap, hcattr, wqe_rate_pp);
1007                 if (attr->qos.flow_meter_aso_sup) {
1008                         attr->qos.log_meter_aso_granularity =
1009                                 MLX5_GET(qos_cap, hcattr,
1010                                         log_meter_aso_granularity);
1011                         attr->qos.log_meter_aso_max_alloc =
1012                                 MLX5_GET(qos_cap, hcattr,
1013                                         log_meter_aso_max_alloc);
1014                         attr->qos.log_max_num_meter_aso =
1015                                 MLX5_GET(qos_cap, hcattr,
1016                                         log_max_num_meter_aso);
1017                 }
1018         }
1019         /*
1020          * Flex item support needs max_num_prog_sample_field
1021          * from the Capabilities 2 table for PARSE_GRAPH_NODE
1022          */
1023         if (attr->parse_graph_flex_node) {
1024                 rc = mlx5_devx_cmd_query_hca_parse_graph_node_cap
1025                         (ctx, &attr->flex);
1026                 if (rc)
1027                         return -1;
1028         }
1029         if (attr->vdpa.valid)
1030                 mlx5_devx_cmd_query_hca_vdpa_attr(ctx, &attr->vdpa);
1031         if (!attr->eth_net_offloads)
1032                 return 0;
1033         /* Query Flow Sampler Capability From FLow Table Properties Layout. */
1034         hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
1035                         MLX5_GET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE |
1036                         MLX5_HCA_CAP_OPMOD_GET_CUR);
1037         if (!hcattr) {
1038                 attr->log_max_ft_sampler_num = 0;
1039                 return rc;
1040         }
1041         attr->log_max_ft_sampler_num = MLX5_GET
1042                 (flow_table_nic_cap, hcattr,
1043                  flow_table_properties_nic_receive.log_max_ft_sampler_num);
1044         attr->flow.tunnel_header_0_1 = MLX5_GET
1045                 (flow_table_nic_cap, hcattr,
1046                  ft_field_support_2_nic_receive.tunnel_header_0_1);
1047         attr->pkt_integrity_match = mlx5_devx_query_pkt_integrity_match(hcattr);
1048         attr->inner_ipv4_ihl = MLX5_GET
1049                 (flow_table_nic_cap, hcattr,
1050                  ft_field_support_2_nic_receive.inner_ipv4_ihl);
1051         attr->outer_ipv4_ihl = MLX5_GET
1052                 (flow_table_nic_cap, hcattr,
1053                  ft_field_support_2_nic_receive.outer_ipv4_ihl);
1054         /* Query HCA offloads for Ethernet protocol. */
1055         hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
1056                         MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS |
1057                         MLX5_HCA_CAP_OPMOD_GET_CUR);
1058         if (!hcattr) {
1059                 attr->eth_net_offloads = 0;
1060                 return rc;
1061         }
1062         attr->wqe_vlan_insert = MLX5_GET(per_protocol_networking_offload_caps,
1063                                          hcattr, wqe_vlan_insert);
1064         attr->csum_cap = MLX5_GET(per_protocol_networking_offload_caps,
1065                                          hcattr, csum_cap);
1066         attr->vlan_cap = MLX5_GET(per_protocol_networking_offload_caps,
1067                                          hcattr, vlan_cap);
1068         attr->lro_cap = MLX5_GET(per_protocol_networking_offload_caps, hcattr,
1069                                  lro_cap);
1070         attr->max_lso_cap = MLX5_GET(per_protocol_networking_offload_caps,
1071                                  hcattr, max_lso_cap);
1072         attr->scatter_fcs = MLX5_GET(per_protocol_networking_offload_caps,
1073                                  hcattr, scatter_fcs);
1074         attr->tunnel_lro_gre = MLX5_GET(per_protocol_networking_offload_caps,
1075                                         hcattr, tunnel_lro_gre);
1076         attr->tunnel_lro_vxlan = MLX5_GET(per_protocol_networking_offload_caps,
1077                                           hcattr, tunnel_lro_vxlan);
1078         attr->swp = MLX5_GET(per_protocol_networking_offload_caps,
1079                                           hcattr, swp);
1080         attr->tunnel_stateless_gre =
1081                                 MLX5_GET(per_protocol_networking_offload_caps,
1082                                           hcattr, tunnel_stateless_gre);
1083         attr->tunnel_stateless_vxlan =
1084                                 MLX5_GET(per_protocol_networking_offload_caps,
1085                                           hcattr, tunnel_stateless_vxlan);
1086         attr->swp_csum = MLX5_GET(per_protocol_networking_offload_caps,
1087                                           hcattr, swp_csum);
1088         attr->swp_lso = MLX5_GET(per_protocol_networking_offload_caps,
1089                                           hcattr, swp_lso);
1090         attr->lro_max_msg_sz_mode = MLX5_GET
1091                                         (per_protocol_networking_offload_caps,
1092                                          hcattr, lro_max_msg_sz_mode);
1093         for (i = 0 ; i < MLX5_LRO_NUM_SUPP_PERIODS ; i++) {
1094                 attr->lro_timer_supported_periods[i] =
1095                         MLX5_GET(per_protocol_networking_offload_caps, hcattr,
1096                                  lro_timer_supported_periods[i]);
1097         }
1098         attr->lro_min_mss_size = MLX5_GET(per_protocol_networking_offload_caps,
1099                                           hcattr, lro_min_mss_size);
1100         attr->tunnel_stateless_geneve_rx =
1101                             MLX5_GET(per_protocol_networking_offload_caps,
1102                                      hcattr, tunnel_stateless_geneve_rx);
1103         attr->geneve_max_opt_len =
1104                     MLX5_GET(per_protocol_networking_offload_caps,
1105                              hcattr, max_geneve_opt_len);
1106         attr->wqe_inline_mode = MLX5_GET(per_protocol_networking_offload_caps,
1107                                          hcattr, wqe_inline_mode);
1108         attr->tunnel_stateless_gtp = MLX5_GET
1109                                         (per_protocol_networking_offload_caps,
1110                                          hcattr, tunnel_stateless_gtp);
1111         attr->rss_ind_tbl_cap = MLX5_GET
1112                                         (per_protocol_networking_offload_caps,
1113                                          hcattr, rss_ind_tbl_cap);
1114         /* Query HCA attribute for ROCE. */
1115         if (attr->roce) {
1116                 hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
1117                                 MLX5_GET_HCA_CAP_OP_MOD_ROCE |
1118                                 MLX5_HCA_CAP_OPMOD_GET_CUR);
1119                 if (!hcattr) {
1120                         DRV_LOG(DEBUG,
1121                                 "Failed to query devx HCA ROCE capabilities");
1122                         return rc;
1123                 }
1124                 attr->qp_ts_format = MLX5_GET(roce_caps, hcattr, qp_ts_format);
1125         }
1126         if (attr->eth_virt &&
1127             attr->wqe_inline_mode == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) {
1128                 rc = mlx5_devx_cmd_query_nic_vport_context(ctx, 0, attr);
1129                 if (rc) {
1130                         attr->eth_virt = 0;
1131                         goto error;
1132                 }
1133         }
1134         if (attr->eswitch_manager) {
1135                 hcattr = mlx5_devx_get_hca_cap(ctx, in, out, &rc,
1136                                 MLX5_SET_HCA_CAP_OP_MOD_ESW |
1137                                 MLX5_HCA_CAP_OPMOD_GET_CUR);
1138                 if (!hcattr)
1139                         return rc;
1140                 attr->esw_mgr_vport_id_valid =
1141                         MLX5_GET(esw_cap, hcattr,
1142                                  esw_manager_vport_number_valid);
1143                 attr->esw_mgr_vport_id =
1144                         MLX5_GET(esw_cap, hcattr, esw_manager_vport_number);
1145         }
1146         return 0;
1147 error:
1148         rc = (rc > 0) ? -rc : rc;
1149         return rc;
1150 }
1151
1152 /**
1153  * Query TIS transport domain from QP verbs object using DevX API.
1154  *
1155  * @param[in] qp
1156  *   Pointer to verbs QP returned by ibv_create_qp .
1157  * @param[in] tis_num
1158  *   TIS number of TIS to query.
1159  * @param[out] tis_td
1160  *   Pointer to TIS transport domain variable, to be set by the routine.
1161  *
1162  * @return
1163  *   0 on success, a negative value otherwise.
1164  */
1165 int
1166 mlx5_devx_cmd_qp_query_tis_td(void *qp, uint32_t tis_num,
1167                               uint32_t *tis_td)
1168 {
1169 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
1170         uint32_t in[MLX5_ST_SZ_DW(query_tis_in)] = {0};
1171         uint32_t out[MLX5_ST_SZ_DW(query_tis_out)] = {0};
1172         int rc;
1173         void *tis_ctx;
1174
1175         MLX5_SET(query_tis_in, in, opcode, MLX5_CMD_OP_QUERY_TIS);
1176         MLX5_SET(query_tis_in, in, tisn, tis_num);
1177         rc = mlx5_glue->devx_qp_query(qp, in, sizeof(in), out, sizeof(out));
1178         if (rc) {
1179                 DRV_LOG(ERR, "Failed to query QP using DevX");
1180                 return -rc;
1181         };
1182         tis_ctx = MLX5_ADDR_OF(query_tis_out, out, tis_context);
1183         *tis_td = MLX5_GET(tisc, tis_ctx, transport_domain);
1184         return 0;
1185 #else
1186         (void)qp;
1187         (void)tis_num;
1188         (void)tis_td;
1189         return -ENOTSUP;
1190 #endif
1191 }
1192
1193 /**
1194  * Fill WQ data for DevX API command.
1195  * Utility function for use when creating DevX objects containing a WQ.
1196  *
1197  * @param[in] wq_ctx
1198  *   Pointer to WQ context to fill with data.
1199  * @param [in] wq_attr
1200  *   Pointer to WQ attributes structure to fill in WQ context.
1201  */
1202 static void
1203 devx_cmd_fill_wq_data(void *wq_ctx, struct mlx5_devx_wq_attr *wq_attr)
1204 {
1205         MLX5_SET(wq, wq_ctx, wq_type, wq_attr->wq_type);
1206         MLX5_SET(wq, wq_ctx, wq_signature, wq_attr->wq_signature);
1207         MLX5_SET(wq, wq_ctx, end_padding_mode, wq_attr->end_padding_mode);
1208         MLX5_SET(wq, wq_ctx, cd_slave, wq_attr->cd_slave);
1209         MLX5_SET(wq, wq_ctx, hds_skip_first_sge, wq_attr->hds_skip_first_sge);
1210         MLX5_SET(wq, wq_ctx, log2_hds_buf_size, wq_attr->log2_hds_buf_size);
1211         MLX5_SET(wq, wq_ctx, page_offset, wq_attr->page_offset);
1212         MLX5_SET(wq, wq_ctx, lwm, wq_attr->lwm);
1213         MLX5_SET(wq, wq_ctx, pd, wq_attr->pd);
1214         MLX5_SET(wq, wq_ctx, uar_page, wq_attr->uar_page);
1215         MLX5_SET64(wq, wq_ctx, dbr_addr, wq_attr->dbr_addr);
1216         MLX5_SET(wq, wq_ctx, hw_counter, wq_attr->hw_counter);
1217         MLX5_SET(wq, wq_ctx, sw_counter, wq_attr->sw_counter);
1218         MLX5_SET(wq, wq_ctx, log_wq_stride, wq_attr->log_wq_stride);
1219         if (wq_attr->log_wq_pg_sz > MLX5_ADAPTER_PAGE_SHIFT)
1220                 MLX5_SET(wq, wq_ctx, log_wq_pg_sz,
1221                          wq_attr->log_wq_pg_sz - MLX5_ADAPTER_PAGE_SHIFT);
1222         MLX5_SET(wq, wq_ctx, log_wq_sz, wq_attr->log_wq_sz);
1223         MLX5_SET(wq, wq_ctx, dbr_umem_valid, wq_attr->dbr_umem_valid);
1224         MLX5_SET(wq, wq_ctx, wq_umem_valid, wq_attr->wq_umem_valid);
1225         MLX5_SET(wq, wq_ctx, log_hairpin_num_packets,
1226                  wq_attr->log_hairpin_num_packets);
1227         MLX5_SET(wq, wq_ctx, log_hairpin_data_sz, wq_attr->log_hairpin_data_sz);
1228         MLX5_SET(wq, wq_ctx, single_wqe_log_num_of_strides,
1229                  wq_attr->single_wqe_log_num_of_strides);
1230         MLX5_SET(wq, wq_ctx, two_byte_shift_en, wq_attr->two_byte_shift_en);
1231         MLX5_SET(wq, wq_ctx, single_stride_log_num_of_bytes,
1232                  wq_attr->single_stride_log_num_of_bytes);
1233         MLX5_SET(wq, wq_ctx, dbr_umem_id, wq_attr->dbr_umem_id);
1234         MLX5_SET(wq, wq_ctx, wq_umem_id, wq_attr->wq_umem_id);
1235         MLX5_SET64(wq, wq_ctx, wq_umem_offset, wq_attr->wq_umem_offset);
1236 }
1237
1238 /**
1239  * Create RQ using DevX API.
1240  *
1241  * @param[in] ctx
1242  *   Context returned from mlx5 open_device() glue function.
1243  * @param [in] rq_attr
1244  *   Pointer to create RQ attributes structure.
1245  * @param [in] socket
1246  *   CPU socket ID for allocations.
1247  *
1248  * @return
1249  *   The DevX object created, NULL otherwise and rte_errno is set.
1250  */
1251 struct mlx5_devx_obj *
1252 mlx5_devx_cmd_create_rq(void *ctx,
1253                         struct mlx5_devx_create_rq_attr *rq_attr,
1254                         int socket)
1255 {
1256         uint32_t in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
1257         uint32_t out[MLX5_ST_SZ_DW(create_rq_out)] = {0};
1258         void *rq_ctx, *wq_ctx;
1259         struct mlx5_devx_wq_attr *wq_attr;
1260         struct mlx5_devx_obj *rq = NULL;
1261
1262         rq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rq), 0, socket);
1263         if (!rq) {
1264                 DRV_LOG(ERR, "Failed to allocate RQ data");
1265                 rte_errno = ENOMEM;
1266                 return NULL;
1267         }
1268         MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
1269         rq_ctx = MLX5_ADDR_OF(create_rq_in, in, ctx);
1270         MLX5_SET(rqc, rq_ctx, rlky, rq_attr->rlky);
1271         MLX5_SET(rqc, rq_ctx, delay_drop_en, rq_attr->delay_drop_en);
1272         MLX5_SET(rqc, rq_ctx, scatter_fcs, rq_attr->scatter_fcs);
1273         MLX5_SET(rqc, rq_ctx, vsd, rq_attr->vsd);
1274         MLX5_SET(rqc, rq_ctx, mem_rq_type, rq_attr->mem_rq_type);
1275         MLX5_SET(rqc, rq_ctx, state, rq_attr->state);
1276         MLX5_SET(rqc, rq_ctx, flush_in_error_en, rq_attr->flush_in_error_en);
1277         MLX5_SET(rqc, rq_ctx, hairpin, rq_attr->hairpin);
1278         MLX5_SET(rqc, rq_ctx, user_index, rq_attr->user_index);
1279         MLX5_SET(rqc, rq_ctx, cqn, rq_attr->cqn);
1280         MLX5_SET(rqc, rq_ctx, counter_set_id, rq_attr->counter_set_id);
1281         MLX5_SET(rqc, rq_ctx, rmpn, rq_attr->rmpn);
1282         MLX5_SET(sqc, rq_ctx, ts_format, rq_attr->ts_format);
1283         wq_ctx = MLX5_ADDR_OF(rqc, rq_ctx, wq);
1284         wq_attr = &rq_attr->wq_attr;
1285         devx_cmd_fill_wq_data(wq_ctx, wq_attr);
1286         rq->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
1287                                                   out, sizeof(out));
1288         if (!rq->obj) {
1289                 DRV_LOG(ERR, "Failed to create RQ using DevX");
1290                 rte_errno = errno;
1291                 mlx5_free(rq);
1292                 return NULL;
1293         }
1294         rq->id = MLX5_GET(create_rq_out, out, rqn);
1295         return rq;
1296 }
1297
1298 /**
1299  * Modify RQ using DevX API.
1300  *
1301  * @param[in] rq
1302  *   Pointer to RQ object structure.
1303  * @param [in] rq_attr
1304  *   Pointer to modify RQ attributes structure.
1305  *
1306  * @return
1307  *   0 on success, a negative errno value otherwise and rte_errno is set.
1308  */
1309 int
1310 mlx5_devx_cmd_modify_rq(struct mlx5_devx_obj *rq,
1311                         struct mlx5_devx_modify_rq_attr *rq_attr)
1312 {
1313         uint32_t in[MLX5_ST_SZ_DW(modify_rq_in)] = {0};
1314         uint32_t out[MLX5_ST_SZ_DW(modify_rq_out)] = {0};
1315         void *rq_ctx, *wq_ctx;
1316         int ret;
1317
1318         MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
1319         MLX5_SET(modify_rq_in, in, rq_state, rq_attr->rq_state);
1320         MLX5_SET(modify_rq_in, in, rqn, rq->id);
1321         MLX5_SET64(modify_rq_in, in, modify_bitmask, rq_attr->modify_bitmask);
1322         rq_ctx = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1323         MLX5_SET(rqc, rq_ctx, state, rq_attr->state);
1324         if (rq_attr->modify_bitmask &
1325                         MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS)
1326                 MLX5_SET(rqc, rq_ctx, scatter_fcs, rq_attr->scatter_fcs);
1327         if (rq_attr->modify_bitmask & MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD)
1328                 MLX5_SET(rqc, rq_ctx, vsd, rq_attr->vsd);
1329         if (rq_attr->modify_bitmask &
1330                         MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID)
1331                 MLX5_SET(rqc, rq_ctx, counter_set_id, rq_attr->counter_set_id);
1332         MLX5_SET(rqc, rq_ctx, hairpin_peer_sq, rq_attr->hairpin_peer_sq);
1333         MLX5_SET(rqc, rq_ctx, hairpin_peer_vhca, rq_attr->hairpin_peer_vhca);
1334         if (rq_attr->modify_bitmask & MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_WQ_LWM) {
1335                 wq_ctx = MLX5_ADDR_OF(rqc, rq_ctx, wq);
1336                 MLX5_SET(wq, wq_ctx, lwm, rq_attr->lwm);
1337         }
1338         ret = mlx5_glue->devx_obj_modify(rq->obj, in, sizeof(in),
1339                                          out, sizeof(out));
1340         if (ret) {
1341                 DRV_LOG(ERR, "Failed to modify RQ using DevX");
1342                 rte_errno = errno;
1343                 return -errno;
1344         }
1345         return ret;
1346 }
1347
1348 /**
1349  * Create RMP using DevX API.
1350  *
1351  * @param[in] ctx
1352  *   Context returned from mlx5 open_device() glue function.
1353  * @param [in] rmp_attr
1354  *   Pointer to create RMP attributes structure.
1355  * @param [in] socket
1356  *   CPU socket ID for allocations.
1357  *
1358  * @return
1359  *   The DevX object created, NULL otherwise and rte_errno is set.
1360  */
1361 struct mlx5_devx_obj *
1362 mlx5_devx_cmd_create_rmp(void *ctx,
1363                          struct mlx5_devx_create_rmp_attr *rmp_attr,
1364                          int socket)
1365 {
1366         uint32_t in[MLX5_ST_SZ_DW(create_rmp_in)] = {0};
1367         uint32_t out[MLX5_ST_SZ_DW(create_rmp_out)] = {0};
1368         void *rmp_ctx, *wq_ctx;
1369         struct mlx5_devx_wq_attr *wq_attr;
1370         struct mlx5_devx_obj *rmp = NULL;
1371
1372         rmp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rmp), 0, socket);
1373         if (!rmp) {
1374                 DRV_LOG(ERR, "Failed to allocate RMP data");
1375                 rte_errno = ENOMEM;
1376                 return NULL;
1377         }
1378         MLX5_SET(create_rmp_in, in, opcode, MLX5_CMD_OP_CREATE_RMP);
1379         rmp_ctx = MLX5_ADDR_OF(create_rmp_in, in, ctx);
1380         MLX5_SET(rmpc, rmp_ctx, state, rmp_attr->state);
1381         MLX5_SET(rmpc, rmp_ctx, basic_cyclic_rcv_wqe,
1382                  rmp_attr->basic_cyclic_rcv_wqe);
1383         wq_ctx = MLX5_ADDR_OF(rmpc, rmp_ctx, wq);
1384         wq_attr = &rmp_attr->wq_attr;
1385         devx_cmd_fill_wq_data(wq_ctx, wq_attr);
1386         rmp->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
1387                                               sizeof(out));
1388         if (!rmp->obj) {
1389                 DRV_LOG(ERR, "Failed to create RMP using DevX");
1390                 rte_errno = errno;
1391                 mlx5_free(rmp);
1392                 return NULL;
1393         }
1394         rmp->id = MLX5_GET(create_rmp_out, out, rmpn);
1395         return rmp;
1396 }
1397
1398 /*
1399  * Create TIR using DevX API.
1400  *
1401  * @param[in] ctx
1402  *  Context returned from mlx5 open_device() glue function.
1403  * @param [in] tir_attr
1404  *   Pointer to TIR attributes structure.
1405  *
1406  * @return
1407  *   The DevX object created, NULL otherwise and rte_errno is set.
1408  */
1409 struct mlx5_devx_obj *
1410 mlx5_devx_cmd_create_tir(void *ctx,
1411                          struct mlx5_devx_tir_attr *tir_attr)
1412 {
1413         uint32_t in[MLX5_ST_SZ_DW(create_tir_in)] = {0};
1414         uint32_t out[MLX5_ST_SZ_DW(create_tir_out)] = {0};
1415         void *tir_ctx, *outer, *inner, *rss_key;
1416         struct mlx5_devx_obj *tir = NULL;
1417
1418         tir = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tir), 0, SOCKET_ID_ANY);
1419         if (!tir) {
1420                 DRV_LOG(ERR, "Failed to allocate TIR data");
1421                 rte_errno = ENOMEM;
1422                 return NULL;
1423         }
1424         MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1425         tir_ctx = MLX5_ADDR_OF(create_tir_in, in, ctx);
1426         MLX5_SET(tirc, tir_ctx, disp_type, tir_attr->disp_type);
1427         MLX5_SET(tirc, tir_ctx, lro_timeout_period_usecs,
1428                  tir_attr->lro_timeout_period_usecs);
1429         MLX5_SET(tirc, tir_ctx, lro_enable_mask, tir_attr->lro_enable_mask);
1430         MLX5_SET(tirc, tir_ctx, lro_max_msg_sz, tir_attr->lro_max_msg_sz);
1431         MLX5_SET(tirc, tir_ctx, inline_rqn, tir_attr->inline_rqn);
1432         MLX5_SET(tirc, tir_ctx, rx_hash_symmetric, tir_attr->rx_hash_symmetric);
1433         MLX5_SET(tirc, tir_ctx, tunneled_offload_en,
1434                  tir_attr->tunneled_offload_en);
1435         MLX5_SET(tirc, tir_ctx, indirect_table, tir_attr->indirect_table);
1436         MLX5_SET(tirc, tir_ctx, rx_hash_fn, tir_attr->rx_hash_fn);
1437         MLX5_SET(tirc, tir_ctx, self_lb_block, tir_attr->self_lb_block);
1438         MLX5_SET(tirc, tir_ctx, transport_domain, tir_attr->transport_domain);
1439         rss_key = MLX5_ADDR_OF(tirc, tir_ctx, rx_hash_toeplitz_key);
1440         memcpy(rss_key, tir_attr->rx_hash_toeplitz_key, MLX5_RSS_HASH_KEY_LEN);
1441         outer = MLX5_ADDR_OF(tirc, tir_ctx, rx_hash_field_selector_outer);
1442         MLX5_SET(rx_hash_field_select, outer, l3_prot_type,
1443                  tir_attr->rx_hash_field_selector_outer.l3_prot_type);
1444         MLX5_SET(rx_hash_field_select, outer, l4_prot_type,
1445                  tir_attr->rx_hash_field_selector_outer.l4_prot_type);
1446         MLX5_SET(rx_hash_field_select, outer, selected_fields,
1447                  tir_attr->rx_hash_field_selector_outer.selected_fields);
1448         inner = MLX5_ADDR_OF(tirc, tir_ctx, rx_hash_field_selector_inner);
1449         MLX5_SET(rx_hash_field_select, inner, l3_prot_type,
1450                  tir_attr->rx_hash_field_selector_inner.l3_prot_type);
1451         MLX5_SET(rx_hash_field_select, inner, l4_prot_type,
1452                  tir_attr->rx_hash_field_selector_inner.l4_prot_type);
1453         MLX5_SET(rx_hash_field_select, inner, selected_fields,
1454                  tir_attr->rx_hash_field_selector_inner.selected_fields);
1455         tir->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
1456                                                    out, sizeof(out));
1457         if (!tir->obj) {
1458                 DRV_LOG(ERR, "Failed to create TIR using DevX");
1459                 rte_errno = errno;
1460                 mlx5_free(tir);
1461                 return NULL;
1462         }
1463         tir->id = MLX5_GET(create_tir_out, out, tirn);
1464         return tir;
1465 }
1466
1467 /**
1468  * Modify TIR using DevX API.
1469  *
1470  * @param[in] tir
1471  *   Pointer to TIR DevX object structure.
1472  * @param [in] modify_tir_attr
1473  *   Pointer to TIR modification attributes structure.
1474  *
1475  * @return
1476  *   0 on success, a negative errno value otherwise and rte_errno is set.
1477  */
1478 int
1479 mlx5_devx_cmd_modify_tir(struct mlx5_devx_obj *tir,
1480                          struct mlx5_devx_modify_tir_attr *modify_tir_attr)
1481 {
1482         struct mlx5_devx_tir_attr *tir_attr = &modify_tir_attr->tir;
1483         uint32_t in[MLX5_ST_SZ_DW(modify_tir_in)] = {0};
1484         uint32_t out[MLX5_ST_SZ_DW(modify_tir_out)] = {0};
1485         void *tir_ctx;
1486         int ret;
1487
1488         MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
1489         MLX5_SET(modify_tir_in, in, tirn, modify_tir_attr->tirn);
1490         MLX5_SET64(modify_tir_in, in, modify_bitmask,
1491                    modify_tir_attr->modify_bitmask);
1492         tir_ctx = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1493         if (modify_tir_attr->modify_bitmask &
1494                         MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_LRO) {
1495                 MLX5_SET(tirc, tir_ctx, lro_timeout_period_usecs,
1496                          tir_attr->lro_timeout_period_usecs);
1497                 MLX5_SET(tirc, tir_ctx, lro_enable_mask,
1498                          tir_attr->lro_enable_mask);
1499                 MLX5_SET(tirc, tir_ctx, lro_max_msg_sz,
1500                          tir_attr->lro_max_msg_sz);
1501         }
1502         if (modify_tir_attr->modify_bitmask &
1503                         MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_INDIRECT_TABLE)
1504                 MLX5_SET(tirc, tir_ctx, indirect_table,
1505                          tir_attr->indirect_table);
1506         if (modify_tir_attr->modify_bitmask &
1507                         MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_HASH) {
1508                 int i;
1509                 void *outer, *inner;
1510
1511                 MLX5_SET(tirc, tir_ctx, rx_hash_symmetric,
1512                          tir_attr->rx_hash_symmetric);
1513                 MLX5_SET(tirc, tir_ctx, rx_hash_fn, tir_attr->rx_hash_fn);
1514                 for (i = 0; i < 10; i++) {
1515                         MLX5_SET(tirc, tir_ctx, rx_hash_toeplitz_key[i],
1516                                  tir_attr->rx_hash_toeplitz_key[i]);
1517                 }
1518                 outer = MLX5_ADDR_OF(tirc, tir_ctx,
1519                                      rx_hash_field_selector_outer);
1520                 MLX5_SET(rx_hash_field_select, outer, l3_prot_type,
1521                          tir_attr->rx_hash_field_selector_outer.l3_prot_type);
1522                 MLX5_SET(rx_hash_field_select, outer, l4_prot_type,
1523                          tir_attr->rx_hash_field_selector_outer.l4_prot_type);
1524                 MLX5_SET
1525                 (rx_hash_field_select, outer, selected_fields,
1526                  tir_attr->rx_hash_field_selector_outer.selected_fields);
1527                 inner = MLX5_ADDR_OF(tirc, tir_ctx,
1528                                      rx_hash_field_selector_inner);
1529                 MLX5_SET(rx_hash_field_select, inner, l3_prot_type,
1530                          tir_attr->rx_hash_field_selector_inner.l3_prot_type);
1531                 MLX5_SET(rx_hash_field_select, inner, l4_prot_type,
1532                          tir_attr->rx_hash_field_selector_inner.l4_prot_type);
1533                 MLX5_SET
1534                 (rx_hash_field_select, inner, selected_fields,
1535                  tir_attr->rx_hash_field_selector_inner.selected_fields);
1536         }
1537         if (modify_tir_attr->modify_bitmask &
1538             MLX5_MODIFY_TIR_IN_MODIFY_BITMASK_SELF_LB_EN) {
1539                 MLX5_SET(tirc, tir_ctx, self_lb_block, tir_attr->self_lb_block);
1540         }
1541         ret = mlx5_glue->devx_obj_modify(tir->obj, in, sizeof(in),
1542                                          out, sizeof(out));
1543         if (ret) {
1544                 DRV_LOG(ERR, "Failed to modify TIR using DevX");
1545                 rte_errno = errno;
1546                 return -errno;
1547         }
1548         return ret;
1549 }
1550
1551 /**
1552  * Create RQT using DevX API.
1553  *
1554  * @param[in] ctx
1555  *   Context returned from mlx5 open_device() glue function.
1556  * @param [in] rqt_attr
1557  *   Pointer to RQT attributes structure.
1558  *
1559  * @return
1560  *   The DevX object created, NULL otherwise and rte_errno is set.
1561  */
1562 struct mlx5_devx_obj *
1563 mlx5_devx_cmd_create_rqt(void *ctx,
1564                          struct mlx5_devx_rqt_attr *rqt_attr)
1565 {
1566         uint32_t *in = NULL;
1567         uint32_t inlen = MLX5_ST_SZ_BYTES(create_rqt_in) +
1568                          rqt_attr->rqt_actual_size * sizeof(uint32_t);
1569         uint32_t out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
1570         void *rqt_ctx;
1571         struct mlx5_devx_obj *rqt = NULL;
1572         int i;
1573
1574         in = mlx5_malloc(MLX5_MEM_ZERO, inlen, 0, SOCKET_ID_ANY);
1575         if (!in) {
1576                 DRV_LOG(ERR, "Failed to allocate RQT IN data");
1577                 rte_errno = ENOMEM;
1578                 return NULL;
1579         }
1580         rqt = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*rqt), 0, SOCKET_ID_ANY);
1581         if (!rqt) {
1582                 DRV_LOG(ERR, "Failed to allocate RQT data");
1583                 rte_errno = ENOMEM;
1584                 mlx5_free(in);
1585                 return NULL;
1586         }
1587         MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
1588         rqt_ctx = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
1589         MLX5_SET(rqtc, rqt_ctx, list_q_type, rqt_attr->rq_type);
1590         MLX5_SET(rqtc, rqt_ctx, rqt_max_size, rqt_attr->rqt_max_size);
1591         MLX5_SET(rqtc, rqt_ctx, rqt_actual_size, rqt_attr->rqt_actual_size);
1592         for (i = 0; i < rqt_attr->rqt_actual_size; i++)
1593                 MLX5_SET(rqtc, rqt_ctx, rq_num[i], rqt_attr->rq_list[i]);
1594         rqt->obj = mlx5_glue->devx_obj_create(ctx, in, inlen, out, sizeof(out));
1595         mlx5_free(in);
1596         if (!rqt->obj) {
1597                 DRV_LOG(ERR, "Failed to create RQT using DevX");
1598                 rte_errno = errno;
1599                 mlx5_free(rqt);
1600                 return NULL;
1601         }
1602         rqt->id = MLX5_GET(create_rqt_out, out, rqtn);
1603         return rqt;
1604 }
1605
1606 /**
1607  * Modify RQT using DevX API.
1608  *
1609  * @param[in] rqt
1610  *   Pointer to RQT DevX object structure.
1611  * @param [in] rqt_attr
1612  *   Pointer to RQT attributes structure.
1613  *
1614  * @return
1615  *   0 on success, a negative errno value otherwise and rte_errno is set.
1616  */
1617 int
1618 mlx5_devx_cmd_modify_rqt(struct mlx5_devx_obj *rqt,
1619                          struct mlx5_devx_rqt_attr *rqt_attr)
1620 {
1621         uint32_t inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) +
1622                          rqt_attr->rqt_actual_size * sizeof(uint32_t);
1623         uint32_t out[MLX5_ST_SZ_DW(modify_rqt_out)] = {0};
1624         uint32_t *in = mlx5_malloc(MLX5_MEM_ZERO, inlen, 0, SOCKET_ID_ANY);
1625         void *rqt_ctx;
1626         int i;
1627         int ret;
1628
1629         if (!in) {
1630                 DRV_LOG(ERR, "Failed to allocate RQT modify IN data.");
1631                 rte_errno = ENOMEM;
1632                 return -ENOMEM;
1633         }
1634         MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
1635         MLX5_SET(modify_rqt_in, in, rqtn, rqt->id);
1636         MLX5_SET64(modify_rqt_in, in, modify_bitmask, 0x1);
1637         rqt_ctx = MLX5_ADDR_OF(modify_rqt_in, in, rqt_context);
1638         MLX5_SET(rqtc, rqt_ctx, list_q_type, rqt_attr->rq_type);
1639         MLX5_SET(rqtc, rqt_ctx, rqt_max_size, rqt_attr->rqt_max_size);
1640         MLX5_SET(rqtc, rqt_ctx, rqt_actual_size, rqt_attr->rqt_actual_size);
1641         for (i = 0; i < rqt_attr->rqt_actual_size; i++)
1642                 MLX5_SET(rqtc, rqt_ctx, rq_num[i], rqt_attr->rq_list[i]);
1643         ret = mlx5_glue->devx_obj_modify(rqt->obj, in, inlen, out, sizeof(out));
1644         mlx5_free(in);
1645         if (ret) {
1646                 DRV_LOG(ERR, "Failed to modify RQT using DevX.");
1647                 rte_errno = errno;
1648                 return -rte_errno;
1649         }
1650         return ret;
1651 }
1652
1653 /**
1654  * Create SQ using DevX API.
1655  *
1656  * @param[in] ctx
1657  *   Context returned from mlx5 open_device() glue function.
1658  * @param [in] sq_attr
1659  *   Pointer to SQ attributes structure.
1660  * @param [in] socket
1661  *   CPU socket ID for allocations.
1662  *
1663  * @return
1664  *   The DevX object created, NULL otherwise and rte_errno is set.
1665  **/
1666 struct mlx5_devx_obj *
1667 mlx5_devx_cmd_create_sq(void *ctx,
1668                         struct mlx5_devx_create_sq_attr *sq_attr)
1669 {
1670         uint32_t in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
1671         uint32_t out[MLX5_ST_SZ_DW(create_sq_out)] = {0};
1672         void *sq_ctx;
1673         void *wq_ctx;
1674         struct mlx5_devx_wq_attr *wq_attr;
1675         struct mlx5_devx_obj *sq = NULL;
1676
1677         sq = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*sq), 0, SOCKET_ID_ANY);
1678         if (!sq) {
1679                 DRV_LOG(ERR, "Failed to allocate SQ data");
1680                 rte_errno = ENOMEM;
1681                 return NULL;
1682         }
1683         MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
1684         sq_ctx = MLX5_ADDR_OF(create_sq_in, in, ctx);
1685         MLX5_SET(sqc, sq_ctx, rlky, sq_attr->rlky);
1686         MLX5_SET(sqc, sq_ctx, cd_master, sq_attr->cd_master);
1687         MLX5_SET(sqc, sq_ctx, fre, sq_attr->fre);
1688         MLX5_SET(sqc, sq_ctx, flush_in_error_en, sq_attr->flush_in_error_en);
1689         MLX5_SET(sqc, sq_ctx, allow_multi_pkt_send_wqe,
1690                  sq_attr->allow_multi_pkt_send_wqe);
1691         MLX5_SET(sqc, sq_ctx, min_wqe_inline_mode,
1692                  sq_attr->min_wqe_inline_mode);
1693         MLX5_SET(sqc, sq_ctx, state, sq_attr->state);
1694         MLX5_SET(sqc, sq_ctx, reg_umr, sq_attr->reg_umr);
1695         MLX5_SET(sqc, sq_ctx, allow_swp, sq_attr->allow_swp);
1696         MLX5_SET(sqc, sq_ctx, hairpin, sq_attr->hairpin);
1697         MLX5_SET(sqc, sq_ctx, non_wire, sq_attr->non_wire);
1698         MLX5_SET(sqc, sq_ctx, static_sq_wq, sq_attr->static_sq_wq);
1699         MLX5_SET(sqc, sq_ctx, user_index, sq_attr->user_index);
1700         MLX5_SET(sqc, sq_ctx, cqn, sq_attr->cqn);
1701         MLX5_SET(sqc, sq_ctx, packet_pacing_rate_limit_index,
1702                  sq_attr->packet_pacing_rate_limit_index);
1703         MLX5_SET(sqc, sq_ctx, tis_lst_sz, sq_attr->tis_lst_sz);
1704         MLX5_SET(sqc, sq_ctx, tis_num_0, sq_attr->tis_num);
1705         MLX5_SET(sqc, sq_ctx, ts_format, sq_attr->ts_format);
1706         wq_ctx = MLX5_ADDR_OF(sqc, sq_ctx, wq);
1707         wq_attr = &sq_attr->wq_attr;
1708         devx_cmd_fill_wq_data(wq_ctx, wq_attr);
1709         sq->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
1710                                              out, sizeof(out));
1711         if (!sq->obj) {
1712                 DRV_LOG(ERR, "Failed to create SQ using DevX");
1713                 rte_errno = errno;
1714                 mlx5_free(sq);
1715                 return NULL;
1716         }
1717         sq->id = MLX5_GET(create_sq_out, out, sqn);
1718         return sq;
1719 }
1720
1721 /**
1722  * Modify SQ using DevX API.
1723  *
1724  * @param[in] sq
1725  *   Pointer to SQ object structure.
1726  * @param [in] sq_attr
1727  *   Pointer to SQ attributes structure.
1728  *
1729  * @return
1730  *   0 on success, a negative errno value otherwise and rte_errno is set.
1731  */
1732 int
1733 mlx5_devx_cmd_modify_sq(struct mlx5_devx_obj *sq,
1734                         struct mlx5_devx_modify_sq_attr *sq_attr)
1735 {
1736         uint32_t in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
1737         uint32_t out[MLX5_ST_SZ_DW(modify_sq_out)] = {0};
1738         void *sq_ctx;
1739         int ret;
1740
1741         MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
1742         MLX5_SET(modify_sq_in, in, sq_state, sq_attr->sq_state);
1743         MLX5_SET(modify_sq_in, in, sqn, sq->id);
1744         sq_ctx = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1745         MLX5_SET(sqc, sq_ctx, state, sq_attr->state);
1746         MLX5_SET(sqc, sq_ctx, hairpin_peer_rq, sq_attr->hairpin_peer_rq);
1747         MLX5_SET(sqc, sq_ctx, hairpin_peer_vhca, sq_attr->hairpin_peer_vhca);
1748         ret = mlx5_glue->devx_obj_modify(sq->obj, in, sizeof(in),
1749                                          out, sizeof(out));
1750         if (ret) {
1751                 DRV_LOG(ERR, "Failed to modify SQ using DevX");
1752                 rte_errno = errno;
1753                 return -rte_errno;
1754         }
1755         return ret;
1756 }
1757
1758 /**
1759  * Create TIS using DevX API.
1760  *
1761  * @param[in] ctx
1762  *   Context returned from mlx5 open_device() glue function.
1763  * @param [in] tis_attr
1764  *   Pointer to TIS attributes structure.
1765  *
1766  * @return
1767  *   The DevX object created, NULL otherwise and rte_errno is set.
1768  */
1769 struct mlx5_devx_obj *
1770 mlx5_devx_cmd_create_tis(void *ctx,
1771                          struct mlx5_devx_tis_attr *tis_attr)
1772 {
1773         uint32_t in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
1774         uint32_t out[MLX5_ST_SZ_DW(create_tis_out)] = {0};
1775         struct mlx5_devx_obj *tis = NULL;
1776         void *tis_ctx;
1777
1778         tis = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tis), 0, SOCKET_ID_ANY);
1779         if (!tis) {
1780                 DRV_LOG(ERR, "Failed to allocate TIS object");
1781                 rte_errno = ENOMEM;
1782                 return NULL;
1783         }
1784         MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
1785         tis_ctx = MLX5_ADDR_OF(create_tis_in, in, ctx);
1786         MLX5_SET(tisc, tis_ctx, strict_lag_tx_port_affinity,
1787                  tis_attr->strict_lag_tx_port_affinity);
1788         MLX5_SET(tisc, tis_ctx, lag_tx_port_affinity,
1789                  tis_attr->lag_tx_port_affinity);
1790         MLX5_SET(tisc, tis_ctx, prio, tis_attr->prio);
1791         MLX5_SET(tisc, tis_ctx, transport_domain,
1792                  tis_attr->transport_domain);
1793         tis->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
1794                                               out, sizeof(out));
1795         if (!tis->obj) {
1796                 DRV_LOG(ERR, "Failed to create TIS using DevX");
1797                 rte_errno = errno;
1798                 mlx5_free(tis);
1799                 return NULL;
1800         }
1801         tis->id = MLX5_GET(create_tis_out, out, tisn);
1802         return tis;
1803 }
1804
1805 /**
1806  * Create transport domain using DevX API.
1807  *
1808  * @param[in] ctx
1809  *   Context returned from mlx5 open_device() glue function.
1810  * @return
1811  *   The DevX object created, NULL otherwise and rte_errno is set.
1812  */
1813 struct mlx5_devx_obj *
1814 mlx5_devx_cmd_create_td(void *ctx)
1815 {
1816         uint32_t in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {0};
1817         uint32_t out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};
1818         struct mlx5_devx_obj *td = NULL;
1819
1820         td = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*td), 0, SOCKET_ID_ANY);
1821         if (!td) {
1822                 DRV_LOG(ERR, "Failed to allocate TD object");
1823                 rte_errno = ENOMEM;
1824                 return NULL;
1825         }
1826         MLX5_SET(alloc_transport_domain_in, in, opcode,
1827                  MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
1828         td->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
1829                                              out, sizeof(out));
1830         if (!td->obj) {
1831                 DRV_LOG(ERR, "Failed to create TIS using DevX");
1832                 rte_errno = errno;
1833                 mlx5_free(td);
1834                 return NULL;
1835         }
1836         td->id = MLX5_GET(alloc_transport_domain_out, out,
1837                            transport_domain);
1838         return td;
1839 }
1840
1841 /**
1842  * Dump all flows to file.
1843  *
1844  * @param[in] fdb_domain
1845  *   FDB domain.
1846  * @param[in] rx_domain
1847  *   RX domain.
1848  * @param[in] tx_domain
1849  *   TX domain.
1850  * @param[out] file
1851  *   Pointer to file stream.
1852  *
1853  * @return
1854  *   0 on success, a negative value otherwise.
1855  */
1856 int
1857 mlx5_devx_cmd_flow_dump(void *fdb_domain __rte_unused,
1858                         void *rx_domain __rte_unused,
1859                         void *tx_domain __rte_unused, FILE *file __rte_unused)
1860 {
1861         int ret = 0;
1862
1863 #ifdef HAVE_MLX5_DR_FLOW_DUMP
1864         if (fdb_domain) {
1865                 ret = mlx5_glue->dr_dump_domain(file, fdb_domain);
1866                 if (ret)
1867                         return ret;
1868         }
1869         MLX5_ASSERT(rx_domain);
1870         ret = mlx5_glue->dr_dump_domain(file, rx_domain);
1871         if (ret)
1872                 return ret;
1873         MLX5_ASSERT(tx_domain);
1874         ret = mlx5_glue->dr_dump_domain(file, tx_domain);
1875 #else
1876         ret = ENOTSUP;
1877 #endif
1878         return -ret;
1879 }
1880
1881 int
1882 mlx5_devx_cmd_flow_single_dump(void *rule_info __rte_unused,
1883                         FILE *file __rte_unused)
1884 {
1885         int ret = 0;
1886 #ifdef HAVE_MLX5_DR_FLOW_DUMP_RULE
1887         if (rule_info)
1888                 ret = mlx5_glue->dr_dump_rule(file, rule_info);
1889 #else
1890         ret = ENOTSUP;
1891 #endif
1892         return -ret;
1893 }
1894
1895 /*
1896  * Create CQ using DevX API.
1897  *
1898  * @param[in] ctx
1899  *   Context returned from mlx5 open_device() glue function.
1900  * @param [in] attr
1901  *   Pointer to CQ attributes structure.
1902  *
1903  * @return
1904  *   The DevX object created, NULL otherwise and rte_errno is set.
1905  */
1906 struct mlx5_devx_obj *
1907 mlx5_devx_cmd_create_cq(void *ctx, struct mlx5_devx_cq_attr *attr)
1908 {
1909         uint32_t in[MLX5_ST_SZ_DW(create_cq_in)] = {0};
1910         uint32_t out[MLX5_ST_SZ_DW(create_cq_out)] = {0};
1911         struct mlx5_devx_obj *cq_obj = mlx5_malloc(MLX5_MEM_ZERO,
1912                                                    sizeof(*cq_obj),
1913                                                    0, SOCKET_ID_ANY);
1914         void *cqctx = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1915
1916         if (!cq_obj) {
1917                 DRV_LOG(ERR, "Failed to allocate CQ object memory.");
1918                 rte_errno = ENOMEM;
1919                 return NULL;
1920         }
1921         MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ);
1922         if (attr->db_umem_valid) {
1923                 MLX5_SET(cqc, cqctx, dbr_umem_valid, attr->db_umem_valid);
1924                 MLX5_SET(cqc, cqctx, dbr_umem_id, attr->db_umem_id);
1925                 MLX5_SET64(cqc, cqctx, dbr_addr, attr->db_umem_offset);
1926         } else {
1927                 MLX5_SET64(cqc, cqctx, dbr_addr, attr->db_addr);
1928         }
1929         MLX5_SET(cqc, cqctx, cqe_sz, (RTE_CACHE_LINE_SIZE == 128) ?
1930                                      MLX5_CQE_SIZE_128B : MLX5_CQE_SIZE_64B);
1931         MLX5_SET(cqc, cqctx, cc, attr->use_first_only);
1932         MLX5_SET(cqc, cqctx, oi, attr->overrun_ignore);
1933         MLX5_SET(cqc, cqctx, log_cq_size, attr->log_cq_size);
1934         if (attr->log_page_size > MLX5_ADAPTER_PAGE_SHIFT)
1935                 MLX5_SET(cqc, cqctx, log_page_size,
1936                          attr->log_page_size - MLX5_ADAPTER_PAGE_SHIFT);
1937         MLX5_SET(cqc, cqctx, c_eqn, attr->eqn);
1938         MLX5_SET(cqc, cqctx, uar_page, attr->uar_page_id);
1939         MLX5_SET(cqc, cqctx, cqe_comp_en, !!attr->cqe_comp_en);
1940         MLX5_SET(cqc, cqctx, mini_cqe_res_format, attr->mini_cqe_res_format);
1941         MLX5_SET(cqc, cqctx, mini_cqe_res_format_ext,
1942                  attr->mini_cqe_res_format_ext);
1943         if (attr->q_umem_valid) {
1944                 MLX5_SET(create_cq_in, in, cq_umem_valid, attr->q_umem_valid);
1945                 MLX5_SET(create_cq_in, in, cq_umem_id, attr->q_umem_id);
1946                 MLX5_SET64(create_cq_in, in, cq_umem_offset,
1947                            attr->q_umem_offset);
1948         }
1949         cq_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
1950                                                  sizeof(out));
1951         if (!cq_obj->obj) {
1952                 rte_errno = errno;
1953                 DRV_LOG(ERR, "Failed to create CQ using DevX errno=%d.", errno);
1954                 mlx5_free(cq_obj);
1955                 return NULL;
1956         }
1957         cq_obj->id = MLX5_GET(create_cq_out, out, cqn);
1958         return cq_obj;
1959 }
1960
1961 /**
1962  * Create VIRTQ using DevX API.
1963  *
1964  * @param[in] ctx
1965  *   Context returned from mlx5 open_device() glue function.
1966  * @param [in] attr
1967  *   Pointer to VIRTQ attributes structure.
1968  *
1969  * @return
1970  *   The DevX object created, NULL otherwise and rte_errno is set.
1971  */
1972 struct mlx5_devx_obj *
1973 mlx5_devx_cmd_create_virtq(void *ctx,
1974                            struct mlx5_devx_virtq_attr *attr)
1975 {
1976         uint32_t in[MLX5_ST_SZ_DW(create_virtq_in)] = {0};
1977         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
1978         struct mlx5_devx_obj *virtq_obj = mlx5_malloc(MLX5_MEM_ZERO,
1979                                                      sizeof(*virtq_obj),
1980                                                      0, SOCKET_ID_ANY);
1981         void *virtq = MLX5_ADDR_OF(create_virtq_in, in, virtq);
1982         void *hdr = MLX5_ADDR_OF(create_virtq_in, in, hdr);
1983         void *virtctx = MLX5_ADDR_OF(virtio_net_q, virtq, virtio_q_context);
1984
1985         if (!virtq_obj) {
1986                 DRV_LOG(ERR, "Failed to allocate virtq data.");
1987                 rte_errno = ENOMEM;
1988                 return NULL;
1989         }
1990         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
1991                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
1992         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
1993                  MLX5_GENERAL_OBJ_TYPE_VIRTQ);
1994         MLX5_SET16(virtio_net_q, virtq, hw_available_index,
1995                    attr->hw_available_index);
1996         MLX5_SET16(virtio_net_q, virtq, hw_used_index, attr->hw_used_index);
1997         MLX5_SET16(virtio_net_q, virtq, tso_ipv4, attr->tso_ipv4);
1998         MLX5_SET16(virtio_net_q, virtq, tso_ipv6, attr->tso_ipv6);
1999         MLX5_SET16(virtio_net_q, virtq, tx_csum, attr->tx_csum);
2000         MLX5_SET16(virtio_net_q, virtq, rx_csum, attr->rx_csum);
2001         MLX5_SET16(virtio_q, virtctx, virtio_version_1_0,
2002                    attr->virtio_version_1_0);
2003         MLX5_SET16(virtio_q, virtctx, event_mode, attr->event_mode);
2004         MLX5_SET(virtio_q, virtctx, event_qpn_or_msix, attr->qp_id);
2005         MLX5_SET64(virtio_q, virtctx, desc_addr, attr->desc_addr);
2006         MLX5_SET64(virtio_q, virtctx, used_addr, attr->used_addr);
2007         MLX5_SET64(virtio_q, virtctx, available_addr, attr->available_addr);
2008         MLX5_SET16(virtio_q, virtctx, queue_index, attr->queue_index);
2009         MLX5_SET16(virtio_q, virtctx, queue_size, attr->q_size);
2010         MLX5_SET(virtio_q, virtctx, virtio_q_mkey, attr->mkey);
2011         MLX5_SET(virtio_q, virtctx, umem_1_id, attr->umems[0].id);
2012         MLX5_SET(virtio_q, virtctx, umem_1_size, attr->umems[0].size);
2013         MLX5_SET64(virtio_q, virtctx, umem_1_offset, attr->umems[0].offset);
2014         MLX5_SET(virtio_q, virtctx, umem_2_id, attr->umems[1].id);
2015         MLX5_SET(virtio_q, virtctx, umem_2_size, attr->umems[1].size);
2016         MLX5_SET64(virtio_q, virtctx, umem_2_offset, attr->umems[1].offset);
2017         MLX5_SET(virtio_q, virtctx, umem_3_id, attr->umems[2].id);
2018         MLX5_SET(virtio_q, virtctx, umem_3_size, attr->umems[2].size);
2019         MLX5_SET64(virtio_q, virtctx, umem_3_offset, attr->umems[2].offset);
2020         MLX5_SET(virtio_q, virtctx, counter_set_id, attr->counters_obj_id);
2021         MLX5_SET(virtio_q, virtctx, pd, attr->pd);
2022         MLX5_SET(virtio_q, virtctx, queue_period_mode, attr->hw_latency_mode);
2023         MLX5_SET(virtio_q, virtctx, queue_period_us, attr->hw_max_latency_us);
2024         MLX5_SET(virtio_q, virtctx, queue_max_count, attr->hw_max_pending_comp);
2025         MLX5_SET(virtio_net_q, virtq, tisn_or_qpn, attr->tis_id);
2026         virtq_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
2027                                                     sizeof(out));
2028         if (!virtq_obj->obj) {
2029                 rte_errno = errno;
2030                 DRV_LOG(ERR, "Failed to create VIRTQ Obj using DevX.");
2031                 mlx5_free(virtq_obj);
2032                 return NULL;
2033         }
2034         virtq_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2035         return virtq_obj;
2036 }
2037
2038 /**
2039  * Modify VIRTQ using DevX API.
2040  *
2041  * @param[in] virtq_obj
2042  *   Pointer to virtq object structure.
2043  * @param [in] attr
2044  *   Pointer to modify virtq attributes structure.
2045  *
2046  * @return
2047  *   0 on success, a negative errno value otherwise and rte_errno is set.
2048  */
2049 int
2050 mlx5_devx_cmd_modify_virtq(struct mlx5_devx_obj *virtq_obj,
2051                            struct mlx5_devx_virtq_attr *attr)
2052 {
2053         uint32_t in[MLX5_ST_SZ_DW(create_virtq_in)] = {0};
2054         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
2055         void *virtq = MLX5_ADDR_OF(create_virtq_in, in, virtq);
2056         void *hdr = MLX5_ADDR_OF(create_virtq_in, in, hdr);
2057         void *virtctx = MLX5_ADDR_OF(virtio_net_q, virtq, virtio_q_context);
2058         int ret;
2059
2060         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
2061                  MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
2062         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
2063                  MLX5_GENERAL_OBJ_TYPE_VIRTQ);
2064         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, virtq_obj->id);
2065         MLX5_SET64(virtio_net_q, virtq, modify_field_select, attr->type);
2066         MLX5_SET16(virtio_q, virtctx, queue_index, attr->queue_index);
2067         switch (attr->type) {
2068         case MLX5_VIRTQ_MODIFY_TYPE_STATE:
2069                 MLX5_SET16(virtio_net_q, virtq, state, attr->state);
2070                 break;
2071         case MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS:
2072                 MLX5_SET(virtio_net_q, virtq, dirty_bitmap_mkey,
2073                          attr->dirty_bitmap_mkey);
2074                 MLX5_SET64(virtio_net_q, virtq, dirty_bitmap_addr,
2075                          attr->dirty_bitmap_addr);
2076                 MLX5_SET(virtio_net_q, virtq, dirty_bitmap_size,
2077                          attr->dirty_bitmap_size);
2078                 break;
2079         case MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE:
2080                 MLX5_SET(virtio_net_q, virtq, dirty_bitmap_dump_enable,
2081                          attr->dirty_bitmap_dump_enable);
2082                 break;
2083         default:
2084                 rte_errno = EINVAL;
2085                 return -rte_errno;
2086         }
2087         ret = mlx5_glue->devx_obj_modify(virtq_obj->obj, in, sizeof(in),
2088                                          out, sizeof(out));
2089         if (ret) {
2090                 DRV_LOG(ERR, "Failed to modify VIRTQ using DevX.");
2091                 rte_errno = errno;
2092                 return -rte_errno;
2093         }
2094         return ret;
2095 }
2096
2097 /**
2098  * Query VIRTQ using DevX API.
2099  *
2100  * @param[in] virtq_obj
2101  *   Pointer to virtq object structure.
2102  * @param [in/out] attr
2103  *   Pointer to virtq attributes structure.
2104  *
2105  * @return
2106  *   0 on success, a negative errno value otherwise and rte_errno is set.
2107  */
2108 int
2109 mlx5_devx_cmd_query_virtq(struct mlx5_devx_obj *virtq_obj,
2110                            struct mlx5_devx_virtq_attr *attr)
2111 {
2112         uint32_t in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {0};
2113         uint32_t out[MLX5_ST_SZ_DW(query_virtq_out)] = {0};
2114         void *hdr = MLX5_ADDR_OF(query_virtq_out, in, hdr);
2115         void *virtq = MLX5_ADDR_OF(query_virtq_out, out, virtq);
2116         int ret;
2117
2118         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
2119                  MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
2120         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
2121                  MLX5_GENERAL_OBJ_TYPE_VIRTQ);
2122         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, virtq_obj->id);
2123         ret = mlx5_glue->devx_obj_query(virtq_obj->obj, in, sizeof(in),
2124                                          out, sizeof(out));
2125         if (ret) {
2126                 DRV_LOG(ERR, "Failed to modify VIRTQ using DevX.");
2127                 rte_errno = errno;
2128                 return -errno;
2129         }
2130         attr->hw_available_index = MLX5_GET16(virtio_net_q, virtq,
2131                                               hw_available_index);
2132         attr->hw_used_index = MLX5_GET16(virtio_net_q, virtq, hw_used_index);
2133         attr->state = MLX5_GET16(virtio_net_q, virtq, state);
2134         attr->error_type = MLX5_GET16(virtio_net_q, virtq,
2135                                       virtio_q_context.error_type);
2136         return ret;
2137 }
2138
2139 /**
2140  * Create QP using DevX API.
2141  *
2142  * @param[in] ctx
2143  *   Context returned from mlx5 open_device() glue function.
2144  * @param [in] attr
2145  *   Pointer to QP attributes structure.
2146  *
2147  * @return
2148  *   The DevX object created, NULL otherwise and rte_errno is set.
2149  */
2150 struct mlx5_devx_obj *
2151 mlx5_devx_cmd_create_qp(void *ctx,
2152                         struct mlx5_devx_qp_attr *attr)
2153 {
2154         uint32_t in[MLX5_ST_SZ_DW(create_qp_in)] = {0};
2155         uint32_t out[MLX5_ST_SZ_DW(create_qp_out)] = {0};
2156         struct mlx5_devx_obj *qp_obj = mlx5_malloc(MLX5_MEM_ZERO,
2157                                                    sizeof(*qp_obj),
2158                                                    0, SOCKET_ID_ANY);
2159         void *qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2160
2161         if (!qp_obj) {
2162                 DRV_LOG(ERR, "Failed to allocate QP data.");
2163                 rte_errno = ENOMEM;
2164                 return NULL;
2165         }
2166         MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
2167         MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
2168         MLX5_SET(qpc, qpc, pd, attr->pd);
2169         MLX5_SET(qpc, qpc, ts_format, attr->ts_format);
2170         MLX5_SET(qpc, qpc, user_index, attr->user_index);
2171         if (attr->uar_index) {
2172                 if (attr->mmo) {
2173                         void *qpc_ext_and_pas_list = MLX5_ADDR_OF(create_qp_in,
2174                                 in, qpc_extension_and_pas_list);
2175                         void *qpc_ext = MLX5_ADDR_OF(qpc_extension_and_pas_list,
2176                                 qpc_ext_and_pas_list, qpc_data_extension);
2177
2178                         MLX5_SET(create_qp_in, in, qpc_ext, 1);
2179                         MLX5_SET(qpc_extension, qpc_ext, mmo, 1);
2180                 }
2181                 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2182                 MLX5_SET(qpc, qpc, uar_page, attr->uar_index);
2183                 if (attr->log_page_size > MLX5_ADAPTER_PAGE_SHIFT)
2184                         MLX5_SET(qpc, qpc, log_page_size,
2185                                  attr->log_page_size - MLX5_ADAPTER_PAGE_SHIFT);
2186                 if (attr->num_of_send_wqbbs) {
2187                         MLX5_ASSERT(RTE_IS_POWER_OF_2(attr->num_of_send_wqbbs));
2188                         MLX5_SET(qpc, qpc, cqn_snd, attr->cqn);
2189                         MLX5_SET(qpc, qpc, log_sq_size,
2190                                  rte_log2_u32(attr->num_of_send_wqbbs));
2191                 } else {
2192                         MLX5_SET(qpc, qpc, no_sq, 1);
2193                 }
2194                 if (attr->num_of_receive_wqes) {
2195                         MLX5_ASSERT(RTE_IS_POWER_OF_2(
2196                                         attr->num_of_receive_wqes));
2197                         MLX5_SET(qpc, qpc, cqn_rcv, attr->cqn);
2198                         MLX5_SET(qpc, qpc, log_rq_stride, attr->log_rq_stride -
2199                                  MLX5_LOG_RQ_STRIDE_SHIFT);
2200                         MLX5_SET(qpc, qpc, log_rq_size,
2201                                  rte_log2_u32(attr->num_of_receive_wqes));
2202                         MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
2203                 } else {
2204                         MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ);
2205                 }
2206                 if (attr->dbr_umem_valid) {
2207                         MLX5_SET(qpc, qpc, dbr_umem_valid,
2208                                  attr->dbr_umem_valid);
2209                         MLX5_SET(qpc, qpc, dbr_umem_id, attr->dbr_umem_id);
2210                 }
2211                 MLX5_SET64(qpc, qpc, dbr_addr, attr->dbr_address);
2212                 MLX5_SET64(create_qp_in, in, wq_umem_offset,
2213                            attr->wq_umem_offset);
2214                 MLX5_SET(create_qp_in, in, wq_umem_id, attr->wq_umem_id);
2215                 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
2216         } else {
2217                 /* Special QP to be managed by FW - no SQ\RQ\CQ\UAR\DB rec. */
2218                 MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ);
2219                 MLX5_SET(qpc, qpc, no_sq, 1);
2220         }
2221         qp_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
2222                                                  sizeof(out));
2223         if (!qp_obj->obj) {
2224                 rte_errno = errno;
2225                 DRV_LOG(ERR, "Failed to create QP Obj using DevX.");
2226                 mlx5_free(qp_obj);
2227                 return NULL;
2228         }
2229         qp_obj->id = MLX5_GET(create_qp_out, out, qpn);
2230         return qp_obj;
2231 }
2232
2233 /**
2234  * Modify QP using DevX API.
2235  * Currently supports only force loop-back QP.
2236  *
2237  * @param[in] qp
2238  *   Pointer to QP object structure.
2239  * @param [in] qp_st_mod_op
2240  *   The QP state modification operation.
2241  * @param [in] remote_qp_id
2242  *   The remote QP ID for MLX5_CMD_OP_INIT2RTR_QP operation.
2243  *
2244  * @return
2245  *   0 on success, a negative errno value otherwise and rte_errno is set.
2246  */
2247 int
2248 mlx5_devx_cmd_modify_qp_state(struct mlx5_devx_obj *qp, uint32_t qp_st_mod_op,
2249                               uint32_t remote_qp_id)
2250 {
2251         union {
2252                 uint32_t rst2init[MLX5_ST_SZ_DW(rst2init_qp_in)];
2253                 uint32_t init2rtr[MLX5_ST_SZ_DW(init2rtr_qp_in)];
2254                 uint32_t rtr2rts[MLX5_ST_SZ_DW(rtr2rts_qp_in)];
2255         } in;
2256         union {
2257                 uint32_t rst2init[MLX5_ST_SZ_DW(rst2init_qp_out)];
2258                 uint32_t init2rtr[MLX5_ST_SZ_DW(init2rtr_qp_out)];
2259                 uint32_t rtr2rts[MLX5_ST_SZ_DW(rtr2rts_qp_out)];
2260         } out;
2261         void *qpc;
2262         int ret;
2263         unsigned int inlen;
2264         unsigned int outlen;
2265
2266         memset(&in, 0, sizeof(in));
2267         memset(&out, 0, sizeof(out));
2268         MLX5_SET(rst2init_qp_in, &in, opcode, qp_st_mod_op);
2269         switch (qp_st_mod_op) {
2270         case MLX5_CMD_OP_RST2INIT_QP:
2271                 MLX5_SET(rst2init_qp_in, &in, qpn, qp->id);
2272                 qpc = MLX5_ADDR_OF(rst2init_qp_in, &in, qpc);
2273                 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1);
2274                 MLX5_SET(qpc, qpc, rre, 1);
2275                 MLX5_SET(qpc, qpc, rwe, 1);
2276                 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2277                 inlen = sizeof(in.rst2init);
2278                 outlen = sizeof(out.rst2init);
2279                 break;
2280         case MLX5_CMD_OP_INIT2RTR_QP:
2281                 MLX5_SET(init2rtr_qp_in, &in, qpn, qp->id);
2282                 qpc = MLX5_ADDR_OF(init2rtr_qp_in, &in, qpc);
2283                 MLX5_SET(qpc, qpc, primary_address_path.fl, 1);
2284                 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1);
2285                 MLX5_SET(qpc, qpc, mtu, 1);
2286                 MLX5_SET(qpc, qpc, log_msg_max, 30);
2287                 MLX5_SET(qpc, qpc, remote_qpn, remote_qp_id);
2288                 MLX5_SET(qpc, qpc, min_rnr_nak, 0);
2289                 inlen = sizeof(in.init2rtr);
2290                 outlen = sizeof(out.init2rtr);
2291                 break;
2292         case MLX5_CMD_OP_RTR2RTS_QP:
2293                 qpc = MLX5_ADDR_OF(rtr2rts_qp_in, &in, qpc);
2294                 MLX5_SET(rtr2rts_qp_in, &in, qpn, qp->id);
2295                 MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 16);
2296                 MLX5_SET(qpc, qpc, log_ack_req_freq, 0);
2297                 MLX5_SET(qpc, qpc, retry_count, 7);
2298                 MLX5_SET(qpc, qpc, rnr_retry, 7);
2299                 inlen = sizeof(in.rtr2rts);
2300                 outlen = sizeof(out.rtr2rts);
2301                 break;
2302         default:
2303                 DRV_LOG(ERR, "Invalid or unsupported QP modify op %u.",
2304                         qp_st_mod_op);
2305                 rte_errno = EINVAL;
2306                 return -rte_errno;
2307         }
2308         ret = mlx5_glue->devx_obj_modify(qp->obj, &in, inlen, &out, outlen);
2309         if (ret) {
2310                 DRV_LOG(ERR, "Failed to modify QP using DevX.");
2311                 rte_errno = errno;
2312                 return -rte_errno;
2313         }
2314         return ret;
2315 }
2316
2317 struct mlx5_devx_obj *
2318 mlx5_devx_cmd_create_virtio_q_counters(void *ctx)
2319 {
2320         uint32_t in[MLX5_ST_SZ_DW(create_virtio_q_counters_in)] = {0};
2321         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
2322         struct mlx5_devx_obj *couners_obj = mlx5_malloc(MLX5_MEM_ZERO,
2323                                                        sizeof(*couners_obj), 0,
2324                                                        SOCKET_ID_ANY);
2325         void *hdr = MLX5_ADDR_OF(create_virtio_q_counters_in, in, hdr);
2326
2327         if (!couners_obj) {
2328                 DRV_LOG(ERR, "Failed to allocate virtio queue counters data.");
2329                 rte_errno = ENOMEM;
2330                 return NULL;
2331         }
2332         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
2333                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2334         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
2335                  MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS);
2336         couners_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
2337                                                       sizeof(out));
2338         if (!couners_obj->obj) {
2339                 rte_errno = errno;
2340                 DRV_LOG(ERR, "Failed to create virtio queue counters Obj using"
2341                         " DevX.");
2342                 mlx5_free(couners_obj);
2343                 return NULL;
2344         }
2345         couners_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2346         return couners_obj;
2347 }
2348
2349 int
2350 mlx5_devx_cmd_query_virtio_q_counters(struct mlx5_devx_obj *couners_obj,
2351                                    struct mlx5_devx_virtio_q_couners_attr *attr)
2352 {
2353         uint32_t in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {0};
2354         uint32_t out[MLX5_ST_SZ_DW(query_virtio_q_counters_out)] = {0};
2355         void *hdr = MLX5_ADDR_OF(query_virtio_q_counters_out, in, hdr);
2356         void *virtio_q_counters = MLX5_ADDR_OF(query_virtio_q_counters_out, out,
2357                                                virtio_q_counters);
2358         int ret;
2359
2360         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
2361                  MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
2362         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
2363                  MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS);
2364         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, couners_obj->id);
2365         ret = mlx5_glue->devx_obj_query(couners_obj->obj, in, sizeof(in), out,
2366                                         sizeof(out));
2367         if (ret) {
2368                 DRV_LOG(ERR, "Failed to query virtio q counters using DevX.");
2369                 rte_errno = errno;
2370                 return -errno;
2371         }
2372         attr->received_desc = MLX5_GET64(virtio_q_counters, virtio_q_counters,
2373                                          received_desc);
2374         attr->completed_desc = MLX5_GET64(virtio_q_counters, virtio_q_counters,
2375                                           completed_desc);
2376         attr->error_cqes = MLX5_GET(virtio_q_counters, virtio_q_counters,
2377                                     error_cqes);
2378         attr->bad_desc_errors = MLX5_GET(virtio_q_counters, virtio_q_counters,
2379                                          bad_desc_errors);
2380         attr->exceed_max_chain = MLX5_GET(virtio_q_counters, virtio_q_counters,
2381                                           exceed_max_chain);
2382         attr->invalid_buffer = MLX5_GET(virtio_q_counters, virtio_q_counters,
2383                                         invalid_buffer);
2384         return ret;
2385 }
2386
2387 /**
2388  * Create general object of type FLOW_HIT_ASO using DevX API.
2389  *
2390  * @param[in] ctx
2391  *   Context returned from mlx5 open_device() glue function.
2392  * @param [in] pd
2393  *   PD value to associate the FLOW_HIT_ASO object with.
2394  *
2395  * @return
2396  *   The DevX object created, NULL otherwise and rte_errno is set.
2397  */
2398 struct mlx5_devx_obj *
2399 mlx5_devx_cmd_create_flow_hit_aso_obj(void *ctx, uint32_t pd)
2400 {
2401         uint32_t in[MLX5_ST_SZ_DW(create_flow_hit_aso_in)] = {0};
2402         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
2403         struct mlx5_devx_obj *flow_hit_aso_obj = NULL;
2404         void *ptr = NULL;
2405
2406         flow_hit_aso_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*flow_hit_aso_obj),
2407                                        0, SOCKET_ID_ANY);
2408         if (!flow_hit_aso_obj) {
2409                 DRV_LOG(ERR, "Failed to allocate FLOW_HIT_ASO object data");
2410                 rte_errno = ENOMEM;
2411                 return NULL;
2412         }
2413         ptr = MLX5_ADDR_OF(create_flow_hit_aso_in, in, hdr);
2414         MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
2415                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2416         MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
2417                  MLX5_GENERAL_OBJ_TYPE_FLOW_HIT_ASO);
2418         ptr = MLX5_ADDR_OF(create_flow_hit_aso_in, in, flow_hit_aso);
2419         MLX5_SET(flow_hit_aso, ptr, access_pd, pd);
2420         flow_hit_aso_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
2421                                                            out, sizeof(out));
2422         if (!flow_hit_aso_obj->obj) {
2423                 rte_errno = errno;
2424                 DRV_LOG(ERR, "Failed to create FLOW_HIT_ASO obj using DevX.");
2425                 mlx5_free(flow_hit_aso_obj);
2426                 return NULL;
2427         }
2428         flow_hit_aso_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2429         return flow_hit_aso_obj;
2430 }
2431
2432 /*
2433  * Create PD using DevX API.
2434  *
2435  * @param[in] ctx
2436  *   Context returned from mlx5 open_device() glue function.
2437  *
2438  * @return
2439  *   The DevX object created, NULL otherwise and rte_errno is set.
2440  */
2441 struct mlx5_devx_obj *
2442 mlx5_devx_cmd_alloc_pd(void *ctx)
2443 {
2444         struct mlx5_devx_obj *ppd =
2445                 mlx5_malloc(MLX5_MEM_ZERO, sizeof(*ppd), 0, SOCKET_ID_ANY);
2446         u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {0};
2447         u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {0};
2448
2449         if (!ppd) {
2450                 DRV_LOG(ERR, "Failed to allocate PD data.");
2451                 rte_errno = ENOMEM;
2452                 return NULL;
2453         }
2454         MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
2455         ppd->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
2456                                 out, sizeof(out));
2457         if (!ppd->obj) {
2458                 mlx5_free(ppd);
2459                 DRV_LOG(ERR, "Failed to allocate PD Obj using DevX.");
2460                 rte_errno = errno;
2461                 return NULL;
2462         }
2463         ppd->id = MLX5_GET(alloc_pd_out, out, pd);
2464         return ppd;
2465 }
2466
2467 /**
2468  * Create general object of type FLOW_METER_ASO using DevX API.
2469  *
2470  * @param[in] ctx
2471  *   Context returned from mlx5 open_device() glue function.
2472  * @param [in] pd
2473  *   PD value to associate the FLOW_METER_ASO object with.
2474  * @param [in] log_obj_size
2475  *   log_obj_size define to allocate number of 2 * meters
2476  *   in one FLOW_METER_ASO object.
2477  *
2478  * @return
2479  *   The DevX object created, NULL otherwise and rte_errno is set.
2480  */
2481 struct mlx5_devx_obj *
2482 mlx5_devx_cmd_create_flow_meter_aso_obj(void *ctx, uint32_t pd,
2483                                                 uint32_t log_obj_size)
2484 {
2485         uint32_t in[MLX5_ST_SZ_DW(create_flow_meter_aso_in)] = {0};
2486         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2487         struct mlx5_devx_obj *flow_meter_aso_obj;
2488         void *ptr;
2489
2490         flow_meter_aso_obj = mlx5_malloc(MLX5_MEM_ZERO,
2491                                                 sizeof(*flow_meter_aso_obj),
2492                                                 0, SOCKET_ID_ANY);
2493         if (!flow_meter_aso_obj) {
2494                 DRV_LOG(ERR, "Failed to allocate FLOW_METER_ASO object data");
2495                 rte_errno = ENOMEM;
2496                 return NULL;
2497         }
2498         ptr = MLX5_ADDR_OF(create_flow_meter_aso_in, in, hdr);
2499         MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
2500                 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2501         MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
2502                 MLX5_GENERAL_OBJ_TYPE_FLOW_METER_ASO);
2503         MLX5_SET(general_obj_in_cmd_hdr, ptr, log_obj_range,
2504                 log_obj_size);
2505         ptr = MLX5_ADDR_OF(create_flow_meter_aso_in, in, flow_meter_aso);
2506         MLX5_SET(flow_meter_aso, ptr, access_pd, pd);
2507         flow_meter_aso_obj->obj = mlx5_glue->devx_obj_create(
2508                                                         ctx, in, sizeof(in),
2509                                                         out, sizeof(out));
2510         if (!flow_meter_aso_obj->obj) {
2511                 rte_errno = errno;
2512                 DRV_LOG(ERR, "Failed to create FLOW_METER_ASO obj using DevX.");
2513                 mlx5_free(flow_meter_aso_obj);
2514                 return NULL;
2515         }
2516         flow_meter_aso_obj->id = MLX5_GET(general_obj_out_cmd_hdr,
2517                                                                 out, obj_id);
2518         return flow_meter_aso_obj;
2519 }
2520
2521 /*
2522  * Create general object of type CONN_TRACK_OFFLOAD using DevX API.
2523  *
2524  * @param[in] ctx
2525  *   Context returned from mlx5 open_device() glue function.
2526  * @param [in] pd
2527  *   PD value to associate the CONN_TRACK_OFFLOAD ASO object with.
2528  * @param [in] log_obj_size
2529  *   log_obj_size to allocate its power of 2 * objects
2530  *   in one CONN_TRACK_OFFLOAD bulk allocation.
2531  *
2532  * @return
2533  *   The DevX object created, NULL otherwise and rte_errno is set.
2534  */
2535 struct mlx5_devx_obj *
2536 mlx5_devx_cmd_create_conn_track_offload_obj(void *ctx, uint32_t pd,
2537                                             uint32_t log_obj_size)
2538 {
2539         uint32_t in[MLX5_ST_SZ_DW(create_conn_track_aso_in)] = {0};
2540         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2541         struct mlx5_devx_obj *ct_aso_obj;
2542         void *ptr;
2543
2544         ct_aso_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*ct_aso_obj),
2545                                  0, SOCKET_ID_ANY);
2546         if (!ct_aso_obj) {
2547                 DRV_LOG(ERR, "Failed to allocate CONN_TRACK_OFFLOAD object.");
2548                 rte_errno = ENOMEM;
2549                 return NULL;
2550         }
2551         ptr = MLX5_ADDR_OF(create_conn_track_aso_in, in, hdr);
2552         MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
2553                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2554         MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
2555                  MLX5_GENERAL_OBJ_TYPE_CONN_TRACK_OFFLOAD);
2556         MLX5_SET(general_obj_in_cmd_hdr, ptr, log_obj_range, log_obj_size);
2557         ptr = MLX5_ADDR_OF(create_conn_track_aso_in, in, conn_track_offload);
2558         MLX5_SET(conn_track_offload, ptr, conn_track_aso_access_pd, pd);
2559         ct_aso_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
2560                                                      out, sizeof(out));
2561         if (!ct_aso_obj->obj) {
2562                 rte_errno = errno;
2563                 DRV_LOG(ERR, "Failed to create CONN_TRACK_OFFLOAD obj by using DevX.");
2564                 mlx5_free(ct_aso_obj);
2565                 return NULL;
2566         }
2567         ct_aso_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2568         return ct_aso_obj;
2569 }
2570
2571 /**
2572  * Create general object of type GENEVE TLV option using DevX API.
2573  *
2574  * @param[in] ctx
2575  *   Context returned from mlx5 open_device() glue function.
2576  * @param [in] class
2577  *   TLV option variable value of class
2578  * @param [in] type
2579  *   TLV option variable value of type
2580  * @param [in] len
2581  *   TLV option variable value of len
2582  *
2583  * @return
2584  *   The DevX object created, NULL otherwise and rte_errno is set.
2585  */
2586 struct mlx5_devx_obj *
2587 mlx5_devx_cmd_create_geneve_tlv_option(void *ctx,
2588                 uint16_t class, uint8_t type, uint8_t len)
2589 {
2590         uint32_t in[MLX5_ST_SZ_DW(create_geneve_tlv_option_in)] = {0};
2591         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
2592         struct mlx5_devx_obj *geneve_tlv_opt_obj = mlx5_malloc(MLX5_MEM_ZERO,
2593                                                    sizeof(*geneve_tlv_opt_obj),
2594                                                    0, SOCKET_ID_ANY);
2595
2596         if (!geneve_tlv_opt_obj) {
2597                 DRV_LOG(ERR, "Failed to allocate geneve tlv option object.");
2598                 rte_errno = ENOMEM;
2599                 return NULL;
2600         }
2601         void *hdr = MLX5_ADDR_OF(create_geneve_tlv_option_in, in, hdr);
2602         void *opt = MLX5_ADDR_OF(create_geneve_tlv_option_in, in,
2603                         geneve_tlv_opt);
2604         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
2605                         MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2606         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
2607                  MLX5_GENERAL_OBJ_TYPE_GENEVE_TLV_OPT);
2608         MLX5_SET(geneve_tlv_option, opt, option_class,
2609                         rte_be_to_cpu_16(class));
2610         MLX5_SET(geneve_tlv_option, opt, option_type, type);
2611         MLX5_SET(geneve_tlv_option, opt, option_data_length, len);
2612         geneve_tlv_opt_obj->obj = mlx5_glue->devx_obj_create(ctx, in,
2613                                         sizeof(in), out, sizeof(out));
2614         if (!geneve_tlv_opt_obj->obj) {
2615                 rte_errno = errno;
2616                 DRV_LOG(ERR, "Failed to create Geneve tlv option "
2617                                 "Obj using DevX.");
2618                 mlx5_free(geneve_tlv_opt_obj);
2619                 return NULL;
2620         }
2621         geneve_tlv_opt_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2622         return geneve_tlv_opt_obj;
2623 }
2624
2625 int
2626 mlx5_devx_cmd_wq_query(void *wq, uint32_t *counter_set_id)
2627 {
2628 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
2629         uint32_t in[MLX5_ST_SZ_DW(query_rq_in)] = {0};
2630         uint32_t out[MLX5_ST_SZ_DW(query_rq_out)] = {0};
2631         int rc;
2632         void *rq_ctx;
2633
2634         MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
2635         MLX5_SET(query_rq_in, in, rqn, ((struct ibv_wq *)wq)->wq_num);
2636         rc = mlx5_glue->devx_wq_query(wq, in, sizeof(in), out, sizeof(out));
2637         if (rc) {
2638                 rte_errno = errno;
2639                 DRV_LOG(ERR, "Failed to query WQ counter set ID using DevX - "
2640                         "rc = %d, errno = %d.", rc, errno);
2641                 return -rc;
2642         };
2643         rq_ctx = MLX5_ADDR_OF(query_rq_out, out, rq_context);
2644         *counter_set_id = MLX5_GET(rqc, rq_ctx, counter_set_id);
2645         return 0;
2646 #else
2647         (void)wq;
2648         (void)counter_set_id;
2649         return -ENOTSUP;
2650 #endif
2651 }
2652
2653 /*
2654  * Allocate queue counters via devx interface.
2655  *
2656  * @param[in] ctx
2657  *   Context returned from mlx5 open_device() glue function.
2658  *
2659  * @return
2660  *   Pointer to counter object on success, a NULL value otherwise and
2661  *   rte_errno is set.
2662  */
2663 struct mlx5_devx_obj *
2664 mlx5_devx_cmd_queue_counter_alloc(void *ctx)
2665 {
2666         struct mlx5_devx_obj *dcs = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*dcs), 0,
2667                                                 SOCKET_ID_ANY);
2668         uint32_t in[MLX5_ST_SZ_DW(alloc_q_counter_in)]   = {0};
2669         uint32_t out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {0};
2670
2671         if (!dcs) {
2672                 rte_errno = ENOMEM;
2673                 return NULL;
2674         }
2675         MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
2676         dcs->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
2677                                               sizeof(out));
2678         if (!dcs->obj) {
2679                 DRV_LOG(DEBUG, "Can't allocate q counter set by DevX - error "
2680                         "%d.", errno);
2681                 rte_errno = errno;
2682                 mlx5_free(dcs);
2683                 return NULL;
2684         }
2685         dcs->id = MLX5_GET(alloc_q_counter_out, out, counter_set_id);
2686         return dcs;
2687 }
2688
2689 /**
2690  * Query queue counters values.
2691  *
2692  * @param[in] dcs
2693  *   devx object of the queue counter set.
2694  * @param[in] clear
2695  *   Whether hardware should clear the counters after the query or not.
2696  *  @param[out] out_of_buffers
2697  *   Number of dropped occurred due to lack of WQE for the associated QPs/RQs.
2698  *
2699  * @return
2700  *   0 on success, a negative value otherwise.
2701  */
2702 int
2703 mlx5_devx_cmd_queue_counter_query(struct mlx5_devx_obj *dcs, int clear,
2704                                   uint32_t *out_of_buffers)
2705 {
2706         uint32_t out[MLX5_ST_SZ_BYTES(query_q_counter_out)] = {0};
2707         uint32_t in[MLX5_ST_SZ_DW(query_q_counter_in)] = {0};
2708         int rc;
2709
2710         MLX5_SET(query_q_counter_in, in, opcode,
2711                  MLX5_CMD_OP_QUERY_Q_COUNTER);
2712         MLX5_SET(query_q_counter_in, in, op_mod, 0);
2713         MLX5_SET(query_q_counter_in, in, counter_set_id, dcs->id);
2714         MLX5_SET(query_q_counter_in, in, clear, !!clear);
2715         rc = mlx5_glue->devx_obj_query(dcs->obj, in, sizeof(in), out,
2716                                        sizeof(out));
2717         if (rc) {
2718                 DRV_LOG(ERR, "Failed to query devx q counter set - rc %d", rc);
2719                 rte_errno = rc;
2720                 return -rc;
2721         }
2722         *out_of_buffers = MLX5_GET(query_q_counter_out, out, out_of_buffer);
2723         return 0;
2724 }
2725
2726 /**
2727  * Create general object of type DEK using DevX API.
2728  *
2729  * @param[in] ctx
2730  *   Context returned from mlx5 open_device() glue function.
2731  * @param [in] attr
2732  *   Pointer to DEK attributes structure.
2733  *
2734  * @return
2735  *   The DevX object created, NULL otherwise and rte_errno is set.
2736  */
2737 struct mlx5_devx_obj *
2738 mlx5_devx_cmd_create_dek_obj(void *ctx, struct mlx5_devx_dek_attr *attr)
2739 {
2740         uint32_t in[MLX5_ST_SZ_DW(create_dek_in)] = {0};
2741         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
2742         struct mlx5_devx_obj *dek_obj = NULL;
2743         void *ptr = NULL, *key_addr = NULL;
2744
2745         dek_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*dek_obj),
2746                               0, SOCKET_ID_ANY);
2747         if (dek_obj == NULL) {
2748                 DRV_LOG(ERR, "Failed to allocate DEK object data");
2749                 rte_errno = ENOMEM;
2750                 return NULL;
2751         }
2752         ptr = MLX5_ADDR_OF(create_dek_in, in, hdr);
2753         MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
2754                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2755         MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
2756                  MLX5_GENERAL_OBJ_TYPE_DEK);
2757         ptr = MLX5_ADDR_OF(create_dek_in, in, dek);
2758         MLX5_SET(dek, ptr, key_size, attr->key_size);
2759         MLX5_SET(dek, ptr, has_keytag, attr->has_keytag);
2760         MLX5_SET(dek, ptr, key_purpose, attr->key_purpose);
2761         MLX5_SET(dek, ptr, pd, attr->pd);
2762         MLX5_SET64(dek, ptr, opaque, attr->opaque);
2763         key_addr = MLX5_ADDR_OF(dek, ptr, key);
2764         memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
2765         dek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
2766                                                   out, sizeof(out));
2767         if (dek_obj->obj == NULL) {
2768                 rte_errno = errno;
2769                 DRV_LOG(ERR, "Failed to create DEK obj using DevX.");
2770                 mlx5_free(dek_obj);
2771                 return NULL;
2772         }
2773         dek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2774         return dek_obj;
2775 }
2776
2777 /**
2778  * Create general object of type IMPORT_KEK using DevX API.
2779  *
2780  * @param[in] ctx
2781  *   Context returned from mlx5 open_device() glue function.
2782  * @param [in] attr
2783  *   Pointer to IMPORT_KEK attributes structure.
2784  *
2785  * @return
2786  *   The DevX object created, NULL otherwise and rte_errno is set.
2787  */
2788 struct mlx5_devx_obj *
2789 mlx5_devx_cmd_create_import_kek_obj(void *ctx,
2790                                     struct mlx5_devx_import_kek_attr *attr)
2791 {
2792         uint32_t in[MLX5_ST_SZ_DW(create_import_kek_in)] = {0};
2793         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
2794         struct mlx5_devx_obj *import_kek_obj = NULL;
2795         void *ptr = NULL, *key_addr = NULL;
2796
2797         import_kek_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*import_kek_obj),
2798                                      0, SOCKET_ID_ANY);
2799         if (import_kek_obj == NULL) {
2800                 DRV_LOG(ERR, "Failed to allocate IMPORT_KEK object data");
2801                 rte_errno = ENOMEM;
2802                 return NULL;
2803         }
2804         ptr = MLX5_ADDR_OF(create_import_kek_in, in, hdr);
2805         MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
2806                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2807         MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
2808                  MLX5_GENERAL_OBJ_TYPE_IMPORT_KEK);
2809         ptr = MLX5_ADDR_OF(create_import_kek_in, in, import_kek);
2810         MLX5_SET(import_kek, ptr, key_size, attr->key_size);
2811         key_addr = MLX5_ADDR_OF(import_kek, ptr, key);
2812         memcpy(key_addr, (void *)(attr->key), MLX5_CRYPTO_KEY_MAX_SIZE);
2813         import_kek_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
2814                                                          out, sizeof(out));
2815         if (import_kek_obj->obj == NULL) {
2816                 rte_errno = errno;
2817                 DRV_LOG(ERR, "Failed to create IMPORT_KEK object using DevX.");
2818                 mlx5_free(import_kek_obj);
2819                 return NULL;
2820         }
2821         import_kek_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2822         return import_kek_obj;
2823 }
2824
2825 /**
2826  * Create general object of type CREDENTIAL using DevX API.
2827  *
2828  * @param[in] ctx
2829  *   Context returned from mlx5 open_device() glue function.
2830  * @param [in] attr
2831  *   Pointer to CREDENTIAL attributes structure.
2832  *
2833  * @return
2834  *   The DevX object created, NULL otherwise and rte_errno is set.
2835  */
2836 struct mlx5_devx_obj *
2837 mlx5_devx_cmd_create_credential_obj(void *ctx,
2838                                     struct mlx5_devx_credential_attr *attr)
2839 {
2840         uint32_t in[MLX5_ST_SZ_DW(create_credential_in)] = {0};
2841         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
2842         struct mlx5_devx_obj *credential_obj = NULL;
2843         void *ptr = NULL, *credential_addr = NULL;
2844
2845         credential_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*credential_obj),
2846                                      0, SOCKET_ID_ANY);
2847         if (credential_obj == NULL) {
2848                 DRV_LOG(ERR, "Failed to allocate CREDENTIAL object data");
2849                 rte_errno = ENOMEM;
2850                 return NULL;
2851         }
2852         ptr = MLX5_ADDR_OF(create_credential_in, in, hdr);
2853         MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
2854                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2855         MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
2856                  MLX5_GENERAL_OBJ_TYPE_CREDENTIAL);
2857         ptr = MLX5_ADDR_OF(create_credential_in, in, credential);
2858         MLX5_SET(credential, ptr, credential_role, attr->credential_role);
2859         credential_addr = MLX5_ADDR_OF(credential, ptr, credential);
2860         memcpy(credential_addr, (void *)(attr->credential),
2861                MLX5_CRYPTO_CREDENTIAL_SIZE);
2862         credential_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
2863                                                          out, sizeof(out));
2864         if (credential_obj->obj == NULL) {
2865                 rte_errno = errno;
2866                 DRV_LOG(ERR, "Failed to create CREDENTIAL object using DevX.");
2867                 mlx5_free(credential_obj);
2868                 return NULL;
2869         }
2870         credential_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2871         return credential_obj;
2872 }
2873
2874 /**
2875  * Create general object of type CRYPTO_LOGIN using DevX API.
2876  *
2877  * @param[in] ctx
2878  *   Context returned from mlx5 open_device() glue function.
2879  * @param [in] attr
2880  *   Pointer to CRYPTO_LOGIN attributes structure.
2881  *
2882  * @return
2883  *   The DevX object created, NULL otherwise and rte_errno is set.
2884  */
2885 struct mlx5_devx_obj *
2886 mlx5_devx_cmd_create_crypto_login_obj(void *ctx,
2887                                       struct mlx5_devx_crypto_login_attr *attr)
2888 {
2889         uint32_t in[MLX5_ST_SZ_DW(create_crypto_login_in)] = {0};
2890         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
2891         struct mlx5_devx_obj *crypto_login_obj = NULL;
2892         void *ptr = NULL, *credential_addr = NULL;
2893
2894         crypto_login_obj = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*crypto_login_obj),
2895                                        0, SOCKET_ID_ANY);
2896         if (crypto_login_obj == NULL) {
2897                 DRV_LOG(ERR, "Failed to allocate CRYPTO_LOGIN object data");
2898                 rte_errno = ENOMEM;
2899                 return NULL;
2900         }
2901         ptr = MLX5_ADDR_OF(create_crypto_login_in, in, hdr);
2902         MLX5_SET(general_obj_in_cmd_hdr, ptr, opcode,
2903                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
2904         MLX5_SET(general_obj_in_cmd_hdr, ptr, obj_type,
2905                  MLX5_GENERAL_OBJ_TYPE_CRYPTO_LOGIN);
2906         ptr = MLX5_ADDR_OF(create_crypto_login_in, in, crypto_login);
2907         MLX5_SET(crypto_login, ptr, credential_pointer,
2908                  attr->credential_pointer);
2909         MLX5_SET(crypto_login, ptr, session_import_kek_ptr,
2910                  attr->session_import_kek_ptr);
2911         credential_addr = MLX5_ADDR_OF(crypto_login, ptr, credential);
2912         memcpy(credential_addr, (void *)(attr->credential),
2913                MLX5_CRYPTO_CREDENTIAL_SIZE);
2914         crypto_login_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
2915                                                            out, sizeof(out));
2916         if (crypto_login_obj->obj == NULL) {
2917                 rte_errno = errno;
2918                 DRV_LOG(ERR, "Failed to create CRYPTO_LOGIN obj using DevX.");
2919                 mlx5_free(crypto_login_obj);
2920                 return NULL;
2921         }
2922         crypto_login_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2923         return crypto_login_obj;
2924 }
2925
2926 /**
2927  * Query LAG context.
2928  *
2929  * @param[in] ctx
2930  *   Pointer to ibv_context, returned from mlx5dv_open_device.
2931  * @param[out] lag_ctx
2932  *   Pointer to struct mlx5_devx_lag_context, to be set by the routine.
2933  *
2934  * @return
2935  *   0 on success, a negative value otherwise.
2936  */
2937 int
2938 mlx5_devx_cmd_query_lag(void *ctx,
2939                         struct mlx5_devx_lag_context *lag_ctx)
2940 {
2941         uint32_t in[MLX5_ST_SZ_DW(query_lag_in)] = {0};
2942         uint32_t out[MLX5_ST_SZ_DW(query_lag_out)] = {0};
2943         void *lctx;
2944         int rc;
2945
2946         MLX5_SET(query_lag_in, in, opcode, MLX5_CMD_OP_QUERY_LAG);
2947         rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
2948         if (rc)
2949                 goto error;
2950         lctx = MLX5_ADDR_OF(query_lag_out, out, context);
2951         lag_ctx->fdb_selection_mode = MLX5_GET(lag_context, lctx,
2952                                                fdb_selection_mode);
2953         lag_ctx->port_select_mode = MLX5_GET(lag_context, lctx,
2954                                                port_select_mode);
2955         lag_ctx->lag_state = MLX5_GET(lag_context, lctx, lag_state);
2956         lag_ctx->tx_remap_affinity_2 = MLX5_GET(lag_context, lctx,
2957                                                 tx_remap_affinity_2);
2958         lag_ctx->tx_remap_affinity_1 = MLX5_GET(lag_context, lctx,
2959                                                 tx_remap_affinity_1);
2960         return 0;
2961 error:
2962         rc = (rc > 0) ? -rc : rc;
2963         return rc;
2964 }