drivers/net/cnxk/cnxk_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4 #include <cnxk_ethdev.h>
   5
   6 #include <rte_eventdev.h>
   7
   8 static inline uint64_t
   9 nix_get_rx_offload_capa(struct cnxk_eth_dev *dev)
  10 {
  11         uint64_t capa = CNXK_NIX_RX_OFFLOAD_CAPA;
  12
  13         if (roc_nix_is_vf_or_sdp(&dev->nix) ||
  14             dev->npc.switch_header_type == ROC_PRIV_FLAGS_HIGIG)
  15                 capa &= ~RTE_ETH_RX_OFFLOAD_TIMESTAMP;
  16
  17         return capa;
  18 }
  19
  20 static inline uint64_t
  21 nix_get_tx_offload_capa(struct cnxk_eth_dev *dev)
  22 {
  23         RTE_SET_USED(dev);
  24         return CNXK_NIX_TX_OFFLOAD_CAPA;
  25 }
  26
  27 static inline uint32_t
  28 nix_get_speed_capa(struct cnxk_eth_dev *dev)
  29 {
  30         uint32_t speed_capa;
  31
  32         /* Auto negotiation disabled */
  33         speed_capa = RTE_ETH_LINK_SPEED_FIXED;
  34         if (!roc_nix_is_vf_or_sdp(&dev->nix) && !roc_nix_is_lbk(&dev->nix)) {
  35                 speed_capa |= RTE_ETH_LINK_SPEED_1G | RTE_ETH_LINK_SPEED_10G |
  36                               RTE_ETH_LINK_SPEED_25G | RTE_ETH_LINK_SPEED_40G |
  37                               RTE_ETH_LINK_SPEED_50G | RTE_ETH_LINK_SPEED_100G;
  38         }
  39
  40         return speed_capa;
  41 }
  42
  43 int
  44 cnxk_nix_inb_mode_set(struct cnxk_eth_dev *dev, bool use_inl_dev)
  45 {
  46         struct roc_nix *nix = &dev->nix;
  47
  48         if (dev->inb.inl_dev == use_inl_dev)
  49                 return 0;
  50
  51         plt_nix_dbg("Security sessions(%u) still active, inl=%u!!!",
  52                     dev->inb.nb_sess, !!dev->inb.inl_dev);
  53
  54         /* Change the mode */
  55         dev->inb.inl_dev = use_inl_dev;
  56
  57         /* Update RoC for NPC rule insertion */
  58         roc_nix_inb_mode_set(nix, use_inl_dev);
  59
  60         /* Setup lookup mem */
  61         return cnxk_nix_lookup_mem_sa_base_set(dev);
  62 }
  63
  64 static int
  65 nix_security_setup(struct cnxk_eth_dev *dev)
  66 {
  67         struct roc_nix *nix = &dev->nix;
  68         int i, rc = 0;
  69
  70         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY) {
  71                 /* Setup Inline Inbound */
  72                 rc = roc_nix_inl_inb_init(nix);
  73                 if (rc) {
  74                         plt_err("Failed to initialize nix inline inb, rc=%d",
  75                                 rc);
  76                         return rc;
  77                 }
  78
  79                 /* By default pick using inline device for poll mode.
  80                  * Will be overridden when event mode rq's are setup.
  81                  */
  82                 cnxk_nix_inb_mode_set(dev, true);
  83
  84                 /* Allocate memory to be used as dptr for CPT ucode
  85                  * WRITE_SA op.
  86                  */
  87                 dev->inb.sa_dptr =
  88                         plt_zmalloc(ROC_NIX_INL_OT_IPSEC_INB_HW_SZ, 0);
  89                 if (!dev->inb.sa_dptr) {
  90                         plt_err("Couldn't allocate memory for SA dptr");
  91                         rc = -ENOMEM;
  92                         goto cleanup;
  93                 }
  94         }
  95
  96         if (dev->tx_offloads & RTE_ETH_TX_OFFLOAD_SECURITY ||
  97             dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY) {
  98                 struct plt_bitmap *bmap;
  99                 size_t bmap_sz;
 100                 void *mem;
 101
 102                 /* Setup enough descriptors for all tx queues */
 103                 nix->outb_nb_desc = dev->outb.nb_desc;
 104                 nix->outb_nb_crypto_qs = dev->outb.nb_crypto_qs;
 105
 106                 /* Setup Inline Outbound */
 107                 rc = roc_nix_inl_outb_init(nix);
 108                 if (rc) {
 109                         plt_err("Failed to initialize nix inline outb, rc=%d",
 110                                 rc);
 111                         goto sa_dptr_free;
 112                 }
 113
 114                 dev->outb.lf_base = roc_nix_inl_outb_lf_base_get(nix);
 115
 116                 /* Skip the rest if DEV_TX_OFFLOAD_SECURITY is not enabled */
 117                 if (!(dev->tx_offloads & RTE_ETH_TX_OFFLOAD_SECURITY))
 118                         return 0;
 119
 120                 /* Allocate memory to be used as dptr for CPT ucode
 121                  * WRITE_SA op.
 122                  */
 123                 dev->outb.sa_dptr =
 124                         plt_zmalloc(ROC_NIX_INL_OT_IPSEC_OUTB_HW_SZ, 0);
 125                 if (!dev->outb.sa_dptr) {
 126                         plt_err("Couldn't allocate memory for SA dptr");
 127                         rc = -ENOMEM;
 128                         goto sa_dptr_free;
 129                 }
 130
 131                 rc = -ENOMEM;
 132                 /* Allocate a bitmap to alloc and free sa indexes */
 133                 bmap_sz = plt_bitmap_get_memory_footprint(dev->outb.max_sa);
 134                 mem = plt_zmalloc(bmap_sz, PLT_CACHE_LINE_SIZE);
 135                 if (mem == NULL) {
 136                         plt_err("Outbound SA bmap alloc failed");
 137
 138                         rc |= roc_nix_inl_outb_fini(nix);
 139                         goto sa_dptr_free;
 140                 }
 141
 142                 rc = -EIO;
 143                 bmap = plt_bitmap_init(dev->outb.max_sa, mem, bmap_sz);
 144                 if (!bmap) {
 145                         plt_err("Outbound SA bmap init failed");
 146
 147                         rc |= roc_nix_inl_outb_fini(nix);
 148                         plt_free(mem);
 149                         goto sa_dptr_free;
 150                 }
 151
 152                 for (i = 0; i < dev->outb.max_sa; i++)
 153                         plt_bitmap_set(bmap, i);
 154
 155                 dev->outb.sa_base = roc_nix_inl_outb_sa_base_get(nix);
 156                 dev->outb.sa_bmap_mem = mem;
 157                 dev->outb.sa_bmap = bmap;
 158         }
 159         return 0;
 160
 161 sa_dptr_free:
 162         if (dev->inb.sa_dptr)
 163                 plt_free(dev->inb.sa_dptr);
 164         if (dev->outb.sa_dptr)
 165                 plt_free(dev->outb.sa_dptr);
 166 cleanup:
 167         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY)
 168                 rc |= roc_nix_inl_inb_fini(nix);
 169         return rc;
 170 }
 171
 172 static int
 173 nix_meter_fini(struct cnxk_eth_dev *dev)
 174 {
 175         struct cnxk_meter_node *next_mtr = NULL;
 176         struct roc_nix_bpf_objs profs = {0};
 177         struct cnxk_meter_node *mtr = NULL;
 178         struct cnxk_mtr *fms = &dev->mtr;
 179         struct roc_nix *nix = &dev->nix;
 180         struct roc_nix_rq *rq;
 181         uint32_t i;
 182         int rc = 0;
 183
 184         RTE_TAILQ_FOREACH_SAFE(mtr, fms, next, next_mtr) {
 185                 for (i = 0; i < mtr->rq_num; i++) {
 186                         rq = &dev->rqs[mtr->rq_id[i]];
 187                         rc |= roc_nix_bpf_ena_dis(nix, mtr->bpf_id, rq, false);
 188                 }
 189
 190                 profs.level = mtr->level;
 191                 profs.count = 1;
 192                 profs.ids[0] = mtr->bpf_id;
 193                 rc = roc_nix_bpf_free(nix, &profs, 1);
 194
 195                 if (rc)
 196                         return rc;
 197
 198                 TAILQ_REMOVE(fms, mtr, next);
 199                 plt_free(mtr);
 200         }
 201         return 0;
 202 }
 203
 204 static int
 205 nix_security_release(struct cnxk_eth_dev *dev)
 206 {
 207         struct rte_eth_dev *eth_dev = dev->eth_dev;
 208         struct cnxk_eth_sec_sess *eth_sec, *tvar;
 209         struct roc_nix *nix = &dev->nix;
 210         int rc, ret = 0;
 211
 212         /* Cleanup Inline inbound */
 213         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY) {
 214                 /* Destroy inbound sessions */
 215                 tvar = NULL;
 216                 RTE_TAILQ_FOREACH_SAFE(eth_sec, &dev->inb.list, entry, tvar)
 217                         cnxk_eth_sec_ops.session_destroy(eth_dev,
 218                                                          eth_sec->sess);
 219
 220                 /* Clear lookup mem */
 221                 cnxk_nix_lookup_mem_sa_base_clear(dev);
 222
 223                 rc = roc_nix_inl_inb_fini(nix);
 224                 if (rc)
 225                         plt_err("Failed to cleanup nix inline inb, rc=%d", rc);
 226                 ret |= rc;
 227
 228                 if (dev->inb.sa_dptr) {
 229                         plt_free(dev->inb.sa_dptr);
 230                         dev->inb.sa_dptr = NULL;
 231                 }
 232         }
 233
 234         /* Cleanup Inline outbound */
 235         if (dev->tx_offloads & RTE_ETH_TX_OFFLOAD_SECURITY ||
 236             dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY) {
 237                 /* Destroy outbound sessions */
 238                 tvar = NULL;
 239                 RTE_TAILQ_FOREACH_SAFE(eth_sec, &dev->outb.list, entry, tvar)
 240                         cnxk_eth_sec_ops.session_destroy(eth_dev,
 241                                                          eth_sec->sess);
 242
 243                 rc = roc_nix_inl_outb_fini(nix);
 244                 if (rc)
 245                         plt_err("Failed to cleanup nix inline outb, rc=%d", rc);
 246                 ret |= rc;
 247
 248                 plt_bitmap_free(dev->outb.sa_bmap);
 249                 plt_free(dev->outb.sa_bmap_mem);
 250                 dev->outb.sa_bmap = NULL;
 251                 dev->outb.sa_bmap_mem = NULL;
 252                 if (dev->outb.sa_dptr) {
 253                         plt_free(dev->outb.sa_dptr);
 254                         dev->outb.sa_dptr = NULL;
 255                 }
 256         }
 257
 258         dev->inb.inl_dev = false;
 259         roc_nix_inb_mode_set(nix, false);
 260         dev->nb_rxq_sso = 0;
 261         dev->inb.nb_sess = 0;
 262         dev->outb.nb_sess = 0;
 263         return ret;
 264 }
 265
 266 static void
 267 nix_enable_mseg_on_jumbo(struct cnxk_eth_rxq_sp *rxq)
 268 {
 269         struct rte_pktmbuf_pool_private *mbp_priv;
 270         struct rte_eth_dev *eth_dev;
 271         struct cnxk_eth_dev *dev;
 272         uint32_t buffsz;
 273
 274         dev = rxq->dev;
 275         eth_dev = dev->eth_dev;
 276
 277         /* Get rx buffer size */
 278         mbp_priv = rte_mempool_get_priv(rxq->qconf.mp);
 279         buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
 280
 281         if (eth_dev->data->mtu + (uint32_t)CNXK_NIX_L2_OVERHEAD > buffsz) {
 282                 dev->rx_offloads |= RTE_ETH_RX_OFFLOAD_SCATTER;
 283                 dev->tx_offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
 284         }
 285 }
 286
 287 int
 288 nix_recalc_mtu(struct rte_eth_dev *eth_dev)
 289 {
 290         struct rte_eth_dev_data *data = eth_dev->data;
 291         struct cnxk_eth_rxq_sp *rxq;
 292         int rc;
 293
 294         rxq = ((struct cnxk_eth_rxq_sp *)data->rx_queues[0]) - 1;
 295         /* Setup scatter mode if needed by jumbo */
 296         nix_enable_mseg_on_jumbo(rxq);
 297
 298         rc = cnxk_nix_mtu_set(eth_dev, data->mtu);
 299         if (rc)
 300                 plt_err("Failed to set default MTU size, rc=%d", rc);
 301
 302         return rc;
 303 }
 304
 305 static int
 306 nix_init_flow_ctrl_config(struct rte_eth_dev *eth_dev)
 307 {
 308         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
 309         struct cnxk_fc_cfg *fc = &dev->fc_cfg;
 310         struct rte_eth_fc_conf fc_conf = {0};
 311         int rc;
 312
 313         /* Both Rx & Tx flow ctrl get enabled(RTE_ETH_FC_FULL) in HW
 314          * by AF driver, update those info in PMD structure.
 315          */
 316         rc = cnxk_nix_flow_ctrl_get(eth_dev, &fc_conf);
 317         if (rc)
 318                 goto exit;
 319
 320         fc->mode = fc_conf.mode;
 321         fc->rx_pause = (fc_conf.mode == RTE_ETH_FC_FULL) ||
 322                         (fc_conf.mode == RTE_ETH_FC_RX_PAUSE);
 323         fc->tx_pause = (fc_conf.mode == RTE_ETH_FC_FULL) ||
 324                         (fc_conf.mode == RTE_ETH_FC_TX_PAUSE);
 325
 326 exit:
 327         return rc;
 328 }
 329
 330 static int
 331 nix_update_flow_ctrl_config(struct rte_eth_dev *eth_dev)
 332 {
 333         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
 334         struct cnxk_fc_cfg *fc = &dev->fc_cfg;
 335         struct rte_eth_fc_conf fc_cfg = {0};
 336
 337         if (roc_nix_is_vf_or_sdp(&dev->nix) && !roc_nix_is_lbk(&dev->nix))
 338                 return 0;
 339
 340         fc_cfg.mode = fc->mode;
 341
 342         /* To avoid Link credit deadlock on Ax, disable Tx FC if it's enabled */
 343         if (roc_model_is_cn96_ax() &&
 344             dev->npc.switch_header_type != ROC_PRIV_FLAGS_HIGIG &&
 345             (fc_cfg.mode == RTE_ETH_FC_FULL || fc_cfg.mode == RTE_ETH_FC_RX_PAUSE)) {
 346                 fc_cfg.mode =
 347                                 (fc_cfg.mode == RTE_ETH_FC_FULL ||
 348                                 fc_cfg.mode == RTE_ETH_FC_TX_PAUSE) ?
 349                                 RTE_ETH_FC_TX_PAUSE : RTE_ETH_FC_NONE;
 350         }
 351
 352         return cnxk_nix_flow_ctrl_set(eth_dev, &fc_cfg);
 353 }
 354
 355 uint64_t
 356 cnxk_nix_rxq_mbuf_setup(struct cnxk_eth_dev *dev)
 357 {
 358         uint16_t port_id = dev->eth_dev->data->port_id;
 359         struct rte_mbuf mb_def;
 360         uint64_t *tmp;
 361
 362         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0);
 363         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) -
 364                                  offsetof(struct rte_mbuf, data_off) !=
 365                          2);
 366         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) -
 367                                  offsetof(struct rte_mbuf, data_off) !=
 368                          4);
 369         RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) -
 370                                  offsetof(struct rte_mbuf, data_off) !=
 371                          6);
 372         mb_def.nb_segs = 1;
 373         mb_def.data_off = RTE_PKTMBUF_HEADROOM +
 374                           (dev->ptp_en * CNXK_NIX_TIMESYNC_RX_OFFSET);
 375         mb_def.port = port_id;
 376         rte_mbuf_refcnt_set(&mb_def, 1);
 377
 378         /* Prevent compiler reordering: rearm_data covers previous fields */
 379         rte_compiler_barrier();
 380         tmp = (uint64_t *)&mb_def.rearm_data;
 381
 382         return *tmp;
 383 }
 384
 385 static inline uint8_t
 386 nix_sq_max_sqe_sz(struct cnxk_eth_dev *dev)
 387 {
 388         /*
 389          * Maximum three segments can be supported with W8, Choose
 390          * NIX_MAXSQESZ_W16 for multi segment offload.
 391          */
 392         if (dev->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS)
 393                 return NIX_MAXSQESZ_W16;
 394         else
 395                 return NIX_MAXSQESZ_W8;
 396 }
 397
 398 int
 399 cnxk_nix_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t qid,
 400                         uint16_t nb_desc, uint16_t fp_tx_q_sz,
 401                         const struct rte_eth_txconf *tx_conf)
 402 {
 403         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
 404         const struct eth_dev_ops *dev_ops = eth_dev->dev_ops;
 405         struct cnxk_eth_txq_sp *txq_sp;
 406         struct roc_nix_sq *sq;
 407         size_t txq_sz;
 408         int rc;
 409
 410         /* Free memory prior to re-allocation if needed. */
 411         if (eth_dev->data->tx_queues[qid] != NULL) {
 412                 plt_nix_dbg("Freeing memory prior to re-allocation %d", qid);
 413                 dev_ops->tx_queue_release(eth_dev, qid);
 414                 eth_dev->data->tx_queues[qid] = NULL;
 415         }
 416
 417         /* When Tx Security offload is enabled, increase tx desc count by
 418          * max possible outbound desc count.
 419          */
 420         if (dev->tx_offloads & RTE_ETH_TX_OFFLOAD_SECURITY)
 421                 nb_desc += dev->outb.nb_desc;
 422
 423         /* Setup ROC SQ */
 424         sq = &dev->sqs[qid];
 425         sq->qid = qid;
 426         sq->nb_desc = nb_desc;
 427         sq->max_sqe_sz = nix_sq_max_sqe_sz(dev);
 428
 429         rc = roc_nix_sq_init(&dev->nix, sq);
 430         if (rc) {
 431                 plt_err("Failed to init sq=%d, rc=%d", qid, rc);
 432                 return rc;
 433         }
 434
 435         rc = -ENOMEM;
 436         txq_sz = sizeof(struct cnxk_eth_txq_sp) + fp_tx_q_sz;
 437         txq_sp = plt_zmalloc(txq_sz, PLT_CACHE_LINE_SIZE);
 438         if (!txq_sp) {
 439                 plt_err("Failed to alloc tx queue mem");
 440                 rc |= roc_nix_sq_fini(sq);
 441                 return rc;
 442         }
 443
 444         txq_sp->dev = dev;
 445         txq_sp->qid = qid;
 446         txq_sp->qconf.conf.tx = *tx_conf;
 447         /* Queue config should reflect global offloads */
 448         txq_sp->qconf.conf.tx.offloads = dev->tx_offloads;
 449         txq_sp->qconf.nb_desc = nb_desc;
 450
 451         plt_nix_dbg("sq=%d fc=%p offload=0x%" PRIx64 " lmt_addr=%p"
 452                     " nb_sqb_bufs=%d sqes_per_sqb_log2=%d",
 453                     qid, sq->fc, dev->tx_offloads, sq->lmt_addr,
 454                     sq->nb_sqb_bufs, sq->sqes_per_sqb_log2);
 455
 456         /* Store start of fast path area */
 457         eth_dev->data->tx_queues[qid] = txq_sp + 1;
 458         eth_dev->data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED;
 459         return 0;
 460 }
 461
 462 static void
 463 cnxk_nix_tx_queue_release(struct rte_eth_dev *eth_dev, uint16_t qid)
 464 {
 465         void *txq = eth_dev->data->tx_queues[qid];
 466         struct cnxk_eth_txq_sp *txq_sp;
 467         struct cnxk_eth_dev *dev;
 468         struct roc_nix_sq *sq;
 469         int rc;
 470
 471         if (!txq)
 472                 return;
 473
 474         txq_sp = cnxk_eth_txq_to_sp(txq);
 475
 476         dev = txq_sp->dev;
 477
 478         plt_nix_dbg("Releasing txq %u", qid);
 479
 480         /* Cleanup ROC SQ */
 481         sq = &dev->sqs[qid];
 482         rc = roc_nix_sq_fini(sq);
 483         if (rc)
 484                 plt_err("Failed to cleanup sq, rc=%d", rc);
 485
 486         /* Finally free */
 487         plt_free(txq_sp);
 488 }
 489
 490 int
 491 cnxk_nix_rx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t qid,
 492                         uint16_t nb_desc, uint16_t fp_rx_q_sz,
 493                         const struct rte_eth_rxconf *rx_conf,
 494                         struct rte_mempool *mp)
 495 {
 496         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
 497         struct roc_nix *nix = &dev->nix;
 498         struct cnxk_eth_rxq_sp *rxq_sp;
 499         struct rte_mempool_ops *ops;
 500         const char *platform_ops;
 501         struct roc_nix_rq *rq;
 502         struct roc_nix_cq *cq;
 503         uint16_t first_skip;
 504         int rc = -EINVAL;
 505         size_t rxq_sz;
 506
 507         /* Sanity checks */
 508         if (rx_conf->rx_deferred_start == 1) {
 509                 plt_err("Deferred Rx start is not supported");
 510                 goto fail;
 511         }
 512
 513         platform_ops = rte_mbuf_platform_mempool_ops();
 514         /* This driver needs cnxk_npa mempool ops to work */
 515         ops = rte_mempool_get_ops(mp->ops_index);
 516         if (strncmp(ops->name, platform_ops, RTE_MEMPOOL_OPS_NAMESIZE)) {
 517                 plt_err("mempool ops should be of cnxk_npa type");
 518                 goto fail;
 519         }
 520
 521         if (mp->pool_id == 0) {
 522                 plt_err("Invalid pool_id");
 523                 goto fail;
 524         }
 525
 526         /* Free memory prior to re-allocation if needed */
 527         if (eth_dev->data->rx_queues[qid] != NULL) {
 528                 const struct eth_dev_ops *dev_ops = eth_dev->dev_ops;
 529
 530                 plt_nix_dbg("Freeing memory prior to re-allocation %d", qid);
 531                 dev_ops->rx_queue_release(eth_dev, qid);
 532                 eth_dev->data->rx_queues[qid] = NULL;
 533         }
 534
 535         /* Clam up cq limit to size of packet pool aura for LBK
 536          * to avoid meta packet drop as LBK does not currently support
 537          * backpressure.
 538          */
 539         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY && roc_nix_is_lbk(nix)) {
 540                 uint64_t pkt_pool_limit = roc_nix_inl_dev_rq_limit_get();
 541
 542                 /* Use current RQ's aura limit if inl rq is not available */
 543                 if (!pkt_pool_limit)
 544                         pkt_pool_limit = roc_npa_aura_op_limit_get(mp->pool_id);
 545                 nb_desc = RTE_MAX(nb_desc, pkt_pool_limit);
 546         }
 547
 548         /* Its a no-op when inline device is not used */
 549         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY ||
 550             dev->tx_offloads & RTE_ETH_TX_OFFLOAD_SECURITY)
 551                 roc_nix_inl_dev_xaq_realloc(mp->pool_id);
 552
 553         /* Setup ROC CQ */
 554         cq = &dev->cqs[qid];
 555         cq->qid = qid;
 556         cq->nb_desc = nb_desc;
 557         rc = roc_nix_cq_init(&dev->nix, cq);
 558         if (rc) {
 559                 plt_err("Failed to init roc cq for rq=%d, rc=%d", qid, rc);
 560                 goto fail;
 561         }
 562
 563         /* Setup ROC RQ */
 564         rq = &dev->rqs[qid];
 565         rq->qid = qid;
 566         rq->aura_handle = mp->pool_id;
 567         rq->flow_tag_width = 32;
 568         rq->sso_ena = false;
 569
 570         /* Calculate first mbuf skip */
 571         first_skip = (sizeof(struct rte_mbuf));
 572         first_skip += RTE_PKTMBUF_HEADROOM;
 573         first_skip += rte_pktmbuf_priv_size(mp);
 574         rq->first_skip = first_skip;
 575         rq->later_skip = sizeof(struct rte_mbuf);
 576         rq->lpb_size = mp->elt_size;
 577
 578         /* Enable Inline IPSec on RQ, will not be used for Poll mode */
 579         if (roc_nix_inl_inb_is_enabled(nix))
 580                 rq->ipsech_ena = true;
 581
 582         rc = roc_nix_rq_init(&dev->nix, rq, !!eth_dev->data->dev_started);
 583         if (rc) {
 584                 plt_err("Failed to init roc rq for rq=%d, rc=%d", qid, rc);
 585                 goto cq_fini;
 586         }
 587
 588         /* Allocate and setup fast path rx queue */
 589         rc = -ENOMEM;
 590         rxq_sz = sizeof(struct cnxk_eth_rxq_sp) + fp_rx_q_sz;
 591         rxq_sp = plt_zmalloc(rxq_sz, PLT_CACHE_LINE_SIZE);
 592         if (!rxq_sp) {
 593                 plt_err("Failed to alloc rx queue for rq=%d", qid);
 594                 goto rq_fini;
 595         }
 596
 597         /* Setup slow path fields */
 598         rxq_sp->dev = dev;
 599         rxq_sp->qid = qid;
 600         rxq_sp->qconf.conf.rx = *rx_conf;
 601         /* Queue config should reflect global offloads */
 602         rxq_sp->qconf.conf.rx.offloads = dev->rx_offloads;
 603         rxq_sp->qconf.nb_desc = nb_desc;
 604         rxq_sp->qconf.mp = mp;
 605
 606         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY) {
 607                 /* Pass a tagmask used to handle error packets in inline device.
 608                  * Ethdev rq's tag_mask field will be overwritten later
 609                  * when sso is setup.
 610                  */
 611                 rq->tag_mask =
 612                         0x0FF00000 | ((uint32_t)RTE_EVENT_TYPE_ETHDEV << 28);
 613
 614                 /* Setup rq reference for inline dev if present */
 615                 rc = roc_nix_inl_dev_rq_get(rq);
 616                 if (rc)
 617                         goto free_mem;
 618         }
 619
 620         plt_nix_dbg("rq=%d pool=%s nb_desc=%d->%d", qid, mp->name, nb_desc,
 621                     cq->nb_desc);
 622
 623         /* Store start of fast path area */
 624         eth_dev->data->rx_queues[qid] = rxq_sp + 1;
 625         eth_dev->data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED;
 626
 627         /* Calculating delta and freq mult between PTP HI clock and tsc.
 628          * These are needed in deriving raw clock value from tsc counter.
 629          * read_clock eth op returns raw clock value.
 630          */
 631         if ((dev->rx_offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) || dev->ptp_en) {
 632                 rc = cnxk_nix_tsc_convert(dev);
 633                 if (rc) {
 634                         plt_err("Failed to calculate delta and freq mult");
 635                         goto rq_fini;
 636                 }
 637         }
 638
 639         return 0;
 640 free_mem:
 641         plt_free(rxq_sp);
 642 rq_fini:
 643         rc |= roc_nix_rq_fini(rq);
 644 cq_fini:
 645         rc |= roc_nix_cq_fini(cq);
 646 fail:
 647         return rc;
 648 }
 649
 650 static void
 651 cnxk_nix_rx_queue_release(struct rte_eth_dev *eth_dev, uint16_t qid)
 652 {
 653         void *rxq = eth_dev->data->rx_queues[qid];
 654         struct cnxk_eth_rxq_sp *rxq_sp;
 655         struct cnxk_eth_dev *dev;
 656         struct roc_nix_rq *rq;
 657         struct roc_nix_cq *cq;
 658         int rc;
 659
 660         if (!rxq)
 661                 return;
 662
 663         rxq_sp = cnxk_eth_rxq_to_sp(rxq);
 664         dev = rxq_sp->dev;
 665         rq = &dev->rqs[qid];
 666
 667         plt_nix_dbg("Releasing rxq %u", qid);
 668
 669         /* Release rq reference for inline dev if present */
 670         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY)
 671                 roc_nix_inl_dev_rq_put(rq);
 672
 673         /* Cleanup ROC RQ */
 674         rc = roc_nix_rq_fini(rq);
 675         if (rc)
 676                 plt_err("Failed to cleanup rq, rc=%d", rc);
 677
 678         /* Cleanup ROC CQ */
 679         cq = &dev->cqs[qid];
 680         rc = roc_nix_cq_fini(cq);
 681         if (rc)
 682                 plt_err("Failed to cleanup cq, rc=%d", rc);
 683
 684         /* Finally free fast path area */
 685         plt_free(rxq_sp);
 686 }
 687
 688 uint32_t
 689 cnxk_rss_ethdev_to_nix(struct cnxk_eth_dev *dev, uint64_t ethdev_rss,
 690                        uint8_t rss_level)
 691 {
 692         uint32_t flow_key_type[RSS_MAX_LEVELS][6] = {
 693                 {FLOW_KEY_TYPE_IPV4, FLOW_KEY_TYPE_IPV6, FLOW_KEY_TYPE_TCP,
 694                  FLOW_KEY_TYPE_UDP, FLOW_KEY_TYPE_SCTP, FLOW_KEY_TYPE_ETH_DMAC},
 695                 {FLOW_KEY_TYPE_INNR_IPV4, FLOW_KEY_TYPE_INNR_IPV6,
 696                  FLOW_KEY_TYPE_INNR_TCP, FLOW_KEY_TYPE_INNR_UDP,
 697                  FLOW_KEY_TYPE_INNR_SCTP, FLOW_KEY_TYPE_INNR_ETH_DMAC},
 698                 {FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_INNR_IPV4,
 699                  FLOW_KEY_TYPE_IPV6 | FLOW_KEY_TYPE_INNR_IPV6,
 700                  FLOW_KEY_TYPE_TCP | FLOW_KEY_TYPE_INNR_TCP,
 701                  FLOW_KEY_TYPE_UDP | FLOW_KEY_TYPE_INNR_UDP,
 702                  FLOW_KEY_TYPE_SCTP | FLOW_KEY_TYPE_INNR_SCTP,
 703                  FLOW_KEY_TYPE_ETH_DMAC | FLOW_KEY_TYPE_INNR_ETH_DMAC}
 704         };
 705         uint32_t flowkey_cfg = 0;
 706
 707         dev->ethdev_rss_hf = ethdev_rss;
 708
 709         if (ethdev_rss & RTE_ETH_RSS_L2_PAYLOAD &&
 710             dev->npc.switch_header_type == ROC_PRIV_FLAGS_LEN_90B) {
 711                 flowkey_cfg |= FLOW_KEY_TYPE_CH_LEN_90B;
 712         }
 713
 714         if (ethdev_rss & RTE_ETH_RSS_C_VLAN)
 715                 flowkey_cfg |= FLOW_KEY_TYPE_VLAN;
 716
 717         if (ethdev_rss & RTE_ETH_RSS_L3_SRC_ONLY)
 718                 flowkey_cfg |= FLOW_KEY_TYPE_L3_SRC;
 719
 720         if (ethdev_rss & RTE_ETH_RSS_L3_DST_ONLY)
 721                 flowkey_cfg |= FLOW_KEY_TYPE_L3_DST;
 722
 723         if (ethdev_rss & RTE_ETH_RSS_L4_SRC_ONLY)
 724                 flowkey_cfg |= FLOW_KEY_TYPE_L4_SRC;
 725
 726         if (ethdev_rss & RTE_ETH_RSS_L4_DST_ONLY)
 727                 flowkey_cfg |= FLOW_KEY_TYPE_L4_DST;
 728
 729         if (ethdev_rss & RSS_IPV4_ENABLE)
 730                 flowkey_cfg |= flow_key_type[rss_level][RSS_IPV4_INDEX];
 731
 732         if (ethdev_rss & RSS_IPV6_ENABLE)
 733                 flowkey_cfg |= flow_key_type[rss_level][RSS_IPV6_INDEX];
 734
 735         if (ethdev_rss & RTE_ETH_RSS_TCP)
 736                 flowkey_cfg |= flow_key_type[rss_level][RSS_TCP_INDEX];
 737
 738         if (ethdev_rss & RTE_ETH_RSS_UDP)
 739                 flowkey_cfg |= flow_key_type[rss_level][RSS_UDP_INDEX];
 740
 741         if (ethdev_rss & RTE_ETH_RSS_SCTP)
 742                 flowkey_cfg |= flow_key_type[rss_level][RSS_SCTP_INDEX];
 743
 744         if (ethdev_rss & RTE_ETH_RSS_L2_PAYLOAD)
 745                 flowkey_cfg |= flow_key_type[rss_level][RSS_DMAC_INDEX];
 746
 747         if (ethdev_rss & RSS_IPV6_EX_ENABLE)
 748                 flowkey_cfg |= FLOW_KEY_TYPE_IPV6_EXT;
 749
 750         if (ethdev_rss & RTE_ETH_RSS_PORT)
 751                 flowkey_cfg |= FLOW_KEY_TYPE_PORT;
 752
 753         if (ethdev_rss & RTE_ETH_RSS_NVGRE)
 754                 flowkey_cfg |= FLOW_KEY_TYPE_NVGRE;
 755
 756         if (ethdev_rss & RTE_ETH_RSS_VXLAN)
 757                 flowkey_cfg |= FLOW_KEY_TYPE_VXLAN;
 758
 759         if (ethdev_rss & RTE_ETH_RSS_GENEVE)
 760                 flowkey_cfg |= FLOW_KEY_TYPE_GENEVE;
 761
 762         if (ethdev_rss & RTE_ETH_RSS_GTPU)
 763                 flowkey_cfg |= FLOW_KEY_TYPE_GTPU;
 764
 765         return flowkey_cfg;
 766 }
 767
 768 static void
 769 nix_free_queue_mem(struct cnxk_eth_dev *dev)
 770 {
 771         plt_free(dev->rqs);
 772         plt_free(dev->cqs);
 773         plt_free(dev->sqs);
 774         dev->rqs = NULL;
 775         dev->cqs = NULL;
 776         dev->sqs = NULL;
 777 }
 778
 779 static int
 780 nix_ingress_policer_setup(struct cnxk_eth_dev *dev)
 781 {
 782         struct rte_eth_dev *eth_dev = dev->eth_dev;
 783         int rc = 0;
 784
 785         TAILQ_INIT(&dev->mtr_profiles);
 786         TAILQ_INIT(&dev->mtr_policy);
 787         TAILQ_INIT(&dev->mtr);
 788
 789         if (eth_dev->dev_ops->mtr_ops_get == NULL)
 790                 return rc;
 791
 792         return nix_mtr_capabilities_init(eth_dev);
 793 }
 794
 795 static int
 796 nix_rss_default_setup(struct cnxk_eth_dev *dev)
 797 {
 798         struct rte_eth_dev *eth_dev = dev->eth_dev;
 799         uint8_t rss_hash_level;
 800         uint32_t flowkey_cfg;
 801         uint64_t rss_hf;
 802
 803         rss_hf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
 804         rss_hash_level = RTE_ETH_RSS_LEVEL(rss_hf);
 805         if (rss_hash_level)
 806                 rss_hash_level -= 1;
 807
 808         flowkey_cfg = cnxk_rss_ethdev_to_nix(dev, rss_hf, rss_hash_level);
 809         return roc_nix_rss_default_setup(&dev->nix, flowkey_cfg);
 810 }
 811
 812 static int
 813 nix_store_queue_cfg_and_then_release(struct rte_eth_dev *eth_dev)
 814 {
 815         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
 816         const struct eth_dev_ops *dev_ops = eth_dev->dev_ops;
 817         struct cnxk_eth_qconf *tx_qconf = NULL;
 818         struct cnxk_eth_qconf *rx_qconf = NULL;
 819         struct cnxk_eth_rxq_sp *rxq_sp;
 820         struct cnxk_eth_txq_sp *txq_sp;
 821         int i, nb_rxq, nb_txq;
 822         void **txq, **rxq;
 823
 824         nb_rxq = RTE_MIN(dev->nb_rxq, eth_dev->data->nb_rx_queues);
 825         nb_txq = RTE_MIN(dev->nb_txq, eth_dev->data->nb_tx_queues);
 826
 827         tx_qconf = malloc(nb_txq * sizeof(*tx_qconf));
 828         if (tx_qconf == NULL) {
 829                 plt_err("Failed to allocate memory for tx_qconf");
 830                 goto fail;
 831         }
 832
 833         rx_qconf = malloc(nb_rxq * sizeof(*rx_qconf));
 834         if (rx_qconf == NULL) {
 835                 plt_err("Failed to allocate memory for rx_qconf");
 836                 goto fail;
 837         }
 838
 839         txq = eth_dev->data->tx_queues;
 840         for (i = 0; i < nb_txq; i++) {
 841                 if (txq[i] == NULL) {
 842                         tx_qconf[i].valid = false;
 843                         plt_info("txq[%d] is already released", i);
 844                         continue;
 845                 }
 846                 txq_sp = cnxk_eth_txq_to_sp(txq[i]);
 847                 memcpy(&tx_qconf[i], &txq_sp->qconf, sizeof(*tx_qconf));
 848                 tx_qconf[i].valid = true;
 849                 dev_ops->tx_queue_release(eth_dev, i);
 850                 eth_dev->data->tx_queues[i] = NULL;
 851         }
 852
 853         rxq = eth_dev->data->rx_queues;
 854         for (i = 0; i < nb_rxq; i++) {
 855                 if (rxq[i] == NULL) {
 856                         rx_qconf[i].valid = false;
 857                         plt_info("rxq[%d] is already released", i);
 858                         continue;
 859                 }
 860                 rxq_sp = cnxk_eth_rxq_to_sp(rxq[i]);
 861                 memcpy(&rx_qconf[i], &rxq_sp->qconf, sizeof(*rx_qconf));
 862                 rx_qconf[i].valid = true;
 863                 dev_ops->rx_queue_release(eth_dev, i);
 864                 eth_dev->data->rx_queues[i] = NULL;
 865         }
 866
 867         dev->tx_qconf = tx_qconf;
 868         dev->rx_qconf = rx_qconf;
 869         return 0;
 870
 871 fail:
 872         free(tx_qconf);
 873         free(rx_qconf);
 874         return -ENOMEM;
 875 }
 876
 877 static int
 878 nix_restore_queue_cfg(struct rte_eth_dev *eth_dev)
 879 {
 880         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
 881         const struct eth_dev_ops *dev_ops = eth_dev->dev_ops;
 882         struct cnxk_eth_qconf *tx_qconf = dev->tx_qconf;
 883         struct cnxk_eth_qconf *rx_qconf = dev->rx_qconf;
 884         int rc, i, nb_rxq, nb_txq;
 885
 886         nb_rxq = RTE_MIN(dev->nb_rxq, eth_dev->data->nb_rx_queues);
 887         nb_txq = RTE_MIN(dev->nb_txq, eth_dev->data->nb_tx_queues);
 888
 889         rc = -ENOMEM;
 890         /* Setup tx & rx queues with previous configuration so
 891          * that the queues can be functional in cases like ports
 892          * are started without re configuring queues.
 893          *
 894          * Usual re config sequence is like below:
 895          * port_configure() {
 896          *      if(reconfigure) {
 897          *              queue_release()
 898          *              queue_setup()
 899          *      }
 900          *      queue_configure() {
 901          *              queue_release()
 902          *              queue_setup()
 903          *      }
 904          * }
 905          * port_start()
 906          *
 907          * In some application's control path, queue_configure() would
 908          * NOT be invoked for TXQs/RXQs in port_configure().
 909          * In such cases, queues can be functional after start as the
 910          * queues are already setup in port_configure().
 911          */
 912         for (i = 0; i < nb_txq; i++) {
 913                 if (!tx_qconf[i].valid)
 914                         continue;
 915                 rc = dev_ops->tx_queue_setup(eth_dev, i, tx_qconf[i].nb_desc, 0,
 916                                              &tx_qconf[i].conf.tx);
 917                 if (rc) {
 918                         plt_err("Failed to setup tx queue rc=%d", rc);
 919                         for (i -= 1; i >= 0; i--)
 920                                 dev_ops->tx_queue_release(eth_dev, i);
 921                         goto fail;
 922                 }
 923         }
 924
 925         free(tx_qconf);
 926         tx_qconf = NULL;
 927
 928         for (i = 0; i < nb_rxq; i++) {
 929                 if (!rx_qconf[i].valid)
 930                         continue;
 931                 rc = dev_ops->rx_queue_setup(eth_dev, i, rx_qconf[i].nb_desc, 0,
 932                                              &rx_qconf[i].conf.rx,
 933                                              rx_qconf[i].mp);
 934                 if (rc) {
 935                         plt_err("Failed to setup rx queue rc=%d", rc);
 936                         for (i -= 1; i >= 0; i--)
 937                                 dev_ops->rx_queue_release(eth_dev, i);
 938                         goto tx_queue_release;
 939                 }
 940         }
 941
 942         free(rx_qconf);
 943         rx_qconf = NULL;
 944
 945         return 0;
 946
 947 tx_queue_release:
 948         for (i = 0; i < eth_dev->data->nb_tx_queues; i++)
 949                 dev_ops->tx_queue_release(eth_dev, i);
 950 fail:
 951         free(tx_qconf);
 952         free(rx_qconf);
 953
 954         return rc;
 955 }
 956
 957 static void
 958 nix_set_nop_rxtx_function(struct rte_eth_dev *eth_dev)
 959 {
 960         /* These dummy functions are required for supporting
 961          * some applications which reconfigure queues without
 962          * stopping tx burst and rx burst threads(eg kni app)
 963          * When the queues context is saved, txq/rxqs are released
 964          * which caused app crash since rx/tx burst is still
 965          * on different lcores
 966          */
 967         eth_dev->tx_pkt_burst = rte_eth_pkt_burst_dummy;
 968         eth_dev->rx_pkt_burst = rte_eth_pkt_burst_dummy;
 969         rte_mb();
 970 }
 971
 972 static int
 973 nix_lso_tun_fmt_update(struct cnxk_eth_dev *dev)
 974 {
 975         uint8_t udp_tun[ROC_NIX_LSO_TUN_MAX];
 976         uint8_t tun[ROC_NIX_LSO_TUN_MAX];
 977         struct roc_nix *nix = &dev->nix;
 978         int rc;
 979
 980         rc = roc_nix_lso_fmt_get(nix, udp_tun, tun);
 981         if (rc)
 982                 return rc;
 983
 984         dev->lso_tun_fmt = ((uint64_t)tun[ROC_NIX_LSO_TUN_V4V4] |
 985                             (uint64_t)tun[ROC_NIX_LSO_TUN_V4V6] << 8 |
 986                             (uint64_t)tun[ROC_NIX_LSO_TUN_V6V4] << 16 |
 987                             (uint64_t)tun[ROC_NIX_LSO_TUN_V6V6] << 24);
 988
 989         dev->lso_tun_fmt |= ((uint64_t)udp_tun[ROC_NIX_LSO_TUN_V4V4] << 32 |
 990                              (uint64_t)udp_tun[ROC_NIX_LSO_TUN_V4V6] << 40 |
 991                              (uint64_t)udp_tun[ROC_NIX_LSO_TUN_V6V4] << 48 |
 992                              (uint64_t)udp_tun[ROC_NIX_LSO_TUN_V6V6] << 56);
 993         return 0;
 994 }
 995
 996 static int
 997 nix_lso_fmt_setup(struct cnxk_eth_dev *dev)
 998 {
 999         struct roc_nix *nix = &dev->nix;
1000         int rc;
1001
1002         /* Nothing much to do if offload is not enabled */
1003         if (!(dev->tx_offloads &
1004               (RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1005                RTE_ETH_TX_OFFLOAD_GENEVE_TNL_TSO | RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO)))
1006                 return 0;
1007
1008         /* Setup LSO formats in AF. Its a no-op if other ethdev has
1009          * already set it up
1010          */
1011         rc = roc_nix_lso_fmt_setup(nix);
1012         if (rc)
1013                 return rc;
1014
1015         return nix_lso_tun_fmt_update(dev);
1016 }
1017
1018 int
1019 cnxk_nix_configure(struct rte_eth_dev *eth_dev)
1020 {
1021         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1022         struct rte_eth_dev_data *data = eth_dev->data;
1023         struct rte_eth_conf *conf = &data->dev_conf;
1024         struct rte_eth_rxmode *rxmode = &conf->rxmode;
1025         struct rte_eth_txmode *txmode = &conf->txmode;
1026         char ea_fmt[RTE_ETHER_ADDR_FMT_SIZE];
1027         struct roc_nix_fc_cfg fc_cfg = {0};
1028         struct roc_nix *nix = &dev->nix;
1029         struct rte_ether_addr *ea;
1030         uint8_t nb_rxq, nb_txq;
1031         uint64_t rx_cfg;
1032         void *qs;
1033         int rc;
1034
1035         rc = -EINVAL;
1036
1037         /* Sanity checks */
1038         if (rte_eal_has_hugepages() == 0) {
1039                 plt_err("Huge page is not configured");
1040                 goto fail_configure;
1041         }
1042
1043         if (conf->dcb_capability_en == 1) {
1044                 plt_err("dcb enable is not supported");
1045                 goto fail_configure;
1046         }
1047
1048         if (conf->fdir_conf.mode != RTE_FDIR_MODE_NONE) {
1049                 plt_err("Flow director is not supported");
1050                 goto fail_configure;
1051         }
1052
1053         if (rxmode->mq_mode != RTE_ETH_MQ_RX_NONE &&
1054             rxmode->mq_mode != RTE_ETH_MQ_RX_RSS) {
1055                 plt_err("Unsupported mq rx mode %d", rxmode->mq_mode);
1056                 goto fail_configure;
1057         }
1058
1059         if (txmode->mq_mode != RTE_ETH_MQ_TX_NONE) {
1060                 plt_err("Unsupported mq tx mode %d", txmode->mq_mode);
1061                 goto fail_configure;
1062         }
1063
1064         /* Free the resources allocated from the previous configure */
1065         if (dev->configured == 1) {
1066                 /* Unregister queue irq's */
1067                 roc_nix_unregister_queue_irqs(nix);
1068
1069                 /* Unregister CQ irqs if present */
1070                 if (eth_dev->data->dev_conf.intr_conf.rxq)
1071                         roc_nix_unregister_cq_irqs(nix);
1072
1073                 /* Set no-op functions */
1074                 nix_set_nop_rxtx_function(eth_dev);
1075                 /* Store queue config for later */
1076                 rc = nix_store_queue_cfg_and_then_release(eth_dev);
1077                 if (rc)
1078                         goto fail_configure;
1079
1080                 /* Disable and free rte_meter entries */
1081                 rc = nix_meter_fini(dev);
1082                 if (rc)
1083                         goto fail_configure;
1084
1085                 /* Cleanup security support */
1086                 rc = nix_security_release(dev);
1087                 if (rc)
1088                         goto fail_configure;
1089
1090                 roc_nix_tm_fini(nix);
1091                 roc_nix_lf_free(nix);
1092         }
1093
1094         dev->rx_offloads = rxmode->offloads;
1095         dev->tx_offloads = txmode->offloads;
1096
1097         /* Prepare rx cfg */
1098         rx_cfg = ROC_NIX_LF_RX_CFG_DIS_APAD;
1099         if (dev->rx_offloads &
1100             (RTE_ETH_RX_OFFLOAD_TCP_CKSUM | RTE_ETH_RX_OFFLOAD_UDP_CKSUM)) {
1101                 rx_cfg |= ROC_NIX_LF_RX_CFG_CSUM_OL4;
1102                 rx_cfg |= ROC_NIX_LF_RX_CFG_CSUM_IL4;
1103         }
1104         rx_cfg |= (ROC_NIX_LF_RX_CFG_DROP_RE | ROC_NIX_LF_RX_CFG_L2_LEN_ERR |
1105                    ROC_NIX_LF_RX_CFG_LEN_IL4 | ROC_NIX_LF_RX_CFG_LEN_IL3 |
1106                    ROC_NIX_LF_RX_CFG_LEN_OL4 | ROC_NIX_LF_RX_CFG_LEN_OL3);
1107
1108         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY) {
1109                 rx_cfg |= ROC_NIX_LF_RX_CFG_IP6_UDP_OPT;
1110                 /* Disable drop re if rx offload security is enabled and
1111                  * platform does not support it.
1112                  */
1113                 if (dev->ipsecd_drop_re_dis)
1114                         rx_cfg &= ~(ROC_NIX_LF_RX_CFG_DROP_RE);
1115         }
1116
1117         nb_rxq = RTE_MAX(data->nb_rx_queues, 1);
1118         nb_txq = RTE_MAX(data->nb_tx_queues, 1);
1119
1120         /* Alloc a nix lf */
1121         rc = roc_nix_lf_alloc(nix, nb_rxq, nb_txq, rx_cfg);
1122         if (rc) {
1123                 plt_err("Failed to init nix_lf rc=%d", rc);
1124                 goto fail_configure;
1125         }
1126
1127         dev->npc.channel = roc_nix_get_base_chan(nix);
1128
1129         nb_rxq = data->nb_rx_queues;
1130         nb_txq = data->nb_tx_queues;
1131         rc = -ENOMEM;
1132         if (nb_rxq) {
1133                 /* Allocate memory for roc rq's and cq's */
1134                 qs = plt_zmalloc(sizeof(struct roc_nix_rq) * nb_rxq, 0);
1135                 if (!qs) {
1136                         plt_err("Failed to alloc rqs");
1137                         goto free_nix_lf;
1138                 }
1139                 dev->rqs = qs;
1140
1141                 qs = plt_zmalloc(sizeof(struct roc_nix_cq) * nb_rxq, 0);
1142                 if (!qs) {
1143                         plt_err("Failed to alloc cqs");
1144                         goto free_nix_lf;
1145                 }
1146                 dev->cqs = qs;
1147         }
1148
1149         if (nb_txq) {
1150                 /* Allocate memory for roc sq's */
1151                 qs = plt_zmalloc(sizeof(struct roc_nix_sq) * nb_txq, 0);
1152                 if (!qs) {
1153                         plt_err("Failed to alloc sqs");
1154                         goto free_nix_lf;
1155                 }
1156                 dev->sqs = qs;
1157         }
1158
1159         /* Re-enable NIX LF error interrupts */
1160         roc_nix_err_intr_ena_dis(nix, true);
1161         roc_nix_ras_intr_ena_dis(nix, true);
1162
1163         if (nix->rx_ptp_ena &&
1164             dev->npc.switch_header_type == ROC_PRIV_FLAGS_HIGIG) {
1165                 plt_err("Both PTP and switch header enabled");
1166                 goto free_nix_lf;
1167         }
1168
1169         rc = roc_nix_switch_hdr_set(nix, dev->npc.switch_header_type,
1170                                     dev->npc.pre_l2_size_offset,
1171                                     dev->npc.pre_l2_size_offset_mask,
1172                                     dev->npc.pre_l2_size_shift_dir);
1173         if (rc) {
1174                 plt_err("Failed to enable switch type nix_lf rc=%d", rc);
1175                 goto free_nix_lf;
1176         }
1177
1178         /* Setup LSO if needed */
1179         rc = nix_lso_fmt_setup(dev);
1180         if (rc) {
1181                 plt_err("Failed to setup nix lso format fields, rc=%d", rc);
1182                 goto free_nix_lf;
1183         }
1184
1185         /* Configure RSS */
1186         rc = nix_rss_default_setup(dev);
1187         if (rc) {
1188                 plt_err("Failed to configure rss rc=%d", rc);
1189                 goto free_nix_lf;
1190         }
1191
1192         /* Init the default TM scheduler hierarchy */
1193         rc = roc_nix_tm_init(nix);
1194         if (rc) {
1195                 plt_err("Failed to init traffic manager, rc=%d", rc);
1196                 goto free_nix_lf;
1197         }
1198
1199         rc = nix_ingress_policer_setup(dev);
1200         if (rc) {
1201                 plt_err("Failed to setup ingress policer rc=%d", rc);
1202                 goto free_nix_lf;
1203         }
1204
1205         rc = roc_nix_tm_hierarchy_enable(nix, ROC_NIX_TM_DEFAULT, false);
1206         if (rc) {
1207                 plt_err("Failed to enable default tm hierarchy, rc=%d", rc);
1208                 goto tm_fini;
1209         }
1210
1211         /* Register queue IRQs */
1212         rc = roc_nix_register_queue_irqs(nix);
1213         if (rc) {
1214                 plt_err("Failed to register queue interrupts rc=%d", rc);
1215                 goto tm_fini;
1216         }
1217
1218         /* Register cq IRQs */
1219         if (eth_dev->data->dev_conf.intr_conf.rxq) {
1220                 if (eth_dev->data->nb_rx_queues > dev->nix.cints) {
1221                         plt_err("Rx interrupt cannot be enabled, rxq > %d",
1222                                 dev->nix.cints);
1223                         goto q_irq_fini;
1224                 }
1225                 /* Rx interrupt feature cannot work with vector mode because,
1226                  * vector mode does not process packets unless min 4 pkts are
1227                  * received, while cq interrupts are generated even for 1 pkt
1228                  * in the CQ.
1229                  */
1230                 dev->scalar_ena = true;
1231
1232                 rc = roc_nix_register_cq_irqs(nix);
1233                 if (rc) {
1234                         plt_err("Failed to register CQ interrupts rc=%d", rc);
1235                         goto q_irq_fini;
1236                 }
1237         }
1238
1239         /* Configure loop back mode */
1240         rc = roc_nix_mac_loopback_enable(nix,
1241                                          eth_dev->data->dev_conf.lpbk_mode);
1242         if (rc) {
1243                 plt_err("Failed to configure cgx loop back mode rc=%d", rc);
1244                 goto cq_fini;
1245         }
1246
1247         /* Setup Inline security support */
1248         rc = nix_security_setup(dev);
1249         if (rc)
1250                 goto cq_fini;
1251
1252         /* Init flow control configuration */
1253         fc_cfg.type = ROC_NIX_FC_RXCHAN_CFG;
1254         fc_cfg.rxchan_cfg.enable = true;
1255         rc = roc_nix_fc_config_set(nix, &fc_cfg);
1256         if (rc) {
1257                 plt_err("Failed to initialize flow control rc=%d", rc);
1258                 goto cq_fini;
1259         }
1260
1261         /* Update flow control configuration to PMD */
1262         rc = nix_init_flow_ctrl_config(eth_dev);
1263         if (rc) {
1264                 plt_err("Failed to initialize flow control rc=%d", rc);
1265                 goto cq_fini;
1266         }
1267
1268         /* Initialize TC to SQ mapping as invalid */
1269         memset(dev->pfc_tc_sq_map, 0xFF, sizeof(dev->pfc_tc_sq_map));
1270         /*
1271          * Restore queue config when reconfigure followed by
1272          * reconfigure and no queue configure invoked from application case.
1273          */
1274         if (dev->configured == 1) {
1275                 rc = nix_restore_queue_cfg(eth_dev);
1276                 if (rc)
1277                         goto sec_release;
1278         }
1279
1280         /* Update the mac address */
1281         ea = eth_dev->data->mac_addrs;
1282         memcpy(ea, dev->mac_addr, RTE_ETHER_ADDR_LEN);
1283         if (rte_is_zero_ether_addr(ea))
1284                 rte_eth_random_addr((uint8_t *)ea);
1285
1286         rte_ether_format_addr(ea_fmt, RTE_ETHER_ADDR_FMT_SIZE, ea);
1287
1288         plt_nix_dbg("Configured port%d mac=%s nb_rxq=%d nb_txq=%d"
1289                     " rx_offloads=0x%" PRIx64 " tx_offloads=0x%" PRIx64 "",
1290                     eth_dev->data->port_id, ea_fmt, nb_rxq, nb_txq,
1291                     dev->rx_offloads, dev->tx_offloads);
1292
1293         /* All good */
1294         dev->configured = 1;
1295         dev->nb_rxq = data->nb_rx_queues;
1296         dev->nb_txq = data->nb_tx_queues;
1297         return 0;
1298
1299 sec_release:
1300         rc |= nix_security_release(dev);
1301 cq_fini:
1302         roc_nix_unregister_cq_irqs(nix);
1303 q_irq_fini:
1304         roc_nix_unregister_queue_irqs(nix);
1305 tm_fini:
1306         roc_nix_tm_fini(nix);
1307 free_nix_lf:
1308         nix_free_queue_mem(dev);
1309         rc |= roc_nix_lf_free(nix);
1310 fail_configure:
1311         dev->configured = 0;
1312         return rc;
1313 }
1314
1315 int
1316 cnxk_nix_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t qid)
1317 {
1318         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1319         struct rte_eth_dev_data *data = eth_dev->data;
1320         struct roc_nix_sq *sq = &dev->sqs[qid];
1321         int rc = -EINVAL;
1322
1323         if (data->tx_queue_state[qid] == RTE_ETH_QUEUE_STATE_STARTED)
1324                 return 0;
1325
1326         rc = roc_nix_tm_sq_aura_fc(sq, true);
1327         if (rc) {
1328                 plt_err("Failed to enable sq aura fc, txq=%u, rc=%d", qid, rc);
1329                 goto done;
1330         }
1331
1332         data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STARTED;
1333 done:
1334         return rc;
1335 }
1336
1337 int
1338 cnxk_nix_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t qid)
1339 {
1340         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1341         struct rte_eth_dev_data *data = eth_dev->data;
1342         struct roc_nix_sq *sq = &dev->sqs[qid];
1343         int rc;
1344
1345         if (data->tx_queue_state[qid] == RTE_ETH_QUEUE_STATE_STOPPED)
1346                 return 0;
1347
1348         rc = roc_nix_tm_sq_aura_fc(sq, false);
1349         if (rc) {
1350                 plt_err("Failed to disable sqb aura fc, txq=%u, rc=%d", qid,
1351                         rc);
1352                 goto done;
1353         }
1354
1355         data->tx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED;
1356 done:
1357         return rc;
1358 }
1359
1360 static int
1361 cnxk_nix_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t qid)
1362 {
1363         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1364         struct rte_eth_dev_data *data = eth_dev->data;
1365         struct roc_nix_rq *rq = &dev->rqs[qid];
1366         int rc;
1367
1368         if (data->rx_queue_state[qid] == RTE_ETH_QUEUE_STATE_STARTED)
1369                 return 0;
1370
1371         rc = roc_nix_rq_ena_dis(rq, true);
1372         if (rc) {
1373                 plt_err("Failed to enable rxq=%u, rc=%d", qid, rc);
1374                 goto done;
1375         }
1376
1377         data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STARTED;
1378 done:
1379         return rc;
1380 }
1381
1382 static int
1383 cnxk_nix_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t qid)
1384 {
1385         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1386         struct rte_eth_dev_data *data = eth_dev->data;
1387         struct roc_nix_rq *rq = &dev->rqs[qid];
1388         int rc;
1389
1390         if (data->rx_queue_state[qid] == RTE_ETH_QUEUE_STATE_STOPPED)
1391                 return 0;
1392
1393         rc = roc_nix_rq_ena_dis(rq, false);
1394         if (rc) {
1395                 plt_err("Failed to disable rxq=%u, rc=%d", qid, rc);
1396                 goto done;
1397         }
1398
1399         data->rx_queue_state[qid] = RTE_ETH_QUEUE_STATE_STOPPED;
1400 done:
1401         return rc;
1402 }
1403
1404 static int
1405 cnxk_nix_dev_stop(struct rte_eth_dev *eth_dev)
1406 {
1407         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1408         const struct eth_dev_ops *dev_ops = eth_dev->dev_ops;
1409         struct rte_mbuf *rx_pkts[32];
1410         struct rte_eth_link link;
1411         int count, i, j, rc;
1412         void *rxq;
1413
1414         /* Disable all the NPC entries */
1415         rc = roc_npc_mcam_enable_all_entries(&dev->npc, 0);
1416         if (rc)
1417                 return rc;
1418
1419         /* Stop link change events */
1420         if (!roc_nix_is_vf_or_sdp(&dev->nix))
1421                 roc_nix_mac_link_event_start_stop(&dev->nix, false);
1422
1423         /* Disable Rx via NPC */
1424         roc_nix_npc_rx_ena_dis(&dev->nix, false);
1425
1426         /* Stop rx queues and free up pkts pending */
1427         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1428                 rc = dev_ops->rx_queue_stop(eth_dev, i);
1429                 if (rc)
1430                         continue;
1431
1432                 rxq = eth_dev->data->rx_queues[i];
1433                 count = dev->rx_pkt_burst_no_offload(rxq, rx_pkts, 32);
1434                 while (count) {
1435                         for (j = 0; j < count; j++)
1436                                 rte_pktmbuf_free(rx_pkts[j]);
1437                         count = dev->rx_pkt_burst_no_offload(rxq, rx_pkts, 32);
1438                 }
1439         }
1440
1441         /* Stop tx queues  */
1442         for (i = 0; i < eth_dev->data->nb_tx_queues; i++)
1443                 dev_ops->tx_queue_stop(eth_dev, i);
1444
1445         /* Bring down link status internally */
1446         memset(&link, 0, sizeof(link));
1447         rte_eth_linkstatus_set(eth_dev, &link);
1448
1449         return 0;
1450 }
1451
1452 int
1453 cnxk_nix_dev_start(struct rte_eth_dev *eth_dev)
1454 {
1455         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1456         int rc, i;
1457
1458         if (eth_dev->data->nb_rx_queues != 0 && !dev->ptp_en) {
1459                 rc = nix_recalc_mtu(eth_dev);
1460                 if (rc)
1461                         return rc;
1462         }
1463
1464         /* Start rx queues */
1465         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1466                 rc = cnxk_nix_rx_queue_start(eth_dev, i);
1467                 if (rc)
1468                         return rc;
1469         }
1470
1471         /* Start tx queues  */
1472         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1473                 rc = cnxk_nix_tx_queue_start(eth_dev, i);
1474                 if (rc)
1475                         return rc;
1476         }
1477
1478         /* Update Flow control configuration */
1479         rc = nix_update_flow_ctrl_config(eth_dev);
1480         if (rc) {
1481                 plt_err("Failed to enable flow control. error code(%d)", rc);
1482                 return rc;
1483         }
1484
1485         /* Enable Rx in NPC */
1486         rc = roc_nix_npc_rx_ena_dis(&dev->nix, true);
1487         if (rc) {
1488                 plt_err("Failed to enable NPC rx %d", rc);
1489                 return rc;
1490         }
1491
1492         rc = roc_npc_mcam_enable_all_entries(&dev->npc, 1);
1493         if (rc) {
1494                 plt_err("Failed to enable NPC entries %d", rc);
1495                 return rc;
1496         }
1497
1498         cnxk_nix_toggle_flag_link_cfg(dev, true);
1499
1500         /* Start link change events */
1501         if (!roc_nix_is_vf_or_sdp(&dev->nix)) {
1502                 rc = roc_nix_mac_link_event_start_stop(&dev->nix, true);
1503                 if (rc) {
1504                         plt_err("Failed to start cgx link event %d", rc);
1505                         goto rx_disable;
1506                 }
1507         }
1508
1509         /* Enable PTP if it is requested by the user or already
1510          * enabled on PF owning this VF
1511          */
1512         memset(&dev->tstamp, 0, sizeof(struct cnxk_timesync_info));
1513         if ((dev->rx_offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) || dev->ptp_en)
1514                 cnxk_eth_dev_ops.timesync_enable(eth_dev);
1515         else
1516                 cnxk_eth_dev_ops.timesync_disable(eth_dev);
1517
1518         if (dev->rx_offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP) {
1519                 rc = rte_mbuf_dyn_rx_timestamp_register
1520                         (&dev->tstamp.tstamp_dynfield_offset,
1521                          &dev->tstamp.rx_tstamp_dynflag);
1522                 if (rc != 0) {
1523                         plt_err("Failed to register Rx timestamp field/flag");
1524                         goto rx_disable;
1525                 }
1526         }
1527
1528         cnxk_nix_toggle_flag_link_cfg(dev, false);
1529
1530         return 0;
1531
1532 rx_disable:
1533         roc_nix_npc_rx_ena_dis(&dev->nix, false);
1534         cnxk_nix_toggle_flag_link_cfg(dev, false);
1535         return rc;
1536 }
1537
1538 static int cnxk_nix_dev_reset(struct rte_eth_dev *eth_dev);
1539 static int cnxk_nix_dev_close(struct rte_eth_dev *eth_dev);
1540
1541 /* CNXK platform independent eth dev ops */
1542 struct eth_dev_ops cnxk_eth_dev_ops = {
1543         .mtu_set = cnxk_nix_mtu_set,
1544         .mac_addr_add = cnxk_nix_mac_addr_add,
1545         .mac_addr_remove = cnxk_nix_mac_addr_del,
1546         .mac_addr_set = cnxk_nix_mac_addr_set,
1547         .dev_infos_get = cnxk_nix_info_get,
1548         .link_update = cnxk_nix_link_update,
1549         .tx_queue_release = cnxk_nix_tx_queue_release,
1550         .rx_queue_release = cnxk_nix_rx_queue_release,
1551         .dev_stop = cnxk_nix_dev_stop,
1552         .dev_close = cnxk_nix_dev_close,
1553         .dev_reset = cnxk_nix_dev_reset,
1554         .tx_queue_start = cnxk_nix_tx_queue_start,
1555         .rx_queue_start = cnxk_nix_rx_queue_start,
1556         .rx_queue_stop = cnxk_nix_rx_queue_stop,
1557         .dev_supported_ptypes_get = cnxk_nix_supported_ptypes_get,
1558         .promiscuous_enable = cnxk_nix_promisc_enable,
1559         .promiscuous_disable = cnxk_nix_promisc_disable,
1560         .allmulticast_enable = cnxk_nix_allmulticast_enable,
1561         .allmulticast_disable = cnxk_nix_allmulticast_disable,
1562         .rx_burst_mode_get = cnxk_nix_rx_burst_mode_get,
1563         .tx_burst_mode_get = cnxk_nix_tx_burst_mode_get,
1564         .flow_ctrl_get = cnxk_nix_flow_ctrl_get,
1565         .flow_ctrl_set = cnxk_nix_flow_ctrl_set,
1566         .priority_flow_ctrl_queue_config =
1567                                 cnxk_nix_priority_flow_ctrl_queue_config,
1568         .priority_flow_ctrl_queue_info_get =
1569                                 cnxk_nix_priority_flow_ctrl_queue_info_get,
1570         .dev_set_link_up = cnxk_nix_set_link_up,
1571         .dev_set_link_down = cnxk_nix_set_link_down,
1572         .get_module_info = cnxk_nix_get_module_info,
1573         .get_module_eeprom = cnxk_nix_get_module_eeprom,
1574         .rx_queue_intr_enable = cnxk_nix_rx_queue_intr_enable,
1575         .rx_queue_intr_disable = cnxk_nix_rx_queue_intr_disable,
1576         .pool_ops_supported = cnxk_nix_pool_ops_supported,
1577         .queue_stats_mapping_set = cnxk_nix_queue_stats_mapping,
1578         .stats_get = cnxk_nix_stats_get,
1579         .stats_reset = cnxk_nix_stats_reset,
1580         .xstats_get = cnxk_nix_xstats_get,
1581         .xstats_get_names = cnxk_nix_xstats_get_names,
1582         .xstats_reset = cnxk_nix_xstats_reset,
1583         .xstats_get_by_id = cnxk_nix_xstats_get_by_id,
1584         .xstats_get_names_by_id = cnxk_nix_xstats_get_names_by_id,
1585         .fw_version_get = cnxk_nix_fw_version_get,
1586         .rxq_info_get = cnxk_nix_rxq_info_get,
1587         .txq_info_get = cnxk_nix_txq_info_get,
1588         .tx_done_cleanup = cnxk_nix_tx_done_cleanup,
1589         .flow_ops_get = cnxk_nix_flow_ops_get,
1590         .get_reg = cnxk_nix_dev_get_reg,
1591         .timesync_read_rx_timestamp = cnxk_nix_timesync_read_rx_timestamp,
1592         .timesync_read_tx_timestamp = cnxk_nix_timesync_read_tx_timestamp,
1593         .timesync_read_time = cnxk_nix_timesync_read_time,
1594         .timesync_write_time = cnxk_nix_timesync_write_time,
1595         .timesync_adjust_time = cnxk_nix_timesync_adjust_time,
1596         .read_clock = cnxk_nix_read_clock,
1597         .reta_update = cnxk_nix_reta_update,
1598         .reta_query = cnxk_nix_reta_query,
1599         .rss_hash_update = cnxk_nix_rss_hash_update,
1600         .rss_hash_conf_get = cnxk_nix_rss_hash_conf_get,
1601         .set_mc_addr_list = cnxk_nix_mc_addr_list_configure,
1602         .set_queue_rate_limit = cnxk_nix_tm_set_queue_rate_limit,
1603         .tm_ops_get = cnxk_nix_tm_ops_get,
1604         .mtr_ops_get = cnxk_nix_mtr_ops_get,
1605 };
1606
1607 static int
1608 cnxk_eth_dev_init(struct rte_eth_dev *eth_dev)
1609 {
1610         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1611         struct rte_security_ctx *sec_ctx;
1612         struct roc_nix *nix = &dev->nix;
1613         struct rte_pci_device *pci_dev;
1614         int rc, max_entries;
1615
1616         eth_dev->dev_ops = &cnxk_eth_dev_ops;
1617         eth_dev->rx_queue_count = cnxk_nix_rx_queue_count;
1618         eth_dev->rx_descriptor_status = cnxk_nix_rx_descriptor_status;
1619         eth_dev->tx_descriptor_status = cnxk_nix_tx_descriptor_status;
1620
1621         /* Alloc security context */
1622         sec_ctx = plt_zmalloc(sizeof(struct rte_security_ctx), 0);
1623         if (!sec_ctx)
1624                 return -ENOMEM;
1625         sec_ctx->device = eth_dev;
1626         sec_ctx->ops = &cnxk_eth_sec_ops;
1627         sec_ctx->flags =
1628                 (RTE_SEC_CTX_F_FAST_SET_MDATA | RTE_SEC_CTX_F_FAST_GET_UDATA);
1629         eth_dev->security_ctx = sec_ctx;
1630
1631         /* For secondary processes, the primary has done all the work */
1632         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1633                 return 0;
1634
1635         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1636         rte_eth_copy_pci_info(eth_dev, pci_dev);
1637
1638         /* Parse devargs string */
1639         rc = cnxk_ethdev_parse_devargs(eth_dev->device->devargs, dev);
1640         if (rc) {
1641                 plt_err("Failed to parse devargs rc=%d", rc);
1642                 goto error;
1643         }
1644
1645         /* Initialize base roc nix */
1646         nix->pci_dev = pci_dev;
1647         nix->hw_vlan_ins = true;
1648         rc = roc_nix_dev_init(nix);
1649         if (rc) {
1650                 plt_err("Failed to initialize roc nix rc=%d", rc);
1651                 goto error;
1652         }
1653
1654         /* Register up msg callbacks */
1655         roc_nix_mac_link_cb_register(nix, cnxk_eth_dev_link_status_cb);
1656
1657         /* Register up msg callbacks */
1658         roc_nix_mac_link_info_get_cb_register(nix,
1659                                               cnxk_eth_dev_link_status_get_cb);
1660
1661         dev->eth_dev = eth_dev;
1662         dev->configured = 0;
1663         dev->ptype_disable = 0;
1664
1665         TAILQ_INIT(&dev->inb.list);
1666         TAILQ_INIT(&dev->outb.list);
1667         rte_spinlock_init(&dev->inb.lock);
1668         rte_spinlock_init(&dev->outb.lock);
1669
1670         /* For vfs, returned max_entries will be 0. but to keep default mac
1671          * address, one entry must be allocated. so setting up to 1.
1672          */
1673         if (roc_nix_is_vf_or_sdp(nix))
1674                 max_entries = 1;
1675         else
1676                 max_entries = roc_nix_mac_max_entries_get(nix);
1677
1678         if (max_entries <= 0) {
1679                 plt_err("Failed to get max entries for mac addr");
1680                 rc = -ENOTSUP;
1681                 goto dev_fini;
1682         }
1683
1684         eth_dev->data->mac_addrs =
1685                 rte_zmalloc("mac_addr", max_entries * RTE_ETHER_ADDR_LEN, 0);
1686         if (eth_dev->data->mac_addrs == NULL) {
1687                 plt_err("Failed to allocate memory for mac addr");
1688                 rc = -ENOMEM;
1689                 goto dev_fini;
1690         }
1691
1692         dev->max_mac_entries = max_entries;
1693         dev->dmac_filter_count = 1;
1694
1695         /* Get mac address */
1696         rc = roc_nix_npc_mac_addr_get(nix, dev->mac_addr);
1697         if (rc) {
1698                 plt_err("Failed to get mac addr, rc=%d", rc);
1699                 goto free_mac_addrs;
1700         }
1701
1702         /* Update the mac address */
1703         memcpy(eth_dev->data->mac_addrs, dev->mac_addr, RTE_ETHER_ADDR_LEN);
1704
1705         if (!roc_nix_is_vf_or_sdp(nix)) {
1706                 /* Sync same MAC address to CGX/RPM table */
1707                 rc = roc_nix_mac_addr_set(nix, dev->mac_addr);
1708                 if (rc) {
1709                         plt_err("Failed to set mac addr, rc=%d", rc);
1710                         goto free_mac_addrs;
1711                 }
1712         }
1713
1714         /* Union of all capabilities supported by CNXK.
1715          * Platform specific capabilities will be
1716          * updated later.
1717          */
1718         dev->rx_offload_capa = nix_get_rx_offload_capa(dev);
1719         dev->tx_offload_capa = nix_get_tx_offload_capa(dev);
1720         dev->speed_capa = nix_get_speed_capa(dev);
1721
1722         /* Initialize roc npc */
1723         dev->npc.roc_nix = nix;
1724         rc = roc_npc_init(&dev->npc);
1725         if (rc)
1726                 goto free_mac_addrs;
1727
1728         plt_nix_dbg("Port=%d pf=%d vf=%d ver=%s hwcap=0x%" PRIx64
1729                     " rxoffload_capa=0x%" PRIx64 " txoffload_capa=0x%" PRIx64,
1730                     eth_dev->data->port_id, roc_nix_get_pf(nix),
1731                     roc_nix_get_vf(nix), CNXK_ETH_DEV_PMD_VERSION, dev->hwcap,
1732                     dev->rx_offload_capa, dev->tx_offload_capa);
1733         return 0;
1734
1735 free_mac_addrs:
1736         rte_free(eth_dev->data->mac_addrs);
1737 dev_fini:
1738         roc_nix_dev_fini(nix);
1739 error:
1740         plt_err("Failed to init nix eth_dev rc=%d", rc);
1741         return rc;
1742 }
1743
1744 static int
1745 cnxk_eth_dev_uninit(struct rte_eth_dev *eth_dev, bool reset)
1746 {
1747         struct cnxk_eth_dev *dev = cnxk_eth_pmd_priv(eth_dev);
1748         const struct eth_dev_ops *dev_ops = eth_dev->dev_ops;
1749         struct rte_eth_pfc_queue_conf pfc_conf;
1750         struct roc_nix *nix = &dev->nix;
1751         struct rte_eth_fc_conf fc_conf;
1752         int rc, i;
1753
1754         /* Disable switch hdr pkind */
1755         roc_nix_switch_hdr_set(&dev->nix, 0, 0, 0, 0);
1756
1757         plt_free(eth_dev->security_ctx);
1758         eth_dev->security_ctx = NULL;
1759
1760         /* Nothing to be done for secondary processes */
1761         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1762                 return 0;
1763
1764         /* Clear the flag since we are closing down */
1765         dev->configured = 0;
1766
1767         roc_nix_npc_rx_ena_dis(nix, false);
1768
1769         /* Restore 802.3 Flow control configuration */
1770         memset(&pfc_conf, 0, sizeof(struct rte_eth_pfc_queue_conf));
1771         memset(&fc_conf, 0, sizeof(struct rte_eth_fc_conf));
1772         fc_conf.mode = RTE_ETH_FC_NONE;
1773         rc = cnxk_nix_flow_ctrl_set(eth_dev, &fc_conf);
1774
1775         pfc_conf.mode = RTE_ETH_FC_NONE;
1776         for (i = 0; i < CNXK_NIX_PFC_CHAN_COUNT; i++) {
1777                 if (dev->pfc_tc_sq_map[i] != 0xFFFF) {
1778                         pfc_conf.rx_pause.tx_qid = dev->pfc_tc_sq_map[i];
1779                         pfc_conf.rx_pause.tc = i;
1780                         pfc_conf.tx_pause.rx_qid = i;
1781                         pfc_conf.tx_pause.tc = i;
1782                         rc = cnxk_nix_priority_flow_ctrl_queue_config(eth_dev,
1783                                 &pfc_conf);
1784                         if (rc)
1785                                 plt_err("Failed to reset PFC. error code(%d)",
1786                                         rc);
1787                 }
1788         }
1789
1790         fc_conf.mode = RTE_ETH_FC_FULL;
1791         rc = cnxk_nix_flow_ctrl_set(eth_dev, &fc_conf);
1792
1793         /* Disable and free rte_meter entries */
1794         nix_meter_fini(dev);
1795
1796         /* Disable and free rte_flow entries */
1797         roc_npc_fini(&dev->npc);
1798
1799         /* Disable link status events */
1800         roc_nix_mac_link_event_start_stop(nix, false);
1801
1802         /* Unregister the link update op, this is required to stop VFs from
1803          * receiving link status updates on exit path.
1804          */
1805         roc_nix_mac_link_cb_unregister(nix);
1806
1807         /* Free up SQs */
1808         for (i = 0; i < eth_dev->data->nb_tx_queues; i++) {
1809                 dev_ops->tx_queue_release(eth_dev, i);
1810                 eth_dev->data->tx_queues[i] = NULL;
1811         }
1812         eth_dev->data->nb_tx_queues = 0;
1813
1814         /* Free up RQ's and CQ's */
1815         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1816                 dev_ops->rx_queue_release(eth_dev, i);
1817                 eth_dev->data->rx_queues[i] = NULL;
1818         }
1819         eth_dev->data->nb_rx_queues = 0;
1820
1821         /* Free security resources */
1822         nix_security_release(dev);
1823
1824         /* Free tm resources */
1825         roc_nix_tm_fini(nix);
1826
1827         /* Unregister queue irqs */
1828         roc_nix_unregister_queue_irqs(nix);
1829
1830         /* Unregister cq irqs */
1831         if (eth_dev->data->dev_conf.intr_conf.rxq)
1832                 roc_nix_unregister_cq_irqs(nix);
1833
1834         /* Free ROC RQ's, SQ's and CQ's memory */
1835         nix_free_queue_mem(dev);
1836
1837         /* Free nix lf resources */
1838         rc = roc_nix_lf_free(nix);
1839         if (rc)
1840                 plt_err("Failed to free nix lf, rc=%d", rc);
1841
1842         rte_free(eth_dev->data->mac_addrs);
1843         eth_dev->data->mac_addrs = NULL;
1844
1845         rc = roc_nix_dev_fini(nix);
1846         /* Can be freed later by PMD if NPA LF is in use */
1847         if (rc == -EAGAIN) {
1848                 if (!reset)
1849                         eth_dev->data->dev_private = NULL;
1850                 return 0;
1851         } else if (rc) {
1852                 plt_err("Failed in nix dev fini, rc=%d", rc);
1853         }
1854
1855         return rc;
1856 }
1857
1858 static int
1859 cnxk_nix_dev_close(struct rte_eth_dev *eth_dev)
1860 {
1861         cnxk_eth_dev_uninit(eth_dev, false);
1862         return 0;
1863 }
1864
1865 static int
1866 cnxk_nix_dev_reset(struct rte_eth_dev *eth_dev)
1867 {
1868         int rc;
1869
1870         rc = cnxk_eth_dev_uninit(eth_dev, true);
1871         if (rc)
1872                 return rc;
1873
1874         return cnxk_eth_dev_init(eth_dev);
1875 }
1876
1877 int
1878 cnxk_nix_remove(struct rte_pci_device *pci_dev)
1879 {
1880         struct rte_eth_dev *eth_dev;
1881         struct roc_nix *nix;
1882         int rc = -EINVAL;
1883
1884         eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
1885         if (eth_dev) {
1886                 /* Cleanup eth dev */
1887                 rc = cnxk_eth_dev_uninit(eth_dev, false);
1888                 if (rc)
1889                         return rc;
1890
1891                 rte_eth_dev_release_port(eth_dev);
1892         }
1893
1894         /* Nothing to be done for secondary processes */
1895         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1896                 return 0;
1897
1898         /* Check if this device is hosting common resource */
1899         nix = roc_idev_npa_nix_get();
1900         if (nix->pci_dev != pci_dev)
1901                 return 0;
1902
1903         /* Try nix fini now */
1904         rc = roc_nix_dev_fini(nix);
1905         if (rc == -EAGAIN) {
1906                 plt_info("%s: common resource in use by other devices",
1907                          pci_dev->name);
1908                 goto exit;
1909         } else if (rc) {
1910                 plt_err("Failed in nix dev fini, rc=%d", rc);
1911                 goto exit;
1912         }
1913
1914         /* Free device pointer as rte_ethdev does not have it anymore */
1915         rte_free(nix);
1916 exit:
1917         return rc;
1918 }
1919
1920 int
1921 cnxk_nix_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
1922 {
1923         int rc;
1924
1925         RTE_SET_USED(pci_drv);
1926
1927         rc = rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct cnxk_eth_dev),
1928                                            cnxk_eth_dev_init);
1929
1930         /* On error on secondary, recheck if port exists in primary or
1931          * in mid of detach state.
1932          */
1933         if (rte_eal_process_type() != RTE_PROC_PRIMARY && rc)
1934                 if (!rte_eth_dev_allocated(pci_dev->device.name))
1935                         return 0;
1936         return rc;
1937 }