drivers/net/octeontx2/otx2_rss.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include "otx2_ethdev.h"
   6
   7 int
   8 otx2_nix_rss_tbl_init(struct otx2_eth_dev *dev,
   9                       uint8_t group, uint16_t *ind_tbl)
  10 {
  11         struct otx2_rss_info *rss = &dev->rss_info;
  12         struct otx2_mbox *mbox = dev->mbox;
  13         struct nix_aq_enq_req *req;
  14         int rc, idx;
  15
  16         for (idx = 0; idx < rss->rss_size; idx++) {
  17                 req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
  18                 if (!req) {
  19                         /* The shared memory buffer can be full.
  20                          * Flush it and retry
  21                          */
  22                         otx2_mbox_msg_send(mbox, 0);
  23                         rc = otx2_mbox_wait_for_rsp(mbox, 0);
  24                         if (rc < 0)
  25                                 return rc;
  26
  27                         req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
  28                         if (!req)
  29                                 return -ENOMEM;
  30                 }
  31                 req->rss.rq = ind_tbl[idx];
  32                 /* Fill AQ info */
  33                 req->qidx = (group * rss->rss_size) + idx;
  34                 req->ctype = NIX_AQ_CTYPE_RSS;
  35                 req->op = NIX_AQ_INSTOP_INIT;
  36
  37                 if (!dev->lock_rx_ctx)
  38                         continue;
  39
  40                 req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
  41                 if (!req) {
  42                         /* The shared memory buffer can be full.
  43                          * Flush it and retry
  44                          */
  45                         otx2_mbox_msg_send(mbox, 0);
  46                         rc = otx2_mbox_wait_for_rsp(mbox, 0);
  47                         if (rc < 0)
  48                                 return rc;
  49
  50                         req = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
  51                         if (!req)
  52                                 return -ENOMEM;
  53                 }
  54                 req->rss.rq = ind_tbl[idx];
  55                 /* Fill AQ info */
  56                 req->qidx = (group * rss->rss_size) + idx;
  57                 req->ctype = NIX_AQ_CTYPE_RSS;
  58                 req->op = NIX_AQ_INSTOP_LOCK;
  59         }
  60
  61         otx2_mbox_msg_send(mbox, 0);
  62         rc = otx2_mbox_wait_for_rsp(mbox, 0);
  63         if (rc < 0)
  64                 return rc;
  65
  66         return 0;
  67 }
  68
  69 int
  70 otx2_nix_dev_reta_update(struct rte_eth_dev *eth_dev,
  71                          struct rte_eth_rss_reta_entry64 *reta_conf,
  72                          uint16_t reta_size)
  73 {
  74         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
  75         struct otx2_rss_info *rss = &dev->rss_info;
  76         int rc, i, j;
  77         int idx = 0;
  78
  79         rc = -EINVAL;
  80         if (reta_size != dev->rss_info.rss_size) {
  81                 otx2_err("Size of hash lookup table configured "
  82                 "(%d) doesn't match the number hardware can supported "
  83                 "(%d)", reta_size, dev->rss_info.rss_size);
  84                 goto fail;
  85         }
  86
  87         /* Copy RETA table */
  88         for (i = 0; i < (dev->rss_info.rss_size / RTE_RETA_GROUP_SIZE); i++) {
  89                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) {
  90                         if ((reta_conf[i].mask >> j) & 0x01)
  91                                 rss->ind_tbl[idx] = reta_conf[i].reta[j];
  92                         idx++;
  93                 }
  94         }
  95
  96         return otx2_nix_rss_tbl_init(dev, 0, dev->rss_info.ind_tbl);
  97
  98 fail:
  99         return rc;
 100 }
 101
 102 int
 103 otx2_nix_dev_reta_query(struct rte_eth_dev *eth_dev,
 104                         struct rte_eth_rss_reta_entry64 *reta_conf,
 105                         uint16_t reta_size)
 106 {
 107         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 108         struct otx2_rss_info *rss = &dev->rss_info;
 109         int rc, i, j;
 110
 111         rc = -EINVAL;
 112
 113         if (reta_size != dev->rss_info.rss_size) {
 114                 otx2_err("Size of hash lookup table configured "
 115                         "(%d) doesn't match the number hardware can supported "
 116                         "(%d)", reta_size, dev->rss_info.rss_size);
 117                 goto fail;
 118         }
 119
 120         /* Copy RETA table */
 121         for (i = 0; i < (dev->rss_info.rss_size / RTE_RETA_GROUP_SIZE); i++) {
 122                 for (j = 0; j < RTE_RETA_GROUP_SIZE; j++)
 123                         if ((reta_conf[i].mask >> j) & 0x01)
 124                                 reta_conf[i].reta[j] = rss->ind_tbl[j];
 125         }
 126
 127         return 0;
 128
 129 fail:
 130         return rc;
 131 }
 132
 133 void
 134 otx2_nix_rss_set_key(struct otx2_eth_dev *dev, uint8_t *key,
 135                      uint32_t key_len)
 136 {
 137         const uint8_t default_key[NIX_HASH_KEY_SIZE] = {
 138                 0xFE, 0xED, 0x0B, 0xAD, 0xFE, 0xED, 0x0B, 0xAD,
 139                 0xFE, 0xED, 0x0B, 0xAD, 0xFE, 0xED, 0x0B, 0xAD,
 140                 0xFE, 0xED, 0x0B, 0xAD, 0xFE, 0xED, 0x0B, 0xAD,
 141                 0xFE, 0xED, 0x0B, 0xAD, 0xFE, 0xED, 0x0B, 0xAD,
 142                 0xFE, 0xED, 0x0B, 0xAD, 0xFE, 0xED, 0x0B, 0xAD,
 143                 0xFE, 0xED, 0x0B, 0xAD, 0xFE, 0xED, 0x0B, 0xAD
 144         };
 145         struct otx2_rss_info *rss = &dev->rss_info;
 146         uint64_t *keyptr;
 147         uint64_t val;
 148         uint32_t idx;
 149
 150         if (key == NULL || key == 0) {
 151                 keyptr = (uint64_t *)(uintptr_t)default_key;
 152                 key_len = NIX_HASH_KEY_SIZE;
 153                 memset(rss->key, 0, key_len);
 154         } else {
 155                 memcpy(rss->key, key, key_len);
 156                 keyptr = (uint64_t *)rss->key;
 157         }
 158
 159         for (idx = 0; idx < (key_len >> 3); idx++) {
 160                 val = rte_cpu_to_be_64(*keyptr);
 161                 otx2_write64(val, dev->base + NIX_LF_RX_SECRETX(idx));
 162                 keyptr++;
 163         }
 164 }
 165
 166 static void
 167 rss_get_key(struct otx2_eth_dev *dev, uint8_t *key)
 168 {
 169         uint64_t *keyptr = (uint64_t *)key;
 170         uint64_t val;
 171         int idx;
 172
 173         for (idx = 0; idx < (NIX_HASH_KEY_SIZE >> 3); idx++) {
 174                 val = otx2_read64(dev->base + NIX_LF_RX_SECRETX(idx));
 175                 *keyptr = rte_be_to_cpu_64(val);
 176                 keyptr++;
 177         }
 178 }
 179
 180 #define RSS_IPV4_ENABLE ( \
 181                           ETH_RSS_IPV4 | \
 182                           ETH_RSS_FRAG_IPV4 | \
 183                           ETH_RSS_NONFRAG_IPV4_UDP | \
 184                           ETH_RSS_NONFRAG_IPV4_TCP | \
 185                           ETH_RSS_NONFRAG_IPV4_SCTP)
 186
 187 #define RSS_IPV6_ENABLE ( \
 188                           ETH_RSS_IPV6 | \
 189                           ETH_RSS_FRAG_IPV6 | \
 190                           ETH_RSS_NONFRAG_IPV6_UDP | \
 191                           ETH_RSS_NONFRAG_IPV6_TCP | \
 192                           ETH_RSS_NONFRAG_IPV6_SCTP)
 193
 194 #define RSS_IPV6_EX_ENABLE ( \
 195                              ETH_RSS_IPV6_EX | \
 196                              ETH_RSS_IPV6_TCP_EX | \
 197                              ETH_RSS_IPV6_UDP_EX)
 198
 199 #define RSS_MAX_LEVELS   3
 200
 201 #define RSS_IPV4_INDEX   0
 202 #define RSS_IPV6_INDEX   1
 203 #define RSS_TCP_INDEX    2
 204 #define RSS_UDP_INDEX    3
 205 #define RSS_SCTP_INDEX   4
 206 #define RSS_DMAC_INDEX   5
 207
 208 uint32_t
 209 otx2_rss_ethdev_to_nix(struct otx2_eth_dev *dev, uint64_t ethdev_rss,
 210                        uint8_t rss_level)
 211 {
 212         uint32_t flow_key_type[RSS_MAX_LEVELS][6] = {
 213                 {
 214                         FLOW_KEY_TYPE_IPV4, FLOW_KEY_TYPE_IPV6,
 215                         FLOW_KEY_TYPE_TCP, FLOW_KEY_TYPE_UDP,
 216                         FLOW_KEY_TYPE_SCTP, FLOW_KEY_TYPE_ETH_DMAC
 217                 },
 218                 {
 219                         FLOW_KEY_TYPE_INNR_IPV4, FLOW_KEY_TYPE_INNR_IPV6,
 220                         FLOW_KEY_TYPE_INNR_TCP, FLOW_KEY_TYPE_INNR_UDP,
 221                         FLOW_KEY_TYPE_INNR_SCTP, FLOW_KEY_TYPE_INNR_ETH_DMAC
 222                 },
 223                 {
 224                         FLOW_KEY_TYPE_IPV4 | FLOW_KEY_TYPE_INNR_IPV4,
 225                         FLOW_KEY_TYPE_IPV6 | FLOW_KEY_TYPE_INNR_IPV6,
 226                         FLOW_KEY_TYPE_TCP | FLOW_KEY_TYPE_INNR_TCP,
 227                         FLOW_KEY_TYPE_UDP | FLOW_KEY_TYPE_INNR_UDP,
 228                         FLOW_KEY_TYPE_SCTP | FLOW_KEY_TYPE_INNR_SCTP,
 229                         FLOW_KEY_TYPE_ETH_DMAC | FLOW_KEY_TYPE_INNR_ETH_DMAC
 230                 }
 231         };
 232         uint32_t flowkey_cfg = 0;
 233
 234         dev->rss_info.nix_rss = ethdev_rss;
 235
 236         if (ethdev_rss & ETH_RSS_L2_PAYLOAD &&
 237             dev->npc_flow.switch_header_type == OTX2_PRIV_FLAGS_CH_LEN_90B) {
 238                 flowkey_cfg |= FLOW_KEY_TYPE_CH_LEN_90B;
 239         }
 240
 241         if (ethdev_rss & ETH_RSS_C_VLAN)
 242                 flowkey_cfg |= FLOW_KEY_TYPE_VLAN;
 243
 244         if (ethdev_rss & ETH_RSS_L3_SRC_ONLY)
 245                 flowkey_cfg |= FLOW_KEY_TYPE_L3_SRC;
 246
 247         if (ethdev_rss & ETH_RSS_L3_DST_ONLY)
 248                 flowkey_cfg |= FLOW_KEY_TYPE_L3_DST;
 249
 250         if (ethdev_rss & ETH_RSS_L4_SRC_ONLY)
 251                 flowkey_cfg |= FLOW_KEY_TYPE_L4_SRC;
 252
 253         if (ethdev_rss & ETH_RSS_L4_DST_ONLY)
 254                 flowkey_cfg |= FLOW_KEY_TYPE_L4_DST;
 255
 256         if (ethdev_rss & RSS_IPV4_ENABLE)
 257                 flowkey_cfg |= flow_key_type[rss_level][RSS_IPV4_INDEX];
 258
 259         if (ethdev_rss & RSS_IPV6_ENABLE)
 260                 flowkey_cfg |= flow_key_type[rss_level][RSS_IPV6_INDEX];
 261
 262         if (ethdev_rss & ETH_RSS_TCP)
 263                 flowkey_cfg |= flow_key_type[rss_level][RSS_TCP_INDEX];
 264
 265         if (ethdev_rss & ETH_RSS_UDP)
 266                 flowkey_cfg |= flow_key_type[rss_level][RSS_UDP_INDEX];
 267
 268         if (ethdev_rss & ETH_RSS_SCTP)
 269                 flowkey_cfg |= flow_key_type[rss_level][RSS_SCTP_INDEX];
 270
 271         if (ethdev_rss & ETH_RSS_L2_PAYLOAD)
 272                 flowkey_cfg |= flow_key_type[rss_level][RSS_DMAC_INDEX];
 273
 274         if (ethdev_rss & RSS_IPV6_EX_ENABLE)
 275                 flowkey_cfg |= FLOW_KEY_TYPE_IPV6_EXT;
 276
 277         if (ethdev_rss & ETH_RSS_PORT)
 278                 flowkey_cfg |= FLOW_KEY_TYPE_PORT;
 279
 280         if (ethdev_rss & ETH_RSS_NVGRE)
 281                 flowkey_cfg |= FLOW_KEY_TYPE_NVGRE;
 282
 283         if (ethdev_rss & ETH_RSS_VXLAN)
 284                 flowkey_cfg |= FLOW_KEY_TYPE_VXLAN;
 285
 286         if (ethdev_rss & ETH_RSS_GENEVE)
 287                 flowkey_cfg |= FLOW_KEY_TYPE_GENEVE;
 288
 289         if (ethdev_rss & ETH_RSS_GTPU)
 290                 flowkey_cfg |= FLOW_KEY_TYPE_GTPU;
 291
 292         return flowkey_cfg;
 293 }
 294
 295 int
 296 otx2_rss_set_hf(struct otx2_eth_dev *dev, uint32_t flowkey_cfg,
 297                 uint8_t *alg_idx, uint8_t group, int mcam_index)
 298 {
 299         struct nix_rss_flowkey_cfg_rsp *rss_rsp;
 300         struct otx2_mbox *mbox = dev->mbox;
 301         struct nix_rss_flowkey_cfg *cfg;
 302         int rc;
 303
 304         rc = -EINVAL;
 305
 306         dev->rss_info.flowkey_cfg = flowkey_cfg;
 307
 308         cfg = otx2_mbox_alloc_msg_nix_rss_flowkey_cfg(mbox);
 309
 310         cfg->flowkey_cfg = flowkey_cfg;
 311         cfg->mcam_index = mcam_index; /* -1 indicates default group */
 312         cfg->group = group; /* 0 is default group */
 313
 314         rc = otx2_mbox_process_msg(mbox, (void *)&rss_rsp);
 315         if (rc)
 316                 return rc;
 317
 318         if (alg_idx)
 319                 *alg_idx = rss_rsp->alg_idx;
 320
 321         return rc;
 322 }
 323
 324 int
 325 otx2_nix_rss_hash_update(struct rte_eth_dev *eth_dev,
 326                          struct rte_eth_rss_conf *rss_conf)
 327 {
 328         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 329         uint8_t rss_hash_level;
 330         uint32_t flowkey_cfg;
 331         uint8_t alg_idx;
 332         int rc;
 333
 334         rc = -EINVAL;
 335
 336         if (rss_conf->rss_key && rss_conf->rss_key_len != NIX_HASH_KEY_SIZE) {
 337                 otx2_err("Hash key size mismatch %d vs %d",
 338                          rss_conf->rss_key_len, NIX_HASH_KEY_SIZE);
 339                 goto fail;
 340         }
 341
 342         if (rss_conf->rss_key)
 343                 otx2_nix_rss_set_key(dev, rss_conf->rss_key,
 344                                      (uint32_t)rss_conf->rss_key_len);
 345
 346         rss_hash_level = ETH_RSS_LEVEL(rss_conf->rss_hf);
 347         if (rss_hash_level)
 348                 rss_hash_level -= 1;
 349         flowkey_cfg =
 350                 otx2_rss_ethdev_to_nix(dev, rss_conf->rss_hf, rss_hash_level);
 351
 352         rc = otx2_rss_set_hf(dev, flowkey_cfg, &alg_idx,
 353                              NIX_DEFAULT_RSS_CTX_GROUP,
 354                              NIX_DEFAULT_RSS_MCAM_IDX);
 355         if (rc) {
 356                 otx2_err("Failed to set RSS hash function rc=%d", rc);
 357                 return rc;
 358         }
 359
 360         dev->rss_info.alg_idx = alg_idx;
 361
 362 fail:
 363         return rc;
 364 }
 365
 366 int
 367 otx2_nix_rss_hash_conf_get(struct rte_eth_dev *eth_dev,
 368                            struct rte_eth_rss_conf *rss_conf)
 369 {
 370         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 371
 372         if (rss_conf->rss_key)
 373                 rss_get_key(dev, rss_conf->rss_key);
 374
 375         rss_conf->rss_key_len = NIX_HASH_KEY_SIZE;
 376         rss_conf->rss_hf = dev->rss_info.nix_rss;
 377
 378         return 0;
 379 }
 380
 381 int
 382 otx2_nix_rss_config(struct rte_eth_dev *eth_dev)
 383 {
 384         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 385         uint32_t idx, qcnt = eth_dev->data->nb_rx_queues;
 386         uint8_t rss_hash_level;
 387         uint32_t flowkey_cfg;
 388         uint64_t rss_hf;
 389         uint8_t alg_idx;
 390         int rc;
 391
 392         /* Skip further configuration if selected mode is not RSS */
 393         if (eth_dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS || !qcnt)
 394                 return 0;
 395
 396         /* Update default RSS key and cfg */
 397         otx2_nix_rss_set_key(dev, NULL, 0);
 398
 399         /* Update default RSS RETA */
 400         for (idx = 0; idx < dev->rss_info.rss_size; idx++)
 401                 dev->rss_info.ind_tbl[idx] = idx % qcnt;
 402
 403         /* Init RSS table context */
 404         rc = otx2_nix_rss_tbl_init(dev, 0, dev->rss_info.ind_tbl);
 405         if (rc) {
 406                 otx2_err("Failed to init RSS table rc=%d", rc);
 407                 return rc;
 408         }
 409
 410         rss_hf = eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
 411         rss_hash_level = ETH_RSS_LEVEL(rss_hf);
 412         if (rss_hash_level)
 413                 rss_hash_level -= 1;
 414         flowkey_cfg = otx2_rss_ethdev_to_nix(dev, rss_hf, rss_hash_level);
 415
 416         rc = otx2_rss_set_hf(dev, flowkey_cfg, &alg_idx,
 417                              NIX_DEFAULT_RSS_CTX_GROUP,
 418                              NIX_DEFAULT_RSS_MCAM_IDX);
 419         if (rc) {
 420                 otx2_err("Failed to set RSS hash function rc=%d", rc);
 421                 return rc;
 422         }
 423
 424         dev->rss_info.alg_idx = alg_idx;
 425
 426         return 0;
 427 }