drivers/raw/octeontx2_ep/otx2_ep_vf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_rawdev.h>
   7 #include <rte_rawdev_pmd.h>
   8
   9 #include "otx2_common.h"
  10 #include "otx2_ep_rawdev.h"
  11 #include "otx2_ep_vf.h"
  12
  13 static int
  14 sdp_vf_reset_iq(struct sdp_device *sdpvf, int q_no)
  15 {
  16         uint64_t loop = SDP_VF_BUSY_LOOP_COUNT;
  17         volatile uint64_t d64 = 0ull;
  18
  19         /* There is no RST for a ring.
  20          * Clear all registers one by one after disabling the ring
  21          */
  22
  23         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
  24         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_INSTR_BADDR(q_no));
  25         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_INSTR_RSIZE(q_no));
  26
  27         d64 = 0xFFFFFFFF; /* ~0ull */
  28         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_INSTR_DBELL(q_no));
  29         d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_INSTR_DBELL(q_no));
  30
  31         while ((d64 != 0) && loop--) {
  32                 otx2_write64(d64, sdpvf->hw_addr +
  33                              SDP_VF_R_IN_INSTR_DBELL(q_no));
  34
  35                 rte_delay_ms(1);
  36
  37                 d64 = otx2_read64(sdpvf->hw_addr +
  38                                   SDP_VF_R_IN_INSTR_DBELL(q_no));
  39         }
  40
  41         loop = SDP_VF_BUSY_LOOP_COUNT;
  42         d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CNTS(q_no));
  43         while ((d64 != 0) && loop--) {
  44                 otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_CNTS(q_no));
  45
  46                 rte_delay_ms(1);
  47
  48                 d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CNTS(q_no));
  49         }
  50
  51         d64 = 0ull;
  52         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_INT_LEVELS(q_no));
  53         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_PKT_CNT(q_no));
  54         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_IN_BYTE_CNT(q_no));
  55
  56         return 0;
  57 }
  58
  59 static int
  60 sdp_vf_reset_oq(struct sdp_device *sdpvf, int q_no)
  61 {
  62         uint64_t loop = SDP_VF_BUSY_LOOP_COUNT;
  63         volatile uint64_t d64 = 0ull;
  64
  65         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
  66
  67         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_SLIST_BADDR(q_no));
  68
  69         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_SLIST_RSIZE(q_no));
  70
  71         d64 = 0xFFFFFFFF;
  72         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_SLIST_DBELL(q_no));
  73         d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_SLIST_DBELL(q_no));
  74
  75         while ((d64 != 0) && loop--) {
  76                 otx2_write64(d64, sdpvf->hw_addr +
  77                              SDP_VF_R_OUT_SLIST_DBELL(q_no));
  78
  79                 rte_delay_ms(1);
  80
  81                 d64 = otx2_read64(sdpvf->hw_addr +
  82                                   SDP_VF_R_OUT_SLIST_DBELL(q_no));
  83         }
  84
  85         loop = SDP_VF_BUSY_LOOP_COUNT;
  86         d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CNTS(q_no));
  87         while ((d64 != 0) && (loop--)) {
  88                 otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_CNTS(q_no));
  89
  90                 rte_delay_ms(1);
  91
  92                 d64 = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CNTS(q_no));
  93         }
  94
  95         d64 = 0ull;
  96         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_INT_LEVELS(q_no));
  97         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_PKT_CNT(q_no));
  98         otx2_write64(d64, sdpvf->hw_addr + SDP_VF_R_OUT_BYTE_CNT(q_no));
  99
 100         return 0;
 101 }
 102
 103 static void
 104 sdp_vf_setup_global_iq_reg(struct sdp_device *sdpvf, int q_no)
 105 {
 106         volatile uint64_t reg_val = 0ull;
 107
 108         /* Select ES, RO, NS, RDSIZE,DPTR Fomat#0 for IQs
 109          * IS_64B is by default enabled.
 110          */
 111         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CONTROL(q_no));
 112
 113         reg_val |= SDP_VF_R_IN_CTL_RDSIZE;
 114         reg_val |= SDP_VF_R_IN_CTL_IS_64B;
 115         reg_val |= SDP_VF_R_IN_CTL_ESR;
 116
 117         otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_CONTROL(q_no));
 118
 119 }
 120
 121 static void
 122 sdp_vf_setup_global_oq_reg(struct sdp_device *sdpvf, int q_no)
 123 {
 124         volatile uint64_t reg_val = 0ull;
 125
 126         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(q_no));
 127
 128         reg_val |= (SDP_VF_R_OUT_CTL_IMODE);
 129
 130         reg_val &= ~(SDP_VF_R_OUT_CTL_ROR_P);
 131         reg_val &= ~(SDP_VF_R_OUT_CTL_NSR_P);
 132         reg_val &= ~(SDP_VF_R_OUT_CTL_ROR_I);
 133         reg_val &= ~(SDP_VF_R_OUT_CTL_NSR_I);
 134         reg_val &= ~(SDP_VF_R_OUT_CTL_ES_I);
 135         reg_val &= ~(SDP_VF_R_OUT_CTL_ROR_D);
 136         reg_val &= ~(SDP_VF_R_OUT_CTL_NSR_D);
 137         reg_val &= ~(SDP_VF_R_OUT_CTL_ES_D);
 138
 139         /* INFO/DATA ptr swap is required  */
 140         reg_val |= (SDP_VF_R_OUT_CTL_ES_P);
 141
 142         otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(q_no));
 143
 144 }
 145
 146 static int
 147 sdp_vf_reset_input_queues(struct sdp_device *sdpvf)
 148 {
 149         uint32_t q_no = 0;
 150
 151         otx2_sdp_dbg("%s :", __func__);
 152
 153         for (q_no = 0; q_no < sdpvf->sriov_info.rings_per_vf; q_no++)
 154                 sdp_vf_reset_iq(sdpvf, q_no);
 155
 156         return 0;
 157 }
 158
 159 static int
 160 sdp_vf_reset_output_queues(struct sdp_device *sdpvf)
 161 {
 162         uint64_t q_no = 0ull;
 163
 164         otx2_sdp_dbg(" %s :", __func__);
 165
 166         for (q_no = 0; q_no < sdpvf->sriov_info.rings_per_vf; q_no++)
 167                 sdp_vf_reset_oq(sdpvf, q_no);
 168
 169         return 0;
 170 }
 171
 172 static void
 173 sdp_vf_setup_global_input_regs(struct sdp_device *sdpvf)
 174 {
 175         uint64_t q_no = 0ull;
 176
 177         sdp_vf_reset_input_queues(sdpvf);
 178
 179         for (q_no = 0; q_no < (sdpvf->sriov_info.rings_per_vf); q_no++)
 180                 sdp_vf_setup_global_iq_reg(sdpvf, q_no);
 181 }
 182
 183 static void
 184 sdp_vf_setup_global_output_regs(struct sdp_device *sdpvf)
 185 {
 186         uint32_t q_no;
 187
 188         sdp_vf_reset_output_queues(sdpvf);
 189
 190         for (q_no = 0; q_no < (sdpvf->sriov_info.rings_per_vf); q_no++)
 191                 sdp_vf_setup_global_oq_reg(sdpvf, q_no);
 192
 193 }
 194
 195 static int
 196 sdp_vf_setup_device_regs(struct sdp_device *sdpvf)
 197 {
 198         sdp_vf_setup_global_input_regs(sdpvf);
 199         sdp_vf_setup_global_output_regs(sdpvf);
 200
 201         return 0;
 202 }
 203
 204 static void
 205 sdp_vf_setup_iq_regs(struct sdp_device *sdpvf, uint32_t iq_no)
 206 {
 207         struct sdp_instr_queue *iq = sdpvf->instr_queue[iq_no];
 208         volatile uint64_t reg_val = 0ull;
 209
 210         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CONTROL(iq_no));
 211
 212         /* Wait till IDLE to set to 1, not supposed to configure BADDR
 213          * as long as IDLE is 0
 214          */
 215         if (!(reg_val & SDP_VF_R_IN_CTL_IDLE)) {
 216                 do {
 217                         reg_val = otx2_read64(sdpvf->hw_addr +
 218                                               SDP_VF_R_IN_CONTROL(iq_no));
 219                 } while (!(reg_val & SDP_VF_R_IN_CTL_IDLE));
 220         }
 221
 222         /* Write the start of the input queue's ring and its size  */
 223         otx2_write64(iq->base_addr_dma, sdpvf->hw_addr +
 224                      SDP_VF_R_IN_INSTR_BADDR(iq_no));
 225         otx2_write64(iq->nb_desc, sdpvf->hw_addr +
 226                      SDP_VF_R_IN_INSTR_RSIZE(iq_no));
 227
 228         /* Remember the doorbell & instruction count register addr
 229          * for this queue
 230          */
 231         iq->doorbell_reg = (uint8_t *) sdpvf->hw_addr +
 232                            SDP_VF_R_IN_INSTR_DBELL(iq_no);
 233         iq->inst_cnt_reg = (uint8_t *) sdpvf->hw_addr +
 234                            SDP_VF_R_IN_CNTS(iq_no);
 235
 236         otx2_sdp_dbg("InstQ[%d]:dbell reg @ 0x%p instcnt_reg @ 0x%p",
 237                      iq_no, iq->doorbell_reg, iq->inst_cnt_reg);
 238
 239         /* Store the current instrn counter(used in flush_iq calculation) */
 240         iq->reset_instr_cnt = rte_read32(iq->inst_cnt_reg);
 241
 242         /* IN INTR_THRESHOLD is set to max(FFFFFFFF) which disable the IN INTR
 243          * to raise
 244          */
 245         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_INT_LEVELS(iq_no));
 246         reg_val = 0xffffffff;
 247
 248         otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_INT_LEVELS(iq_no));
 249
 250 }
 251
 252 static void
 253 sdp_vf_setup_oq_regs(struct sdp_device *sdpvf, uint32_t oq_no)
 254 {
 255         volatile uint64_t reg_val = 0ull;
 256         uint64_t oq_ctl = 0ull;
 257
 258         struct sdp_droq *droq = sdpvf->droq[oq_no];
 259
 260         /* Wait on IDLE to set to 1, supposed to configure BADDR
 261          * as log as IDLE is 0
 262          */
 263         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(oq_no));
 264
 265         while (!(reg_val & SDP_VF_R_OUT_CTL_IDLE)) {
 266                 reg_val = otx2_read64(sdpvf->hw_addr +
 267                                       SDP_VF_R_OUT_CONTROL(oq_no));
 268         }
 269
 270         otx2_write64(droq->desc_ring_dma, sdpvf->hw_addr +
 271                      SDP_VF_R_OUT_SLIST_BADDR(oq_no));
 272         otx2_write64(droq->nb_desc, sdpvf->hw_addr +
 273                      SDP_VF_R_OUT_SLIST_RSIZE(oq_no));
 274
 275         oq_ctl = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(oq_no));
 276
 277         /* Clear the ISIZE and BSIZE (22-0) */
 278         oq_ctl &= ~(0x7fffffull);
 279
 280         /* Populate the BSIZE (15-0) */
 281         oq_ctl |= (droq->buffer_size & 0xffff);
 282
 283         /* Populate ISIZE(22-16) */
 284         oq_ctl |= ((SDP_RH_SIZE << 16) & 0x7fffff);
 285         otx2_write64(oq_ctl, sdpvf->hw_addr + SDP_VF_R_OUT_CONTROL(oq_no));
 286
 287         /* Mapped address of the pkt_sent and pkts_credit regs */
 288         droq->pkts_sent_reg = (uint8_t *) sdpvf->hw_addr +
 289                               SDP_VF_R_OUT_CNTS(oq_no);
 290         droq->pkts_credit_reg = (uint8_t *) sdpvf->hw_addr +
 291                                 SDP_VF_R_OUT_SLIST_DBELL(oq_no);
 292
 293         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_INT_LEVELS(oq_no));
 294
 295         /* Clear PKT_CNT register */
 296         rte_write64(0xFFFFFFFFF, (uint8_t *)sdpvf->hw_addr +
 297                     SDP_VF_R_OUT_PKT_CNT(oq_no));
 298
 299         /* Clear the OQ doorbell  */
 300         rte_write32(0xFFFFFFFF, droq->pkts_credit_reg);
 301         while ((rte_read32(droq->pkts_credit_reg) != 0ull)) {
 302                 rte_write32(0xFFFFFFFF, droq->pkts_credit_reg);
 303                 rte_delay_ms(1);
 304         }
 305         otx2_sdp_dbg("SDP_R[%d]_credit:%x", oq_no,
 306                      rte_read32(droq->pkts_credit_reg));
 307
 308         /* Clear the OQ_OUT_CNTS doorbell  */
 309         reg_val = rte_read32(droq->pkts_sent_reg);
 310         rte_write32((uint32_t)reg_val, droq->pkts_sent_reg);
 311
 312         otx2_sdp_dbg("SDP_R[%d]_sent: %x", oq_no,
 313                      rte_read32(droq->pkts_sent_reg));
 314
 315         while (((rte_read32(droq->pkts_sent_reg)) != 0ull)) {
 316                 reg_val = rte_read32(droq->pkts_sent_reg);
 317                 rte_write32((uint32_t)reg_val, droq->pkts_sent_reg);
 318                 rte_delay_ms(1);
 319         }
 320
 321 }
 322
 323 static void
 324 sdp_vf_enable_iq(struct sdp_device *sdpvf, uint32_t q_no)
 325 {
 326         volatile uint64_t reg_val = 0ull;
 327         uint64_t loop = SDP_VF_BUSY_LOOP_COUNT;
 328
 329         /* Resetting doorbells during IQ enabling also to handle abrupt
 330          * guest reboot. IQ reset does not clear the doorbells.
 331          */
 332         otx2_write64(0xFFFFFFFF, sdpvf->hw_addr +
 333                      SDP_VF_R_IN_INSTR_DBELL(q_no));
 334
 335         while (((otx2_read64(sdpvf->hw_addr +
 336                  SDP_VF_R_IN_INSTR_DBELL(q_no))) != 0ull) && loop--) {
 337
 338                 rte_delay_ms(1);
 339         }
 340
 341         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
 342         reg_val |= 0x1ull;
 343
 344         otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
 345
 346         otx2_info("IQ[%d] enable done", q_no);
 347
 348 }
 349
 350 static void
 351 sdp_vf_enable_oq(struct sdp_device *sdpvf, uint32_t q_no)
 352 {
 353         volatile uint64_t reg_val = 0ull;
 354
 355         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
 356         reg_val |= 0x1ull;
 357         otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
 358
 359         otx2_info("OQ[%d] enable done", q_no);
 360 }
 361
 362 static void
 363 sdp_vf_enable_io_queues(struct sdp_device *sdpvf)
 364 {
 365         uint32_t q_no = 0;
 366
 367         for (q_no = 0; q_no < sdpvf->num_iqs; q_no++)
 368                 sdp_vf_enable_iq(sdpvf, q_no);
 369
 370         for (q_no = 0; q_no < sdpvf->num_oqs; q_no++)
 371                 sdp_vf_enable_oq(sdpvf, q_no);
 372 }
 373
 374 static void
 375 sdp_vf_disable_iq(struct sdp_device *sdpvf, uint32_t q_no)
 376 {
 377         volatile uint64_t reg_val = 0ull;
 378
 379         /* Reset the doorbell register for this Input Queue. */
 380         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
 381         reg_val &= ~0x1ull;
 382
 383         otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_IN_ENABLE(q_no));
 384 }
 385
 386 static void
 387 sdp_vf_disable_oq(struct sdp_device *sdpvf, uint32_t q_no)
 388 {
 389         volatile uint64_t reg_val = 0ull;
 390
 391         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
 392         reg_val &= ~0x1ull;
 393
 394         otx2_write64(reg_val, sdpvf->hw_addr + SDP_VF_R_OUT_ENABLE(q_no));
 395
 396 }
 397
 398 static void
 399 sdp_vf_disable_io_queues(struct sdp_device *sdpvf)
 400 {
 401         uint32_t q_no = 0;
 402
 403         /* Disable Input Queues. */
 404         for (q_no = 0; q_no < sdpvf->num_iqs; q_no++)
 405                 sdp_vf_disable_iq(sdpvf, q_no);
 406
 407         /* Disable Output Queues. */
 408         for (q_no = 0; q_no < sdpvf->num_oqs; q_no++)
 409                 sdp_vf_disable_oq(sdpvf, q_no);
 410 }
 411
 412 static uint32_t
 413 sdp_vf_update_read_index(struct sdp_instr_queue *iq)
 414 {
 415         uint32_t new_idx = rte_read32(iq->inst_cnt_reg);
 416
 417         /* The new instr cnt reg is a 32-bit counter that can roll over.
 418          * We have noted the counter's initial value at init time into
 419          * reset_instr_cnt
 420          */
 421         if (iq->reset_instr_cnt < new_idx)
 422                 new_idx -= iq->reset_instr_cnt;
 423         else
 424                 new_idx += (0xffffffff - iq->reset_instr_cnt) + 1;
 425
 426         /* Modulo of the new index with the IQ size will give us
 427          * the new index.
 428          */
 429         new_idx %= iq->nb_desc;
 430
 431         return new_idx;
 432 }
 433
 434 int
 435 sdp_vf_setup_device(struct sdp_device *sdpvf)
 436 {
 437         uint64_t reg_val = 0ull;
 438
 439         /* If application doesn't provide its conf, use driver default conf */
 440         if (sdpvf->conf == NULL) {
 441                 sdpvf->conf = sdp_get_defconf(sdpvf);
 442                 if (sdpvf->conf == NULL) {
 443                         otx2_err("SDP VF default config not found");
 444                         return -ENOMEM;
 445                 }
 446                 otx2_info("Default config is used");
 447         }
 448
 449         /* Get IOQs (RPVF] count */
 450         reg_val = otx2_read64(sdpvf->hw_addr + SDP_VF_R_IN_CONTROL(0));
 451
 452         sdpvf->sriov_info.rings_per_vf = ((reg_val >> SDP_VF_R_IN_CTL_RPVF_POS)
 453                                           & SDP_VF_R_IN_CTL_RPVF_MASK);
 454
 455         otx2_info("SDP RPVF: %d", sdpvf->sriov_info.rings_per_vf);
 456
 457         sdpvf->fn_list.setup_iq_regs       = sdp_vf_setup_iq_regs;
 458         sdpvf->fn_list.setup_oq_regs       = sdp_vf_setup_oq_regs;
 459
 460         sdpvf->fn_list.setup_device_regs   = sdp_vf_setup_device_regs;
 461         sdpvf->fn_list.update_iq_read_idx  = sdp_vf_update_read_index;
 462
 463         sdpvf->fn_list.enable_io_queues    = sdp_vf_enable_io_queues;
 464         sdpvf->fn_list.disable_io_queues   = sdp_vf_disable_io_queues;
 465
 466         sdpvf->fn_list.enable_iq           = sdp_vf_enable_iq;
 467         sdpvf->fn_list.disable_iq          = sdp_vf_disable_iq;
 468
 469         sdpvf->fn_list.enable_oq           = sdp_vf_enable_oq;
 470         sdpvf->fn_list.disable_oq          = sdp_vf_disable_oq;
 471
 472
 473         return 0;
 474
 475 }