drivers/crypto/octeontx/otx_cryptodev_hw_access.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Cavium, Inc
   3  */
   4 #ifndef _OTX_CRYPTODEV_HW_ACCESS_H_
   5 #define _OTX_CRYPTODEV_HW_ACCESS_H_
   6
   7 #include <stdbool.h>
   8
   9 #include <rte_branch_prediction.h>
  10 #include <rte_cryptodev.h>
  11 #include <rte_cycles.h>
  12 #include <rte_io.h>
  13 #include <rte_memory.h>
  14 #include <rte_prefetch.h>
  15
  16 #include "otx_cryptodev.h"
  17
  18 #include "cpt_common.h"
  19 #include "cpt_hw_types.h"
  20 #include "cpt_mcode_defines.h"
  21 #include "cpt_pmd_logs.h"
  22
  23 #define CPT_INTR_POLL_INTERVAL_MS       (50)
  24
  25 /* Default command queue length */
  26 #define DEFAULT_CMD_QLEN        2048
  27 #define DEFAULT_CMD_QCHUNKS     2
  28
  29 /* Instruction memory benefits from being 1023, so introduce
  30  * reserved entries so we can't overrun the instruction queue
  31  */
  32 #define DEFAULT_CMD_QRSVD_SLOTS DEFAULT_CMD_QCHUNKS
  33 #define DEFAULT_CMD_QCHUNK_SIZE \
  34                 ((DEFAULT_CMD_QLEN - DEFAULT_CMD_QRSVD_SLOTS) / \
  35                 DEFAULT_CMD_QCHUNKS)
  36
  37 #define CPT_CSR_REG_BASE(cpt)           ((cpt)->reg_base)
  38
  39 /* Read hw register */
  40 #define CPT_READ_CSR(__hw_addr, __offset) \
  41         rte_read64_relaxed((uint8_t *)__hw_addr + __offset)
  42
  43 /* Write hw register */
  44 #define CPT_WRITE_CSR(__hw_addr, __offset, __val) \
  45         rte_write64_relaxed((__val), ((uint8_t *)__hw_addr + __offset))
  46
  47 /* cpt instance */
  48 struct cpt_instance {
  49         uint32_t queue_id;
  50         uintptr_t rsvd;
  51         struct rte_mempool *sess_mp;
  52         struct rte_mempool *sess_mp_priv;
  53         struct cpt_qp_meta_info meta_info;
  54         uint8_t ca_enabled;
  55 };
  56
  57 struct command_chunk {
  58         /** 128-byte aligned real_vaddr */
  59         uint8_t *head;
  60         /** 128-byte aligned real_dma_addr */
  61         phys_addr_t dma_addr;
  62 };
  63
  64 /**
  65  * Command queue structure
  66  */
  67 struct command_queue {
  68         /** Command queue host write idx */
  69         uint32_t idx;
  70         /** Command queue chunk */
  71         uint32_t cchunk;
  72         /** Command queue head; instructions are inserted here */
  73         uint8_t *qhead;
  74         /** Command chunk list head */
  75         struct command_chunk chead[DEFAULT_CMD_QCHUNKS];
  76 };
  77
  78 /**
  79  * CPT VF device structure
  80  */
  81 struct cpt_vf {
  82         /** CPT instance */
  83         struct cpt_instance instance;
  84         /** Register start address */
  85         uint8_t *reg_base;
  86         /** Command queue information */
  87         struct command_queue cqueue;
  88         /** Pending queue information */
  89         struct pending_queue pqueue;
  90
  91         /** Below fields are accessed only in control path */
  92
  93         /** Env specific pdev representing the pci dev */
  94         void *pdev;
  95         /** Calculated queue size */
  96         uint32_t qsize;
  97         /** Device index (0...CPT_MAX_VQ_NUM)*/
  98         uint8_t  vfid;
  99         /** VF type of cpt_vf_type_t (SE_TYPE(2) or AE_TYPE(1) */
 100         uint8_t  vftype;
 101         /** VF group (0 - 8) */
 102         uint8_t  vfgrp;
 103         /** Operating node: Bits (46:44) in BAR0 address */
 104         uint8_t  node;
 105
 106         /** VF-PF mailbox communication */
 107
 108         /** Flag if acked */
 109         bool pf_acked;
 110         /** Flag if not acked */
 111         bool pf_nacked;
 112
 113         /** Device name */
 114         char dev_name[32];
 115 } __rte_cache_aligned;
 116
 117 /*
 118  * CPT Registers map for 81xx
 119  */
 120
 121 /* VF registers */
 122 #define CPTX_VQX_CTL(a, b)              (0x0000100ll + 0x1000000000ll * \
 123                                          ((a) & 0x0) + 0x100000ll * (b))
 124 #define CPTX_VQX_SADDR(a, b)            (0x0000200ll + 0x1000000000ll * \
 125                                          ((a) & 0x0) + 0x100000ll * (b))
 126 #define CPTX_VQX_DONE_WAIT(a, b)        (0x0000400ll + 0x1000000000ll * \
 127                                          ((a) & 0x0) + 0x100000ll * (b))
 128 #define CPTX_VQX_INPROG(a, b)           (0x0000410ll + 0x1000000000ll * \
 129                                          ((a) & 0x0) + 0x100000ll * (b))
 130 #define CPTX_VQX_DONE(a, b)             (0x0000420ll + 0x1000000000ll * \
 131                                          ((a) & 0x1) + 0x100000ll * (b))
 132 #define CPTX_VQX_DONE_ACK(a, b)         (0x0000440ll + 0x1000000000ll * \
 133                                          ((a) & 0x1) + 0x100000ll * (b))
 134 #define CPTX_VQX_DONE_INT_W1S(a, b)     (0x0000460ll + 0x1000000000ll * \
 135                                          ((a) & 0x1) + 0x100000ll * (b))
 136 #define CPTX_VQX_DONE_INT_W1C(a, b)     (0x0000468ll + 0x1000000000ll * \
 137                                          ((a) & 0x1) + 0x100000ll * (b))
 138 #define CPTX_VQX_DONE_ENA_W1S(a, b)     (0x0000470ll + 0x1000000000ll * \
 139                                          ((a) & 0x1) + 0x100000ll * (b))
 140 #define CPTX_VQX_DONE_ENA_W1C(a, b)     (0x0000478ll + 0x1000000000ll * \
 141                                          ((a) & 0x1) + 0x100000ll * (b))
 142 #define CPTX_VQX_MISC_INT(a, b)         (0x0000500ll + 0x1000000000ll * \
 143                                          ((a) & 0x1) + 0x100000ll * (b))
 144 #define CPTX_VQX_MISC_INT_W1S(a, b)     (0x0000508ll + 0x1000000000ll * \
 145                                          ((a) & 0x1) + 0x100000ll * (b))
 146 #define CPTX_VQX_MISC_ENA_W1S(a, b)     (0x0000510ll + 0x1000000000ll * \
 147                                          ((a) & 0x1) + 0x100000ll * (b))
 148 #define CPTX_VQX_MISC_ENA_W1C(a, b)     (0x0000518ll + 0x1000000000ll * \
 149                                          ((a) & 0x1) + 0x100000ll * (b))
 150 #define CPTX_VQX_DOORBELL(a, b)         (0x0000600ll + 0x1000000000ll * \
 151                                          ((a) & 0x1) + 0x100000ll * (b))
 152 #define CPTX_VFX_PF_MBOXX(a, b, c)      (0x0001000ll + 0x1000000000ll * \
 153                                          ((a) & 0x1) + 0x100000ll * (b) + \
 154                                          8ll * ((c) & 0x1))
 155
 156 /* VF HAL functions */
 157
 158 void
 159 otx_cpt_poll_misc(struct cpt_vf *cptvf);
 160
 161 int
 162 otx_cpt_hw_init(struct cpt_vf *cptvf, void *pdev, void *reg_base, char *name);
 163
 164 int
 165 otx_cpt_deinit_device(void *dev);
 166
 167 int
 168 otx_cpt_get_resource(const struct rte_cryptodev *dev, uint8_t group,
 169                      struct cpt_instance **instance, uint16_t qp_id);
 170
 171 int
 172 otx_cpt_put_resource(struct cpt_instance *instance);
 173
 174 int
 175 otx_cpt_start_device(void *cptvf);
 176
 177 void
 178 otx_cpt_stop_device(void *cptvf);
 179
 180 /* Write to VQX_DOORBELL register
 181  */
 182 static __rte_always_inline void
 183 otx_cpt_write_vq_doorbell(struct cpt_vf *cptvf, uint32_t val)
 184 {
 185         cptx_vqx_doorbell_t vqx_dbell;
 186
 187         vqx_dbell.u = 0;
 188         vqx_dbell.s.dbell_cnt = val * 8; /* Num of Instructions * 8 words */
 189         CPT_WRITE_CSR(CPT_CSR_REG_BASE(cptvf),
 190                       CPTX_VQX_DOORBELL(0, 0), vqx_dbell.u);
 191 }
 192
 193 static __rte_always_inline uint32_t
 194 otx_cpt_read_vq_doorbell(struct cpt_vf *cptvf)
 195 {
 196         cptx_vqx_doorbell_t vqx_dbell;
 197
 198         vqx_dbell.u = CPT_READ_CSR(CPT_CSR_REG_BASE(cptvf),
 199                                    CPTX_VQX_DOORBELL(0, 0));
 200         return vqx_dbell.s.dbell_cnt;
 201 }
 202
 203 static __rte_always_inline void
 204 otx_cpt_ring_dbell(struct cpt_instance *instance, uint16_t count)
 205 {
 206         struct cpt_vf *cptvf = (struct cpt_vf *)instance;
 207         /* Memory barrier to flush pending writes */
 208         rte_smp_wmb();
 209         otx_cpt_write_vq_doorbell(cptvf, count);
 210 }
 211
 212 static __rte_always_inline void *
 213 get_cpt_inst(struct command_queue *cqueue)
 214 {
 215         CPT_LOG_DP_DEBUG("CPT queue idx %u\n", cqueue->idx);
 216         return &cqueue->qhead[cqueue->idx * CPT_INST_SIZE];
 217 }
 218
 219 static __rte_always_inline void
 220 fill_cpt_inst(struct cpt_instance *instance, void *req, uint64_t ucmd_w3)
 221 {
 222         struct command_queue *cqueue;
 223         cpt_inst_s_t *cpt_ist_p;
 224         struct cpt_vf *cptvf = (struct cpt_vf *)instance;
 225         struct cpt_request_info *user_req = (struct cpt_request_info *)req;
 226         cqueue = &cptvf->cqueue;
 227         cpt_ist_p = get_cpt_inst(cqueue);
 228         rte_prefetch_non_temporal(cpt_ist_p);
 229
 230         /* EI0, EI1, EI2, EI3 are already prepared */
 231         /* HW W0 */
 232         cpt_ist_p->u[0] = 0;
 233         /* HW W1 */
 234         cpt_ist_p->s8x.res_addr = user_req->comp_baddr;
 235         /* HW W2 */
 236         cpt_ist_p->u[2] = 0;
 237         /* HW W3 */
 238         cpt_ist_p->s8x.wq_ptr = 0;
 239
 240         /* MC EI0 */
 241         cpt_ist_p->s8x.ei0 = user_req->ist.ei0;
 242         /* MC EI1 */
 243         cpt_ist_p->s8x.ei1 = user_req->ist.ei1;
 244         /* MC EI2 */
 245         cpt_ist_p->s8x.ei2 = user_req->ist.ei2;
 246         /* MC EI3 */
 247         cpt_ist_p->s8x.ei3 = ucmd_w3;
 248 }
 249
 250 static __rte_always_inline void
 251 mark_cpt_inst(struct cpt_instance *instance)
 252 {
 253         struct cpt_vf *cptvf = (struct cpt_vf *)instance;
 254         struct command_queue *queue = &cptvf->cqueue;
 255         if (unlikely(++queue->idx >= DEFAULT_CMD_QCHUNK_SIZE)) {
 256                 uint32_t cchunk = queue->cchunk;
 257                 MOD_INC(cchunk, DEFAULT_CMD_QCHUNKS);
 258                 queue->qhead = queue->chead[cchunk].head;
 259                 queue->idx = 0;
 260                 queue->cchunk = cchunk;
 261         }
 262 }
 263
 264 static __rte_always_inline uint8_t
 265 check_nb_command_id(struct cpt_request_info *user_req,
 266                 struct cpt_instance *instance)
 267 {
 268         uint8_t ret = ERR_REQ_PENDING;
 269         struct cpt_vf *cptvf = (struct cpt_vf *)instance;
 270         volatile cpt_res_s_t *cptres;
 271
 272         cptres = (volatile cpt_res_s_t *)user_req->completion_addr;
 273
 274         if (unlikely(cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE)) {
 275                 /*
 276                  * Wait for some time for this command to get completed
 277                  * before timing out
 278                  */
 279                 if (rte_get_timer_cycles() < user_req->time_out)
 280                         return ret;
 281                 /*
 282                  * TODO: See if alternate caddr can be used to not loop
 283                  * longer than needed.
 284                  */
 285                 if ((cptres->s8x.compcode == CPT_8X_COMP_E_NOTDONE) &&
 286                     (user_req->extra_time < TIME_IN_RESET_COUNT)) {
 287                         user_req->extra_time++;
 288                         return ret;
 289                 }
 290
 291                 if (cptres->s8x.compcode != CPT_8X_COMP_E_NOTDONE)
 292                         goto complete;
 293
 294                 ret = ERR_REQ_TIMEOUT;
 295                 CPT_LOG_DP_ERR("Request %p timedout", user_req);
 296                 otx_cpt_poll_misc(cptvf);
 297                 goto exit;
 298         }
 299
 300 complete:
 301         if (likely(cptres->s8x.compcode == CPT_8X_COMP_E_GOOD)) {
 302                 ret = 0; /* success */
 303                 if (unlikely((uint8_t)*user_req->alternate_caddr)) {
 304                         ret = (uint8_t)*user_req->alternate_caddr;
 305                         CPT_LOG_DP_ERR("Request %p : failed with microcode"
 306                                 " error, MC completion code : 0x%x", user_req,
 307                                 ret);
 308                 }
 309                 CPT_LOG_DP_DEBUG("MC status %.8x\n",
 310                            *((volatile uint32_t *)user_req->alternate_caddr));
 311                 CPT_LOG_DP_DEBUG("HW status %.8x\n",
 312                            *((volatile uint32_t *)user_req->completion_addr));
 313         } else if ((cptres->s8x.compcode == CPT_8X_COMP_E_SWERR) ||
 314                    (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT)) {
 315                 ret = (uint8_t)*user_req->alternate_caddr;
 316                 if (!ret)
 317                         ret = ERR_BAD_ALT_CCODE;
 318                 CPT_LOG_DP_DEBUG("Request %p : failed with %s : err code :%x",
 319                            user_req,
 320                            (cptres->s8x.compcode == CPT_8X_COMP_E_FAULT) ?
 321                            "DMA Fault" : "Software error", ret);
 322         } else {
 323                 CPT_LOG_DP_ERR("Request %p : unexpected completion code %d",
 324                            user_req, cptres->s8x.compcode);
 325                 ret = (uint8_t)*user_req->alternate_caddr;
 326         }
 327
 328 exit:
 329         return ret;
 330 }
 331
 332 #endif /* _OTX_CRYPTODEV_HW_ACCESS_H_ */