drivers/crypto/cnxk/cn9k_cryptodev_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include <rte_cryptodev.h>
   6 #include <rte_cryptodev_pmd.h>
   7
   8 #include "cn9k_cryptodev.h"
   9 #include "cn9k_cryptodev_ops.h"
  10 #include "cnxk_cryptodev.h"
  11 #include "cnxk_cryptodev_ops.h"
  12 #include "cnxk_se.h"
  13
  14 static __rte_always_inline int __rte_hot
  15 cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
  16                        struct cnxk_se_sess *sess,
  17                        struct cpt_inflight_req *infl_req,
  18                        struct cpt_inst_s *inst)
  19 {
  20         uint64_t cpt_op;
  21         int ret = -1;
  22
  23         cpt_op = sess->cpt_op;
  24
  25         if (cpt_op & ROC_SE_OP_CIPHER_MASK)
  26                 ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst);
  27
  28         return ret;
  29 }
  30
  31 static inline struct cnxk_se_sess *
  32 cn9k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
  33 {
  34         const int driver_id = cn9k_cryptodev_driver_id;
  35         struct rte_crypto_sym_op *sym_op = op->sym;
  36         struct rte_cryptodev_sym_session *sess;
  37         struct cnxk_se_sess *priv;
  38         int ret;
  39
  40         /* Create temporary session */
  41         sess = rte_cryptodev_sym_session_create(qp->sess_mp);
  42         if (sess == NULL)
  43                 return NULL;
  44
  45         ret = sym_session_configure(qp->lf.roc_cpt, driver_id, sym_op->xform,
  46                                     sess, qp->sess_mp_priv);
  47         if (ret)
  48                 goto sess_put;
  49
  50         priv = get_sym_session_private_data(sess, driver_id);
  51
  52         sym_op->session = sess;
  53
  54         return priv;
  55
  56 sess_put:
  57         rte_mempool_put(qp->sess_mp, sess);
  58         return NULL;
  59 }
  60
  61 static uint16_t
  62 cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
  63 {
  64         struct cpt_inflight_req *infl_req;
  65         struct rte_crypto_sym_op *sym_op;
  66         uint16_t nb_allowed, count = 0;
  67         struct cnxk_cpt_qp *qp = qptr;
  68         struct pending_queue *pend_q;
  69         struct cnxk_se_sess *sess;
  70         struct rte_crypto_op *op;
  71         struct cpt_inst_s inst;
  72         uint64_t lmt_status;
  73         uint64_t lmtline;
  74         uint64_t io_addr;
  75         int ret;
  76
  77         pend_q = &qp->pend_q;
  78
  79         lmtline = qp->lmtline.lmt_base;
  80         io_addr = qp->lmtline.io_addr;
  81
  82         inst.w0.u64 = 0;
  83         inst.w2.u64 = 0;
  84         inst.w3.u64 = 0;
  85
  86         nb_allowed = qp->lf.nb_desc - pend_q->pending_count;
  87         nb_ops = RTE_MIN(nb_ops, nb_allowed);
  88
  89         for (count = 0; count < nb_ops; count++) {
  90                 op = ops[count];
  91                 infl_req = &pend_q->req_queue[pend_q->enq_tail];
  92                 infl_req->op_flags = 0;
  93
  94                 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
  95                         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
  96                                 sym_op = op->sym;
  97                                 sess = get_sym_session_private_data(
  98                                         sym_op->session,
  99                                         cn9k_cryptodev_driver_id);
 100                                 ret = cn9k_cpt_sym_inst_fill(qp, op, sess,
 101                                                              infl_req, &inst);
 102                         } else {
 103                                 sess = cn9k_cpt_sym_temp_sess_create(qp, op);
 104                                 if (unlikely(sess == NULL)) {
 105                                         plt_dp_err(
 106                                                 "Could not create temp session");
 107                                         break;
 108                                 }
 109
 110                                 ret = cn9k_cpt_sym_inst_fill(qp, op, sess,
 111                                                              infl_req, &inst);
 112                                 if (unlikely(ret)) {
 113                                         sym_session_clear(
 114                                                 cn9k_cryptodev_driver_id,
 115                                                 op->sym->session);
 116                                         rte_mempool_put(qp->sess_mp,
 117                                                         op->sym->session);
 118                                 }
 119                         }
 120                 } else {
 121                         plt_dp_err("Unsupported op type");
 122                         break;
 123                 }
 124
 125                 if (unlikely(ret)) {
 126                         plt_dp_err("Could not process op: %p", op);
 127                         break;
 128                 }
 129
 130                 infl_req->cop = op;
 131
 132                 infl_req->res.cn9k.compcode = CPT_COMP_NOT_DONE;
 133                 inst.res_addr = (uint64_t)&infl_req->res;
 134                 inst.w7.u64 = sess->cpt_inst_w7;
 135
 136                 do {
 137                         /* Copy CPT command to LMTLINE */
 138                         memcpy((void *)lmtline, &inst, sizeof(inst));
 139
 140                         /*
 141                          * Make sure compiler does not reorder memcpy and ldeor.
 142                          * LMTST transactions are always flushed from the write
 143                          * buffer immediately, a DMB is not required to push out
 144                          * LMTSTs.
 145                          */
 146                         rte_io_wmb();
 147                         lmt_status = roc_lmt_submit_ldeor(io_addr);
 148                 } while (lmt_status == 0);
 149
 150                 MOD_INC(pend_q->enq_tail, qp->lf.nb_desc);
 151         }
 152
 153         pend_q->pending_count += count;
 154         pend_q->time_out = rte_get_timer_cycles() +
 155                            DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
 156
 157         return count;
 158 }
 159
 160 static inline void
 161 cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop,
 162                               struct cpt_inflight_req *infl_req)
 163 {
 164         struct cpt_cn9k_res_s *res = (struct cpt_cn9k_res_s *)&infl_req->res;
 165         unsigned int sz;
 166
 167         if (likely(res->compcode == CPT_COMP_GOOD)) {
 168                 if (unlikely(res->uc_compcode)) {
 169                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 170
 171                         plt_dp_info("Request failed with microcode error");
 172                         plt_dp_info("MC completion code 0x%x",
 173                                     res->uc_compcode);
 174                         goto temp_sess_free;
 175                 }
 176
 177                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 178         } else {
 179                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 180                 plt_dp_info("HW completion code 0x%x", res->compcode);
 181
 182                 switch (res->compcode) {
 183                 case CPT_COMP_INSTERR:
 184                         plt_dp_err("Request failed with instruction error");
 185                         break;
 186                 case CPT_COMP_FAULT:
 187                         plt_dp_err("Request failed with DMA fault");
 188                         break;
 189                 case CPT_COMP_HWERR:
 190                         plt_dp_err("Request failed with hardware error");
 191                         break;
 192                 default:
 193                         plt_dp_err(
 194                                 "Request failed with unknown completion code");
 195                 }
 196         }
 197
 198 temp_sess_free:
 199         if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
 200                 if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 201                         sym_session_clear(cn9k_cryptodev_driver_id,
 202                                           cop->sym->session);
 203                         sz = rte_cryptodev_sym_get_existing_header_session_size(
 204                                 cop->sym->session);
 205                         memset(cop->sym->session, 0, sz);
 206                         rte_mempool_put(qp->sess_mp, cop->sym->session);
 207                         cop->sym->session = NULL;
 208                 }
 209         }
 210 }
 211
 212 static uint16_t
 213 cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 214 {
 215         struct cnxk_cpt_qp *qp = qptr;
 216         struct pending_queue *pend_q;
 217         struct cpt_inflight_req *infl_req;
 218         struct cpt_cn9k_res_s *res;
 219         struct rte_crypto_op *cop;
 220         uint32_t pq_deq_head;
 221         int i;
 222
 223         pend_q = &qp->pend_q;
 224
 225         nb_ops = RTE_MIN(nb_ops, pend_q->pending_count);
 226
 227         pq_deq_head = pend_q->deq_head;
 228
 229         for (i = 0; i < nb_ops; i++) {
 230                 infl_req = &pend_q->req_queue[pq_deq_head];
 231
 232                 res = (struct cpt_cn9k_res_s *)&infl_req->res;
 233
 234                 if (unlikely(res->compcode == CPT_COMP_NOT_DONE)) {
 235                         if (unlikely(rte_get_timer_cycles() >
 236                                      pend_q->time_out)) {
 237                                 plt_err("Request timed out");
 238                                 pend_q->time_out = rte_get_timer_cycles() +
 239                                                    DEFAULT_COMMAND_TIMEOUT *
 240                                                            rte_get_timer_hz();
 241                         }
 242                         break;
 243                 }
 244
 245                 MOD_INC(pq_deq_head, qp->lf.nb_desc);
 246
 247                 cop = infl_req->cop;
 248
 249                 ops[i] = cop;
 250
 251                 cn9k_cpt_dequeue_post_process(qp, cop, infl_req);
 252
 253                 if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF))
 254                         rte_mempool_put(qp->meta_info.pool, infl_req->mdata);
 255         }
 256
 257         pend_q->pending_count -= i;
 258         pend_q->deq_head = pq_deq_head;
 259
 260         return i;
 261 }
 262 void
 263 cn9k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev)
 264 {
 265         dev->enqueue_burst = cn9k_cpt_enqueue_burst;
 266         dev->dequeue_burst = cn9k_cpt_dequeue_burst;
 267
 268         rte_mb();
 269 }
 270
 271 static void
 272 cn9k_cpt_dev_info_get(struct rte_cryptodev *dev,
 273                       struct rte_cryptodev_info *info)
 274 {
 275         if (info != NULL) {
 276                 cnxk_cpt_dev_info_get(dev, info);
 277                 info->driver_id = cn9k_cryptodev_driver_id;
 278         }
 279 }
 280
 281 struct rte_cryptodev_ops cn9k_cpt_ops = {
 282         /* Device control ops */
 283         .dev_configure = cnxk_cpt_dev_config,
 284         .dev_start = cnxk_cpt_dev_start,
 285         .dev_stop = cnxk_cpt_dev_stop,
 286         .dev_close = cnxk_cpt_dev_close,
 287         .dev_infos_get = cn9k_cpt_dev_info_get,
 288
 289         .stats_get = NULL,
 290         .stats_reset = NULL,
 291         .queue_pair_setup = cnxk_cpt_queue_pair_setup,
 292         .queue_pair_release = cnxk_cpt_queue_pair_release,
 293
 294         /* Symmetric crypto ops */
 295         .sym_session_get_size = cnxk_cpt_sym_session_get_size,
 296         .sym_session_configure = cnxk_cpt_sym_session_configure,
 297         .sym_session_clear = cnxk_cpt_sym_session_clear,
 298
 299         /* Asymmetric crypto ops */
 300         .asym_session_get_size = NULL,
 301         .asym_session_configure = NULL,
 302         .asym_session_clear = NULL,
 303
 304 };