drivers/crypto/cnxk/cn9k_cryptodev_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include <rte_cryptodev.h>
   6 #include <rte_cryptodev_pmd.h>
   7
   8 #include "cn9k_cryptodev.h"
   9 #include "cn9k_cryptodev_ops.h"
  10 #include "cnxk_cryptodev.h"
  11 #include "cnxk_cryptodev_ops.h"
  12 #include "cnxk_se.h"
  13
  14 static __rte_always_inline int __rte_hot
  15 cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
  16                        struct cnxk_se_sess *sess,
  17                        struct cpt_inflight_req *infl_req,
  18                        struct cpt_inst_s *inst)
  19 {
  20         uint64_t cpt_op;
  21         int ret = -1;
  22
  23         cpt_op = sess->cpt_op;
  24
  25         if (cpt_op & ROC_SE_OP_CIPHER_MASK)
  26                 ret = fill_fc_params(op, sess, &qp->meta_info, infl_req, inst);
  27         else
  28                 ret = fill_digest_params(op, sess, &qp->meta_info, infl_req,
  29                                          inst);
  30
  31         return ret;
  32 }
  33
  34 static inline struct cnxk_se_sess *
  35 cn9k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
  36 {
  37         const int driver_id = cn9k_cryptodev_driver_id;
  38         struct rte_crypto_sym_op *sym_op = op->sym;
  39         struct rte_cryptodev_sym_session *sess;
  40         struct cnxk_se_sess *priv;
  41         int ret;
  42
  43         /* Create temporary session */
  44         sess = rte_cryptodev_sym_session_create(qp->sess_mp);
  45         if (sess == NULL)
  46                 return NULL;
  47
  48         ret = sym_session_configure(qp->lf.roc_cpt, driver_id, sym_op->xform,
  49                                     sess, qp->sess_mp_priv);
  50         if (ret)
  51                 goto sess_put;
  52
  53         priv = get_sym_session_private_data(sess, driver_id);
  54
  55         sym_op->session = sess;
  56
  57         return priv;
  58
  59 sess_put:
  60         rte_mempool_put(qp->sess_mp, sess);
  61         return NULL;
  62 }
  63
  64 static uint16_t
  65 cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
  66 {
  67         struct cpt_inflight_req *infl_req;
  68         struct rte_crypto_sym_op *sym_op;
  69         uint16_t nb_allowed, count = 0;
  70         struct cnxk_cpt_qp *qp = qptr;
  71         struct pending_queue *pend_q;
  72         struct cnxk_se_sess *sess;
  73         struct rte_crypto_op *op;
  74         struct cpt_inst_s inst;
  75         uint64_t lmt_status;
  76         uint64_t lmtline;
  77         uint64_t io_addr;
  78         int ret;
  79
  80         pend_q = &qp->pend_q;
  81
  82         lmtline = qp->lmtline.lmt_base;
  83         io_addr = qp->lmtline.io_addr;
  84
  85         inst.w0.u64 = 0;
  86         inst.w2.u64 = 0;
  87         inst.w3.u64 = 0;
  88
  89         nb_allowed = qp->lf.nb_desc - pend_q->pending_count;
  90         nb_ops = RTE_MIN(nb_ops, nb_allowed);
  91
  92         for (count = 0; count < nb_ops; count++) {
  93                 op = ops[count];
  94                 infl_req = &pend_q->req_queue[pend_q->enq_tail];
  95                 infl_req->op_flags = 0;
  96
  97                 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
  98                         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
  99                                 sym_op = op->sym;
 100                                 sess = get_sym_session_private_data(
 101                                         sym_op->session,
 102                                         cn9k_cryptodev_driver_id);
 103                                 ret = cn9k_cpt_sym_inst_fill(qp, op, sess,
 104                                                              infl_req, &inst);
 105                         } else {
 106                                 sess = cn9k_cpt_sym_temp_sess_create(qp, op);
 107                                 if (unlikely(sess == NULL)) {
 108                                         plt_dp_err(
 109                                                 "Could not create temp session");
 110                                         break;
 111                                 }
 112
 113                                 ret = cn9k_cpt_sym_inst_fill(qp, op, sess,
 114                                                              infl_req, &inst);
 115                                 if (unlikely(ret)) {
 116                                         sym_session_clear(
 117                                                 cn9k_cryptodev_driver_id,
 118                                                 op->sym->session);
 119                                         rte_mempool_put(qp->sess_mp,
 120                                                         op->sym->session);
 121                                 }
 122                         }
 123                 } else {
 124                         plt_dp_err("Unsupported op type");
 125                         break;
 126                 }
 127
 128                 if (unlikely(ret)) {
 129                         plt_dp_err("Could not process op: %p", op);
 130                         break;
 131                 }
 132
 133                 infl_req->cop = op;
 134
 135                 infl_req->res.cn9k.compcode = CPT_COMP_NOT_DONE;
 136                 inst.res_addr = (uint64_t)&infl_req->res;
 137                 inst.w7.u64 = sess->cpt_inst_w7;
 138
 139                 do {
 140                         /* Copy CPT command to LMTLINE */
 141                         memcpy((void *)lmtline, &inst, sizeof(inst));
 142
 143                         /*
 144                          * Make sure compiler does not reorder memcpy and ldeor.
 145                          * LMTST transactions are always flushed from the write
 146                          * buffer immediately, a DMB is not required to push out
 147                          * LMTSTs.
 148                          */
 149                         rte_io_wmb();
 150                         lmt_status = roc_lmt_submit_ldeor(io_addr);
 151                 } while (lmt_status == 0);
 152
 153                 MOD_INC(pend_q->enq_tail, qp->lf.nb_desc);
 154         }
 155
 156         pend_q->pending_count += count;
 157         pend_q->time_out = rte_get_timer_cycles() +
 158                            DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
 159
 160         return count;
 161 }
 162
 163 static inline void
 164 cn9k_cpt_dequeue_post_process(struct cnxk_cpt_qp *qp, struct rte_crypto_op *cop,
 165                               struct cpt_inflight_req *infl_req)
 166 {
 167         struct cpt_cn9k_res_s *res = (struct cpt_cn9k_res_s *)&infl_req->res;
 168         unsigned int sz;
 169
 170         if (likely(res->compcode == CPT_COMP_GOOD)) {
 171                 if (unlikely(res->uc_compcode)) {
 172                         if (res->uc_compcode == ROC_SE_ERR_GC_ICV_MISCOMPARE)
 173                                 cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 174                         else
 175                                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 176
 177                         plt_dp_info("Request failed with microcode error");
 178                         plt_dp_info("MC completion code 0x%x",
 179                                     res->uc_compcode);
 180                         goto temp_sess_free;
 181                 }
 182
 183                 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 184                 if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 185                         /* Verify authentication data if required */
 186                         if (unlikely(infl_req->op_flags &
 187                                      CPT_OP_FLAGS_AUTH_VERIFY)) {
 188                                 uintptr_t *rsp = infl_req->mdata;
 189                                 compl_auth_verify(cop, (uint8_t *)rsp[0],
 190                                                   rsp[1]);
 191                         }
 192                 }
 193         } else {
 194                 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 195                 plt_dp_info("HW completion code 0x%x", res->compcode);
 196
 197                 switch (res->compcode) {
 198                 case CPT_COMP_INSTERR:
 199                         plt_dp_err("Request failed with instruction error");
 200                         break;
 201                 case CPT_COMP_FAULT:
 202                         plt_dp_err("Request failed with DMA fault");
 203                         break;
 204                 case CPT_COMP_HWERR:
 205                         plt_dp_err("Request failed with hardware error");
 206                         break;
 207                 default:
 208                         plt_dp_err(
 209                                 "Request failed with unknown completion code");
 210                 }
 211         }
 212
 213 temp_sess_free:
 214         if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
 215                 if (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
 216                         sym_session_clear(cn9k_cryptodev_driver_id,
 217                                           cop->sym->session);
 218                         sz = rte_cryptodev_sym_get_existing_header_session_size(
 219                                 cop->sym->session);
 220                         memset(cop->sym->session, 0, sz);
 221                         rte_mempool_put(qp->sess_mp, cop->sym->session);
 222                         cop->sym->session = NULL;
 223                 }
 224         }
 225 }
 226
 227 static uint16_t
 228 cn9k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 229 {
 230         struct cnxk_cpt_qp *qp = qptr;
 231         struct pending_queue *pend_q;
 232         struct cpt_inflight_req *infl_req;
 233         struct cpt_cn9k_res_s *res;
 234         struct rte_crypto_op *cop;
 235         uint32_t pq_deq_head;
 236         int i;
 237
 238         pend_q = &qp->pend_q;
 239
 240         nb_ops = RTE_MIN(nb_ops, pend_q->pending_count);
 241
 242         pq_deq_head = pend_q->deq_head;
 243
 244         for (i = 0; i < nb_ops; i++) {
 245                 infl_req = &pend_q->req_queue[pq_deq_head];
 246
 247                 res = (struct cpt_cn9k_res_s *)&infl_req->res;
 248
 249                 if (unlikely(res->compcode == CPT_COMP_NOT_DONE)) {
 250                         if (unlikely(rte_get_timer_cycles() >
 251                                      pend_q->time_out)) {
 252                                 plt_err("Request timed out");
 253                                 pend_q->time_out = rte_get_timer_cycles() +
 254                                                    DEFAULT_COMMAND_TIMEOUT *
 255                                                            rte_get_timer_hz();
 256                         }
 257                         break;
 258                 }
 259
 260                 MOD_INC(pq_deq_head, qp->lf.nb_desc);
 261
 262                 cop = infl_req->cop;
 263
 264                 ops[i] = cop;
 265
 266                 cn9k_cpt_dequeue_post_process(qp, cop, infl_req);
 267
 268                 if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF))
 269                         rte_mempool_put(qp->meta_info.pool, infl_req->mdata);
 270         }
 271
 272         pend_q->pending_count -= i;
 273         pend_q->deq_head = pq_deq_head;
 274
 275         return i;
 276 }
 277 void
 278 cn9k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev)
 279 {
 280         dev->enqueue_burst = cn9k_cpt_enqueue_burst;
 281         dev->dequeue_burst = cn9k_cpt_dequeue_burst;
 282
 283         rte_mb();
 284 }
 285
 286 static void
 287 cn9k_cpt_dev_info_get(struct rte_cryptodev *dev,
 288                       struct rte_cryptodev_info *info)
 289 {
 290         if (info != NULL) {
 291                 cnxk_cpt_dev_info_get(dev, info);
 292                 info->driver_id = cn9k_cryptodev_driver_id;
 293         }
 294 }
 295
 296 struct rte_cryptodev_ops cn9k_cpt_ops = {
 297         /* Device control ops */
 298         .dev_configure = cnxk_cpt_dev_config,
 299         .dev_start = cnxk_cpt_dev_start,
 300         .dev_stop = cnxk_cpt_dev_stop,
 301         .dev_close = cnxk_cpt_dev_close,
 302         .dev_infos_get = cn9k_cpt_dev_info_get,
 303
 304         .stats_get = NULL,
 305         .stats_reset = NULL,
 306         .queue_pair_setup = cnxk_cpt_queue_pair_setup,
 307         .queue_pair_release = cnxk_cpt_queue_pair_release,
 308
 309         /* Symmetric crypto ops */
 310         .sym_session_get_size = cnxk_cpt_sym_session_get_size,
 311         .sym_session_configure = cnxk_cpt_sym_session_configure,
 312         .sym_session_clear = cnxk_cpt_sym_session_clear,
 313
 314         /* Asymmetric crypto ops */
 315         .asym_session_get_size = NULL,
 316         .asym_session_configure = NULL,
 317         .asym_session_clear = NULL,
 318
 319 };