1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2022 Intel Corporation
8 #include <cryptodev_pmd.h>
9 #ifdef RTE_LIB_SECURITY
10 #include <rte_net_crc.h>
14 #include <openssl/evp.h>
16 #include "qat_common.h"
17 #include "qat_sym_session.h"
18 #include "qat_crypto.h"
22 /* bpi is only used for partial blocks of DES and AES
23 * so AES block len can be assumed as max len for iv, src and dst
25 #define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
27 /** Intel(R) QAT Symmetric Crypto PMD name */
28 #define CRYPTODEV_NAME_QAT_SYM_PMD crypto_qat
30 /* Internal capabilities */
31 #define QAT_SYM_CAP_MIXED_CRYPTO (1 << 0)
32 #define QAT_SYM_CAP_VALID (1 << 31)
35 * Macro to add a sym capability
36 * helper function to add an sym capability
37 * <n: name> <b: block size> <k: key size> <d: digest size>
38 * <a: aad_size> <i: iv_size>
40 #define QAT_SYM_PLAIN_AUTH_CAP(n, b, d) \
42 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \
44 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \
46 .algo = RTE_CRYPTO_AUTH_##n, \
52 #define QAT_SYM_AUTH_CAP(n, b, k, d, a, i) \
54 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \
56 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH, \
58 .algo = RTE_CRYPTO_AUTH_##n, \
64 #define QAT_SYM_AEAD_CAP(n, b, k, d, a, i) \
66 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \
68 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD, \
70 .algo = RTE_CRYPTO_AEAD_##n, \
76 #define QAT_SYM_CIPHER_CAP(n, b, k, i) \
78 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC, \
80 .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER, \
82 .algo = RTE_CRYPTO_CIPHER_##n, \
89 * Maximum number of SGL entries
91 #define QAT_SYM_SGL_MAX_NUMBER 16
93 /* Maximum data length for single pass GMAC: 2^14-1 */
94 #define QAT_AES_GMAC_SPC_MAX_SIZE 16383
96 struct qat_sym_session;
100 struct qat_flat_buf buffers[QAT_SYM_SGL_MAX_NUMBER];
101 } __rte_packed __rte_cache_aligned;
103 struct qat_sym_op_cookie {
104 struct qat_sym_sgl qat_sgl_src;
105 struct qat_sym_sgl qat_sgl_dst;
106 phys_addr_t qat_sgl_src_phys_addr;
107 phys_addr_t qat_sgl_dst_phys_addr;
109 /* Used for Single-Pass AES-GMAC only */
111 struct icp_qat_hw_cipher_algo_blk cd_cipher
112 __rte_packed __rte_cache_aligned;
113 phys_addr_t cd_phys_addr;
118 struct qat_sym_dp_ctx {
119 struct qat_sym_session *session;
122 uint16_t cached_enqueue;
123 uint16_t cached_dequeue;
127 qat_sym_enqueue_burst(void *qp, struct rte_crypto_op **ops,
131 qat_sym_dequeue_burst(void *qp, struct rte_crypto_op **ops,
135 qat_sym_build_request(void *in_op, uint8_t *out_msg,
136 void *op_cookie, enum qat_device_gen qat_dev_gen);
139 /** Encrypt a single partial block
140 * Depends on openssl libcrypto
141 * Uses ECB+XOR to do CFB encryption, same result, more performant
144 bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
145 uint8_t *iv, int ivlen, int srclen,
148 EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
150 uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
151 uint8_t *encr = encrypted_iv;
153 /* ECB method: encrypt the IV, then XOR this with plaintext */
154 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
156 goto cipher_encrypt_err;
158 for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
164 QAT_DP_LOG(ERR, "libcrypto ECB cipher encrypt failed");
168 static inline uint32_t
169 qat_bpicipher_postprocess(struct qat_sym_session *ctx,
170 struct rte_crypto_op *op)
172 int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
173 struct rte_crypto_sym_op *sym_op = op->sym;
174 uint8_t last_block_len = block_len > 0 ?
175 sym_op->cipher.data.length % block_len : 0;
177 if (last_block_len > 0 &&
178 ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
180 /* Encrypt last block */
181 uint8_t *last_block, *dst, *iv;
182 uint32_t last_block_offset;
184 last_block_offset = sym_op->cipher.data.offset +
185 sym_op->cipher.data.length - last_block_len;
186 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
187 uint8_t *, last_block_offset);
189 if (unlikely(sym_op->m_dst != NULL))
190 /* out-of-place operation (OOP) */
191 dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
192 uint8_t *, last_block_offset);
196 if (last_block_len < sym_op->cipher.data.length)
197 /* use previous block ciphertext as IV */
198 iv = dst - block_len;
200 /* runt block, i.e. less than one full block */
201 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
202 ctx->cipher_iv.offset);
204 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
205 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before post-process:",
206 last_block, last_block_len);
207 if (sym_op->m_dst != NULL)
208 QAT_DP_HEXDUMP_LOG(DEBUG,
209 "BPI: dst before post-process:",
210 dst, last_block_len);
212 bpi_cipher_encrypt(last_block, dst, iv, block_len,
213 last_block_len, ctx->bpi_ctx);
214 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
215 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after post-process:",
216 last_block, last_block_len);
217 if (sym_op->m_dst != NULL)
218 QAT_DP_HEXDUMP_LOG(DEBUG,
219 "BPI: dst after post-process:",
220 dst, last_block_len);
223 return sym_op->cipher.data.length - last_block_len;
226 #ifdef RTE_LIB_SECURITY
228 qat_crc_verify(struct qat_sym_session *ctx, struct rte_crypto_op *op)
230 struct rte_crypto_sym_op *sym_op = op->sym;
231 uint32_t crc_data_ofs, crc_data_len, crc;
234 if (ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT &&
235 sym_op->auth.data.length != 0) {
237 crc_data_ofs = sym_op->auth.data.offset;
238 crc_data_len = sym_op->auth.data.length;
239 crc_data = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *,
242 crc = rte_net_crc_calc(crc_data, crc_data_len,
245 if (crc != *(uint32_t *)(crc_data + crc_data_len))
246 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
251 qat_crc_generate(struct qat_sym_session *ctx,
252 struct rte_crypto_op *op)
254 struct rte_crypto_sym_op *sym_op = op->sym;
255 uint32_t *crc, crc_data_len;
258 if (ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT &&
259 sym_op->auth.data.length != 0 &&
260 sym_op->m_src->nb_segs == 1) {
262 crc_data_len = sym_op->auth.data.length;
263 crc_data = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *,
264 sym_op->auth.data.offset);
265 crc = (uint32_t *)(crc_data + crc_data_len);
266 *crc = rte_net_crc_calc(crc_data, crc_data_len,
272 qat_sym_preprocess_requests(void **ops, uint16_t nb_ops)
274 struct rte_crypto_op *op;
275 struct qat_sym_session *ctx;
278 for (i = 0; i < nb_ops; i++) {
279 op = (struct rte_crypto_op *)ops[i];
281 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
282 ctx = (struct qat_sym_session *)
283 get_sec_session_private_data(
284 op->sym->sec_session);
286 if (ctx == NULL || ctx->bpi_ctx == NULL)
289 qat_crc_generate(ctx, op);
295 static __rte_always_inline int
296 qat_sym_process_response(void **op, uint8_t *resp, void *op_cookie,
297 uint64_t *dequeue_err_count __rte_unused)
299 struct icp_qat_fw_comn_resp *resp_msg =
300 (struct icp_qat_fw_comn_resp *)resp;
301 struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
302 (resp_msg->opaque_data);
303 struct qat_sym_session *sess;
304 uint8_t is_docsis_sec;
306 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
307 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_response:", (uint8_t *)resp_msg,
308 sizeof(struct icp_qat_fw_comn_resp));
311 #ifdef RTE_LIB_SECURITY
312 if (rx_op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
314 * Assuming at this point that if it's a security
315 * op, that this is for DOCSIS
317 sess = (struct qat_sym_session *)
318 get_sec_session_private_data(
319 rx_op->sym->sec_session);
324 sess = (struct qat_sym_session *)
325 get_sym_session_private_data(
331 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
332 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
333 resp_msg->comn_hdr.comn_status)) {
335 rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
337 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
340 qat_bpicipher_postprocess(sess, rx_op);
341 #ifdef RTE_LIB_SECURITY
343 qat_crc_verify(sess, rx_op);
348 if (sess->is_single_pass_gmac) {
349 struct qat_sym_op_cookie *cookie =
350 (struct qat_sym_op_cookie *) op_cookie;
351 memset(cookie->opt.spc_gmac.cd_cipher.key, 0,
352 sess->auth_key_length);
358 * return 1 as dequeue op only move on to the next op
359 * if one was ready to return to API
365 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
366 struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
367 enum rte_crypto_op_sess_type sess_type,
368 union rte_cryptodev_session_ctx session_ctx, uint8_t is_update);
371 qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev);
374 qat_sym_init_op_cookie(void *cookie);
376 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
377 static __rte_always_inline void
378 qat_sym_debug_log_dump(struct icp_qat_fw_la_bulk_req *qat_req,
379 struct qat_sym_session *ctx,
380 struct rte_crypto_vec *vec, uint32_t vec_len,
381 struct rte_crypto_va_iova_ptr *cipher_iv,
382 struct rte_crypto_va_iova_ptr *auth_iv,
383 struct rte_crypto_va_iova_ptr *aad,
384 struct rte_crypto_va_iova_ptr *digest)
388 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req,
389 sizeof(struct icp_qat_fw_la_bulk_req));
390 for (i = 0; i < vec_len; i++)
391 QAT_DP_HEXDUMP_LOG(DEBUG, "src_data:", vec[i].base, vec[i].len);
392 if (cipher_iv && ctx->cipher_iv.length > 0)
393 QAT_DP_HEXDUMP_LOG(DEBUG, "cipher iv:", cipher_iv->va,
394 ctx->cipher_iv.length);
395 if (auth_iv && ctx->auth_iv.length > 0)
396 QAT_DP_HEXDUMP_LOG(DEBUG, "auth iv:", auth_iv->va,
397 ctx->auth_iv.length);
398 if (aad && ctx->aad_len > 0)
399 QAT_DP_HEXDUMP_LOG(DEBUG, "aad:", aad->va,
401 if (digest && ctx->digest_length > 0)
402 QAT_DP_HEXDUMP_LOG(DEBUG, "digest:", digest->va,
406 static __rte_always_inline void
407 qat_sym_debug_log_dump(struct icp_qat_fw_la_bulk_req *qat_req __rte_unused,
408 struct qat_sym_session *ctx __rte_unused,
409 struct rte_crypto_vec *vec __rte_unused,
410 uint32_t vec_len __rte_unused,
411 struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
412 struct rte_crypto_va_iova_ptr *auth_iv __rte_unused,
413 struct rte_crypto_va_iova_ptr *aad __rte_unused,
414 struct rte_crypto_va_iova_ptr *digest __rte_unused)
422 qat_sym_preprocess_requests(void **ops __rte_unused,
423 uint16_t nb_ops __rte_unused)
428 qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
429 void *op_cookie __rte_unused)
433 #endif /* BUILD_QAT_SYM */
434 #endif /* _QAT_SYM_H_ */