1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015-2018 Intel Corporation
8 #include <cryptodev_pmd.h>
9 #ifdef RTE_LIB_SECURITY
10 #include <rte_net_crc.h>
14 #include <openssl/evp.h>
16 #include "qat_common.h"
17 #include "qat_sym_session.h"
18 #include "qat_sym_pmd.h"
22 /* bpi is only used for partial blocks of DES and AES
23 * so AES block len can be assumed as max len for iv, src and dst
25 #define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
28 * Maximum number of SGL entries
30 #define QAT_SYM_SGL_MAX_NUMBER 16
32 /* Maximum data length for single pass GMAC: 2^14-1 */
33 #define QAT_AES_GMAC_SPC_MAX_SIZE 16383
35 struct qat_sym_session;
39 struct qat_flat_buf buffers[QAT_SYM_SGL_MAX_NUMBER];
40 } __rte_packed __rte_cache_aligned;
42 struct qat_sym_op_cookie {
43 struct qat_sym_sgl qat_sgl_src;
44 struct qat_sym_sgl qat_sgl_dst;
45 phys_addr_t qat_sgl_src_phys_addr;
46 phys_addr_t qat_sgl_dst_phys_addr;
48 /* Used for Single-Pass AES-GMAC only */
50 struct icp_qat_hw_cipher_algo_blk cd_cipher
51 __rte_packed __rte_cache_aligned;
52 phys_addr_t cd_phys_addr;
58 qat_sym_build_request(void *in_op, uint8_t *out_msg,
59 void *op_cookie, enum qat_device_gen qat_dev_gen);
62 /** Encrypt a single partial block
63 * Depends on openssl libcrypto
64 * Uses ECB+XOR to do CFB encryption, same result, more performant
67 bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
68 uint8_t *iv, int ivlen, int srclen,
71 EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
73 uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
74 uint8_t *encr = encrypted_iv;
76 /* ECB method: encrypt the IV, then XOR this with plaintext */
77 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
79 goto cipher_encrypt_err;
81 for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
87 QAT_DP_LOG(ERR, "libcrypto ECB cipher encrypt failed");
91 static inline uint32_t
92 qat_bpicipher_postprocess(struct qat_sym_session *ctx,
93 struct rte_crypto_op *op)
95 int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
96 struct rte_crypto_sym_op *sym_op = op->sym;
97 uint8_t last_block_len = block_len > 0 ?
98 sym_op->cipher.data.length % block_len : 0;
100 if (last_block_len > 0 &&
101 ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
103 /* Encrypt last block */
104 uint8_t *last_block, *dst, *iv;
105 uint32_t last_block_offset;
107 last_block_offset = sym_op->cipher.data.offset +
108 sym_op->cipher.data.length - last_block_len;
109 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
110 uint8_t *, last_block_offset);
112 if (unlikely(sym_op->m_dst != NULL))
113 /* out-of-place operation (OOP) */
114 dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
115 uint8_t *, last_block_offset);
119 if (last_block_len < sym_op->cipher.data.length)
120 /* use previous block ciphertext as IV */
121 iv = dst - block_len;
123 /* runt block, i.e. less than one full block */
124 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
125 ctx->cipher_iv.offset);
127 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
128 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before post-process:",
129 last_block, last_block_len);
130 if (sym_op->m_dst != NULL)
131 QAT_DP_HEXDUMP_LOG(DEBUG,
132 "BPI: dst before post-process:",
133 dst, last_block_len);
135 bpi_cipher_encrypt(last_block, dst, iv, block_len,
136 last_block_len, ctx->bpi_ctx);
137 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
138 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after post-process:",
139 last_block, last_block_len);
140 if (sym_op->m_dst != NULL)
141 QAT_DP_HEXDUMP_LOG(DEBUG,
142 "BPI: dst after post-process:",
143 dst, last_block_len);
146 return sym_op->cipher.data.length - last_block_len;
149 #ifdef RTE_LIB_SECURITY
151 qat_crc_verify(struct qat_sym_session *ctx, struct rte_crypto_op *op)
153 struct rte_crypto_sym_op *sym_op = op->sym;
154 uint32_t crc_data_ofs, crc_data_len, crc;
157 if (ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT &&
158 sym_op->auth.data.length != 0) {
160 crc_data_ofs = sym_op->auth.data.offset;
161 crc_data_len = sym_op->auth.data.length;
162 crc_data = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *,
165 crc = rte_net_crc_calc(crc_data, crc_data_len,
168 if (crc != *(uint32_t *)(crc_data + crc_data_len))
169 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
174 qat_crc_generate(struct qat_sym_session *ctx,
175 struct rte_crypto_op *op)
177 struct rte_crypto_sym_op *sym_op = op->sym;
178 uint32_t *crc, crc_data_len;
181 if (ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT &&
182 sym_op->auth.data.length != 0 &&
183 sym_op->m_src->nb_segs == 1) {
185 crc_data_len = sym_op->auth.data.length;
186 crc_data = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *,
187 sym_op->auth.data.offset);
188 crc = (uint32_t *)(crc_data + crc_data_len);
189 *crc = rte_net_crc_calc(crc_data, crc_data_len,
195 qat_sym_preprocess_requests(void **ops, uint16_t nb_ops)
197 struct rte_crypto_op *op;
198 struct qat_sym_session *ctx;
201 for (i = 0; i < nb_ops; i++) {
202 op = (struct rte_crypto_op *)ops[i];
204 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
205 ctx = (struct qat_sym_session *)
206 get_sec_session_private_data(
207 op->sym->sec_session);
209 if (ctx == NULL || ctx->bpi_ctx == NULL)
212 qat_crc_generate(ctx, op);
219 qat_sym_preprocess_requests(void **ops __rte_unused,
220 uint16_t nb_ops __rte_unused)
226 qat_sym_process_response(void **op, uint8_t *resp, void *op_cookie)
228 struct icp_qat_fw_comn_resp *resp_msg =
229 (struct icp_qat_fw_comn_resp *)resp;
230 struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
231 (resp_msg->opaque_data);
232 struct qat_sym_session *sess;
233 uint8_t is_docsis_sec;
235 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
236 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_response:", (uint8_t *)resp_msg,
237 sizeof(struct icp_qat_fw_comn_resp));
240 #ifdef RTE_LIB_SECURITY
241 if (rx_op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
243 * Assuming at this point that if it's a security
244 * op, that this is for DOCSIS
246 sess = (struct qat_sym_session *)
247 get_sec_session_private_data(
248 rx_op->sym->sec_session);
253 sess = (struct qat_sym_session *)
254 get_sym_session_private_data(
260 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
261 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
262 resp_msg->comn_hdr.comn_status)) {
264 rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
266 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
269 qat_bpicipher_postprocess(sess, rx_op);
270 #ifdef RTE_LIB_SECURITY
272 qat_crc_verify(sess, rx_op);
277 if (sess->is_single_pass_gmac) {
278 struct qat_sym_op_cookie *cookie =
279 (struct qat_sym_op_cookie *) op_cookie;
280 memset(cookie->opt.spc_gmac.cd_cipher.key, 0,
281 sess->auth_key_length);
288 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
289 struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
290 enum rte_crypto_op_sess_type sess_type,
291 union rte_cryptodev_session_ctx session_ctx, uint8_t is_update);
294 qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev);
299 qat_sym_preprocess_requests(void **ops __rte_unused,
300 uint16_t nb_ops __rte_unused)
305 qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused,
306 void *op_cookie __rte_unused)
311 #endif /* _QAT_SYM_H_ */