crypto/octeontx2: move capabilities init into probe
[dpdk.git] / drivers / crypto / qat / qat_sym.h
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2015-2018 Intel Corporation
3  */
4
5 #ifndef _QAT_SYM_H_
6 #define _QAT_SYM_H_
7
8 #include <rte_cryptodev_pmd.h>
9 #ifdef RTE_LIBRTE_SECURITY
10 #include <rte_net_crc.h>
11 #endif
12
13 #ifdef BUILD_QAT_SYM
14 #include <openssl/evp.h>
15
16 #include "qat_common.h"
17 #include "qat_sym_session.h"
18 #include "qat_sym_pmd.h"
19 #include "qat_logs.h"
20
21 #define BYTE_LENGTH    8
22 /* bpi is only used for partial blocks of DES and AES
23  * so AES block len can be assumed as max len for iv, src and dst
24  */
25 #define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
26
27 /*
28  * Maximum number of SGL entries
29  */
30 #define QAT_SYM_SGL_MAX_NUMBER  16
31
32 struct qat_sym_session;
33
34 struct qat_sym_sgl {
35         qat_sgl_hdr;
36         struct qat_flat_buf buffers[QAT_SYM_SGL_MAX_NUMBER];
37 } __rte_packed __rte_cache_aligned;
38
39 struct qat_sym_op_cookie {
40         struct qat_sym_sgl qat_sgl_src;
41         struct qat_sym_sgl qat_sgl_dst;
42         phys_addr_t qat_sgl_src_phys_addr;
43         phys_addr_t qat_sgl_dst_phys_addr;
44 };
45
46 int
47 qat_sym_build_request(void *in_op, uint8_t *out_msg,
48                 void *op_cookie, enum qat_device_gen qat_dev_gen);
49
50
51 /** Encrypt a single partial block
52  *  Depends on openssl libcrypto
53  *  Uses ECB+XOR to do CFB encryption, same result, more performant
54  */
55 static inline int
56 bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
57                 uint8_t *iv, int ivlen, int srclen,
58                 void *bpi_ctx)
59 {
60         EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
61         int encrypted_ivlen;
62         uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
63         uint8_t *encr = encrypted_iv;
64
65         /* ECB method: encrypt the IV, then XOR this with plaintext */
66         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
67                                                                 <= 0)
68                 goto cipher_encrypt_err;
69
70         for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
71                 *dst = *src ^ *encr;
72
73         return 0;
74
75 cipher_encrypt_err:
76         QAT_DP_LOG(ERR, "libcrypto ECB cipher encrypt failed");
77         return -EINVAL;
78 }
79
80 static inline uint32_t
81 qat_bpicipher_postprocess(struct qat_sym_session *ctx,
82                                 struct rte_crypto_op *op)
83 {
84         int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
85         struct rte_crypto_sym_op *sym_op = op->sym;
86         uint8_t last_block_len = block_len > 0 ?
87                         sym_op->cipher.data.length % block_len : 0;
88
89         if (last_block_len > 0 &&
90                         ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
91
92                 /* Encrypt last block */
93                 uint8_t *last_block, *dst, *iv;
94                 uint32_t last_block_offset;
95
96                 last_block_offset = sym_op->cipher.data.offset +
97                                 sym_op->cipher.data.length - last_block_len;
98                 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
99                                 uint8_t *, last_block_offset);
100
101                 if (unlikely(sym_op->m_dst != NULL))
102                         /* out-of-place operation (OOP) */
103                         dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
104                                                 uint8_t *, last_block_offset);
105                 else
106                         dst = last_block;
107
108                 if (last_block_len < sym_op->cipher.data.length)
109                         /* use previous block ciphertext as IV */
110                         iv = dst - block_len;
111                 else
112                         /* runt block, i.e. less than one full block */
113                         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
114                                         ctx->cipher_iv.offset);
115
116 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
117                 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before post-process:",
118                         last_block, last_block_len);
119                 if (sym_op->m_dst != NULL)
120                         QAT_DP_HEXDUMP_LOG(DEBUG,
121                                 "BPI: dst before post-process:",
122                                 dst, last_block_len);
123 #endif
124                 bpi_cipher_encrypt(last_block, dst, iv, block_len,
125                                 last_block_len, ctx->bpi_ctx);
126 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
127                 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after post-process:",
128                                 last_block, last_block_len);
129                 if (sym_op->m_dst != NULL)
130                         QAT_DP_HEXDUMP_LOG(DEBUG,
131                                 "BPI: dst after post-process:",
132                                 dst, last_block_len);
133 #endif
134         }
135         return sym_op->cipher.data.length - last_block_len;
136 }
137
138 #ifdef RTE_LIBRTE_SECURITY
139 static inline void
140 qat_crc_verify(struct qat_sym_session *ctx, struct rte_crypto_op *op)
141 {
142         struct rte_crypto_sym_op *sym_op = op->sym;
143         uint32_t crc_offset, crc_length, crc;
144         uint8_t *crc_data;
145
146         if (ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT &&
147                         sym_op->auth.data.length != 0) {
148
149                 crc_offset = sym_op->auth.data.offset;
150                 crc_length = sym_op->auth.data.length;
151                 crc_data = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *,
152                                 crc_offset);
153
154                 crc = rte_net_crc_calc(crc_data, crc_length, RTE_NET_CRC32_ETH);
155
156                 if (crc != *(uint32_t *)(crc_data + crc_length))
157                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
158         }
159 }
160 #endif
161
162 static inline void
163 qat_sym_process_response(void **op, uint8_t *resp)
164 {
165         struct icp_qat_fw_comn_resp *resp_msg =
166                         (struct icp_qat_fw_comn_resp *)resp;
167         struct rte_crypto_op *rx_op = (struct rte_crypto_op *)(uintptr_t)
168                         (resp_msg->opaque_data);
169         struct qat_sym_session *sess;
170
171 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
172         QAT_DP_HEXDUMP_LOG(DEBUG, "qat_response:", (uint8_t *)resp_msg,
173                         sizeof(struct icp_qat_fw_comn_resp));
174 #endif
175
176         if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
177                         ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
178                         resp_msg->comn_hdr.comn_status)) {
179
180                 rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
181         } else {
182 #ifdef RTE_LIBRTE_SECURITY
183                 uint8_t is_docsis_sec = 0;
184
185                 if (rx_op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
186                         /*
187                          * Assuming at this point that if it's a security
188                          * op, that this is for DOCSIS
189                          */
190                         sess = (struct qat_sym_session *)
191                                         get_sec_session_private_data(
192                                         rx_op->sym->sec_session);
193                         is_docsis_sec = 1;
194                 } else
195 #endif
196                 {
197                         sess = (struct qat_sym_session *)
198                                         get_sym_session_private_data(
199                                         rx_op->sym->session,
200                                         qat_sym_driver_id);
201                 }
202
203                 rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
204
205                 if (sess->bpi_ctx) {
206                         qat_bpicipher_postprocess(sess, rx_op);
207 #ifdef RTE_LIBRTE_SECURITY
208                         if (is_docsis_sec)
209                                 qat_crc_verify(sess, rx_op);
210 #endif
211                 }
212         }
213         *op = (void *)rx_op;
214 }
215 #else
216
217 static inline void
218 qat_sym_process_response(void **op __rte_unused, uint8_t *resp __rte_unused)
219 {
220 }
221 #endif
222 #endif /* _QAT_SYM_H_ */