1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Intel Corporation
5 #ifndef _IPSEC_MB_PRIVATE_H_
6 #define _IPSEC_MB_PRIVATE_H_
8 #include <intel-ipsec-mb.h>
9 #include <cryptodev_pmd.h>
10 #include <rte_bus_vdev.h>
12 #if defined(RTE_LIB_SECURITY)
13 #define IPSEC_MB_DOCSIS_SEC_ENABLED 1
14 #include <rte_security.h>
15 #include <rte_security_driver.h>
18 /* Maximum length for digest */
19 #define DIGEST_LENGTH_MAX 64
21 /* Maximum length for memzone name */
22 #define IPSEC_MB_MAX_MZ_NAME 32
24 enum ipsec_mb_vector_mode {
25 IPSEC_MB_NOT_SUPPORTED = 0,
32 extern enum ipsec_mb_vector_mode vector_mode;
34 /** IMB_MGR instances, one per thread */
35 extern RTE_DEFINE_PER_LCORE(IMB_MGR *, mb_mgr);
37 #define CRYPTODEV_NAME_AESNI_MB_PMD crypto_aesni_mb
38 /**< IPSEC Multi buffer aesni_mb PMD device name */
40 #define CRYPTODEV_NAME_AESNI_GCM_PMD crypto_aesni_gcm
41 /**< IPSEC Multi buffer PMD aesni_gcm device name */
43 #define CRYPTODEV_NAME_KASUMI_PMD crypto_kasumi
44 /**< IPSEC Multi buffer PMD kasumi device name */
46 #define CRYPTODEV_NAME_SNOW3G_PMD crypto_snow3g
47 /**< IPSEC Multi buffer PMD snow3g device name */
49 #define CRYPTODEV_NAME_ZUC_PMD crypto_zuc
50 /**< IPSEC Multi buffer PMD zuc device name */
52 #define CRYPTODEV_NAME_CHACHA20_POLY1305_PMD crypto_chacha20_poly1305
53 /**< IPSEC Multi buffer PMD chacha20_poly1305 device name */
55 /** PMD LOGTYPE DRIVER, common to all PMDs */
56 extern int ipsec_mb_logtype_driver;
57 #define IPSEC_MB_LOG(level, fmt, ...) \
58 rte_log(RTE_LOG_##level, ipsec_mb_logtype_driver, \
59 "%s() line %u: " fmt "\n", __func__, __LINE__, ##__VA_ARGS__)
61 /** All supported device types */
62 enum ipsec_mb_pmd_types {
63 IPSEC_MB_PMD_TYPE_AESNI_MB = 0,
64 IPSEC_MB_PMD_TYPE_AESNI_GCM,
65 IPSEC_MB_PMD_TYPE_KASUMI,
66 IPSEC_MB_PMD_TYPE_SNOW3G,
67 IPSEC_MB_PMD_TYPE_ZUC,
68 IPSEC_MB_PMD_TYPE_CHACHA20_POLY1305,
72 /** Crypto operations */
73 enum ipsec_mb_operation {
74 IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN = 0,
75 IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT,
76 IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT,
77 IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY,
78 IPSEC_MB_OP_ENCRYPT_ONLY,
79 IPSEC_MB_OP_DECRYPT_ONLY,
80 IPSEC_MB_OP_HASH_GEN_ONLY,
81 IPSEC_MB_OP_HASH_VERIFY_ONLY,
82 IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT,
83 IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT,
84 IPSEC_MB_OP_NOT_SUPPORTED
87 extern uint8_t pmd_driver_id_aesni_mb;
88 extern uint8_t pmd_driver_id_aesni_gcm;
89 extern uint8_t pmd_driver_id_kasumi;
90 extern uint8_t pmd_driver_id_snow3g;
91 extern uint8_t pmd_driver_id_zuc;
92 extern uint8_t pmd_driver_id_chacha20_poly1305;
94 /** Helper function. Gets driver ID based on PMD type */
95 static __rte_always_inline uint8_t
96 ipsec_mb_get_driver_id(enum ipsec_mb_pmd_types pmd_type)
99 case IPSEC_MB_PMD_TYPE_AESNI_MB:
100 return pmd_driver_id_aesni_mb;
101 case IPSEC_MB_PMD_TYPE_AESNI_GCM:
102 return pmd_driver_id_aesni_gcm;
103 case IPSEC_MB_PMD_TYPE_KASUMI:
104 return pmd_driver_id_kasumi;
105 case IPSEC_MB_PMD_TYPE_SNOW3G:
106 return pmd_driver_id_snow3g;
107 case IPSEC_MB_PMD_TYPE_ZUC:
108 return pmd_driver_id_zuc;
109 case IPSEC_MB_PMD_TYPE_CHACHA20_POLY1305:
110 return pmd_driver_id_chacha20_poly1305;
117 /** Common private data structure for each PMD */
118 struct ipsec_mb_dev_private {
119 enum ipsec_mb_pmd_types pmd_type;
121 uint32_t max_nb_queue_pairs;
122 /**< Max number of queue pairs supported by device */
123 __extension__ uint8_t priv[0];
126 /** IPSEC Multi buffer queue pair common queue pair data for all PMDs */
129 /**< Queue Pair Identifier */
130 char name[RTE_CRYPTODEV_NAME_MAX_LEN];
131 struct rte_ring *ingress_queue;
132 /**< Ring for placing operations ready for processing */
133 struct rte_mempool *sess_mp;
134 /**< Session Mempool */
135 struct rte_mempool *sess_mp_priv;
136 /**< Session Private Data Mempool */
137 struct rte_cryptodev_stats stats;
138 /**< Queue pair statistics */
139 enum ipsec_mb_pmd_types pmd_type;
142 /**< Index of the next
143 * slot to be used in temp_digests,
144 * to store the digest for a given operation
147 /* Multi buffer manager */
148 const struct rte_memzone *mb_mgr_mz;
149 /* Shared memzone for storing mb_mgr */
150 __extension__ uint8_t additional_data[0];
151 /**< Storing PMD specific additional data */
154 static __rte_always_inline void *
155 ipsec_mb_get_qp_private_data(struct ipsec_mb_qp *qp)
157 return (void *)qp->additional_data;
160 /** Helper function. Allocates job manager */
161 static __rte_always_inline IMB_MGR *
162 alloc_init_mb_mgr(void)
164 IMB_MGR *mb_mgr = alloc_mb_mgr(0);
166 if (unlikely(mb_mgr == NULL)) {
167 IPSEC_MB_LOG(ERR, "Failed to allocate IMB_MGR data\n");
171 init_mb_mgr_auto(mb_mgr, NULL);
176 /** Helper function. Gets per thread job manager */
177 static __rte_always_inline IMB_MGR *
178 get_per_thread_mb_mgr(void)
180 if (unlikely(RTE_PER_LCORE(mb_mgr) == NULL))
181 RTE_PER_LCORE(mb_mgr) = alloc_init_mb_mgr();
183 return RTE_PER_LCORE(mb_mgr);
186 /** Helper function. Gets mode and chained xforms from the xform */
187 static __rte_always_inline int
188 ipsec_mb_parse_xform(const struct rte_crypto_sym_xform *xform,
189 enum ipsec_mb_operation *mode,
190 const struct rte_crypto_sym_xform **auth_xform,
191 const struct rte_crypto_sym_xform **cipher_xform,
192 const struct rte_crypto_sym_xform **aead_xform)
194 const struct rte_crypto_sym_xform *next = xform->next;
197 *mode = IPSEC_MB_OP_NOT_SUPPORTED;
201 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
203 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
204 *mode = IPSEC_MB_OP_ENCRYPT_ONLY;
205 *cipher_xform = xform;
209 *mode = IPSEC_MB_OP_DECRYPT_ONLY;
210 *cipher_xform = xform;
215 if (next->type != RTE_CRYPTO_SYM_XFORM_AUTH) {
216 *mode = IPSEC_MB_OP_NOT_SUPPORTED;
220 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
221 if (next->auth.op != RTE_CRYPTO_AUTH_OP_GENERATE) {
222 *mode = IPSEC_MB_OP_NOT_SUPPORTED;
226 *mode = IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN;
227 *cipher_xform = xform;
228 *auth_xform = xform->next;
231 if (next->auth.op != RTE_CRYPTO_AUTH_OP_VERIFY) {
232 *mode = IPSEC_MB_OP_NOT_SUPPORTED;
236 *mode = IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY;
237 *cipher_xform = xform;
238 *auth_xform = xform->next;
242 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
244 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
245 *mode = IPSEC_MB_OP_HASH_GEN_ONLY;
247 *cipher_xform = NULL;
250 *mode = IPSEC_MB_OP_HASH_VERIFY_ONLY;
252 *cipher_xform = NULL;
256 if (next->type != RTE_CRYPTO_SYM_XFORM_CIPHER) {
257 *mode = IPSEC_MB_OP_NOT_SUPPORTED;
261 if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) {
262 if (next->cipher.op != RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
263 *mode = IPSEC_MB_OP_NOT_SUPPORTED;
267 *mode = IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT;
269 *cipher_xform = xform->next;
272 if (next->cipher.op != RTE_CRYPTO_CIPHER_OP_DECRYPT) {
273 *mode = IPSEC_MB_OP_NOT_SUPPORTED;
277 *mode = IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT;
279 *cipher_xform = xform->next;
283 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
284 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
286 * CCM requires to hash first and cipher later
289 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
290 *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT;
295 IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT;
300 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) {
301 *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT;
305 *mode = IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT;
311 *mode = IPSEC_MB_OP_NOT_SUPPORTED;
315 /** Device creation function */
317 ipsec_mb_create(struct rte_vdev_device *vdev,
318 enum ipsec_mb_pmd_types pmd_type);
320 /** Device remove function */
322 ipsec_mb_remove(struct rte_vdev_device *vdev);
324 /** Configure queue pair PMD type specific data */
325 typedef int (*ipsec_mb_queue_pair_configure_t)(struct ipsec_mb_qp *qp);
327 /** Configure session PMD type specific data */
328 typedef int (*ipsec_mb_session_configure_t)(IMB_MGR *mbr_mgr,
329 void *session_private,
330 const struct rte_crypto_sym_xform *xform);
332 /** Configure internals PMD type specific data */
333 typedef int (*ipsec_mb_dev_configure_t)(struct rte_cryptodev *dev);
335 /** Per PMD type operation and data */
336 struct ipsec_mb_internals {
337 uint8_t is_configured;
338 dequeue_pkt_burst_t dequeue_burst;
339 ipsec_mb_dev_configure_t dev_config;
340 ipsec_mb_queue_pair_configure_t queue_pair_configure;
341 ipsec_mb_session_configure_t session_configure;
342 const struct rte_cryptodev_capabilities *caps;
343 struct rte_cryptodev_ops *ops;
344 struct rte_security_ops *security_ops;
345 uint64_t feature_flags;
346 uint32_t session_priv_size;
347 uint32_t qp_priv_size;
348 uint32_t internals_priv_size;
351 /** Global PMD type specific data */
352 extern struct ipsec_mb_internals ipsec_mb_pmds[IPSEC_MB_N_PMD_TYPES];
355 ipsec_mb_config(struct rte_cryptodev *dev,
356 struct rte_cryptodev_config *config);
359 ipsec_mb_start(struct rte_cryptodev *dev);
362 ipsec_mb_stop(struct rte_cryptodev *dev);
365 ipsec_mb_close(struct rte_cryptodev *dev);
368 ipsec_mb_stats_get(struct rte_cryptodev *dev,
369 struct rte_cryptodev_stats *stats);
372 ipsec_mb_stats_reset(struct rte_cryptodev *dev);
375 ipsec_mb_info_get(struct rte_cryptodev *dev,
376 struct rte_cryptodev_info *dev_info);
379 ipsec_mb_qp_release(struct rte_cryptodev *dev, uint16_t qp_id);
382 ipsec_mb_qp_set_unique_name(struct rte_cryptodev *dev, struct ipsec_mb_qp *qp);
385 ipsec_mb_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
386 const struct rte_cryptodev_qp_conf *qp_conf,
389 /** Returns the size of the aesni multi-buffer session structure */
391 ipsec_mb_sym_session_get_size(struct rte_cryptodev *dev);
393 /** Configure an aesni multi-buffer session from a crypto xform chain */
394 int ipsec_mb_sym_session_configure(
395 struct rte_cryptodev *dev,
396 struct rte_crypto_sym_xform *xform,
397 struct rte_cryptodev_sym_session *sess,
398 struct rte_mempool *mempool);
400 /** Clear the memory of session so it does not leave key material behind */
402 ipsec_mb_sym_session_clear(struct rte_cryptodev *dev,
403 struct rte_cryptodev_sym_session *sess);
405 /** Get session from op. If sessionless create a session */
406 static __rte_always_inline void *
407 ipsec_mb_get_session_private(struct ipsec_mb_qp *qp, struct rte_crypto_op *op)
410 uint32_t driver_id = ipsec_mb_get_driver_id(qp->pmd_type);
411 struct rte_crypto_sym_op *sym_op = op->sym;
412 uint8_t sess_type = op->sess_type;
414 void *_sess_private_data = NULL;
415 struct ipsec_mb_internals *pmd_data = &ipsec_mb_pmds[qp->pmd_type];
418 case RTE_CRYPTO_OP_WITH_SESSION:
419 if (likely(sym_op->session != NULL))
420 sess = get_sym_session_private_data(sym_op->session,
423 case RTE_CRYPTO_OP_SESSIONLESS:
425 rte_mempool_get(qp->sess_mp, (void **)&_sess))
428 if (!qp->sess_mp_priv ||
429 rte_mempool_get(qp->sess_mp_priv,
430 (void **)&_sess_private_data))
433 sess = _sess_private_data;
434 if (unlikely(pmd_data->session_configure(qp->mb_mgr,
435 sess, sym_op->xform) != 0)) {
436 rte_mempool_put(qp->sess_mp, _sess);
437 rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
441 sym_op->session = (struct rte_cryptodev_sym_session *)_sess;
442 set_sym_session_private_data(sym_op->session, driver_id,
446 IPSEC_MB_LOG(ERR, "Unrecognized session type %u", sess_type);
449 if (unlikely(sess == NULL))
450 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
455 #endif /* _IPSEC_MB_PRIVATE_H_ */