drivers/crypto/aesni_mb/aesni_mb_pmd_private.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2016 Intel Corporation
   3  */
   4
   5 #ifndef _AESNI_MB_PMD_PRIVATE_H_
   6 #define _AESNI_MB_PMD_PRIVATE_H_
   7
   8 #include <intel-ipsec-mb.h>
   9
  10 #if defined(RTE_LIBRTE_SECURITY) && (IMB_VERSION_NUM) >= IMB_VERSION(0, 54, 0)
  11 #define AESNI_MB_DOCSIS_SEC_ENABLED 1
  12 #include <rte_security.h>
  13 #include <rte_security_driver.h>
  14 #endif
  15
  16 enum aesni_mb_vector_mode {
  17         RTE_AESNI_MB_NOT_SUPPORTED = 0,
  18         RTE_AESNI_MB_SSE,
  19         RTE_AESNI_MB_AVX,
  20         RTE_AESNI_MB_AVX2,
  21         RTE_AESNI_MB_AVX512
  22 };
  23
  24 #define CRYPTODEV_NAME_AESNI_MB_PMD     crypto_aesni_mb
  25 /**< AES-NI Multi buffer PMD device name */
  26
  27 /** AESNI_MB PMD LOGTYPE DRIVER */
  28 extern int aesni_mb_logtype_driver;
  29
  30 #define AESNI_MB_LOG(level, fmt, ...)  \
  31         rte_log(RTE_LOG_ ## level, aesni_mb_logtype_driver,  \
  32                         "%s() line %u: " fmt "\n", __func__, __LINE__,  \
  33                                         ## __VA_ARGS__)
  34
  35
  36 #define HMAC_IPAD_VALUE                 (0x36)
  37 #define HMAC_OPAD_VALUE                 (0x5C)
  38
  39 /* Maximum length for digest */
  40 #define DIGEST_LENGTH_MAX 64
  41 static const unsigned auth_blocksize[] = {
  42                 [NULL_HASH]     = 0,
  43                 [MD5]           = 64,
  44                 [SHA1]          = 64,
  45                 [SHA_224]       = 64,
  46                 [SHA_256]       = 64,
  47                 [SHA_384]       = 128,
  48                 [SHA_512]       = 128,
  49                 [AES_XCBC]      = 16,
  50                 [AES_CCM]       = 16,
  51                 [AES_CMAC]      = 16,
  52                 [AES_GMAC]      = 16,
  53                 [PLAIN_SHA1]    = 64,
  54                 [PLAIN_SHA_224] = 64,
  55                 [PLAIN_SHA_256] = 64,
  56                 [PLAIN_SHA_384] = 128,
  57                 [PLAIN_SHA_512] = 128
  58 };
  59
  60 /**
  61  * Get the blocksize in bytes for a specified authentication algorithm
  62  *
  63  * @Note: this function will not return a valid value for a non-valid
  64  * authentication algorithm
  65  */
  66 static inline unsigned
  67 get_auth_algo_blocksize(JOB_HASH_ALG algo)
  68 {
  69         return auth_blocksize[algo];
  70 }
  71
  72 static const unsigned auth_truncated_digest_byte_lengths[] = {
  73                 [MD5]           = 12,
  74                 [SHA1]          = 12,
  75                 [SHA_224]       = 14,
  76                 [SHA_256]       = 16,
  77                 [SHA_384]       = 24,
  78                 [SHA_512]       = 32,
  79                 [AES_XCBC]      = 12,
  80                 [AES_CMAC]      = 12,
  81                 [AES_CCM]       = 8,
  82                 [NULL_HASH]     = 0,
  83                 [AES_GMAC]      = 16,
  84                 [PLAIN_SHA1]    = 20,
  85                 [PLAIN_SHA_224] = 28,
  86                 [PLAIN_SHA_256] = 32,
  87                 [PLAIN_SHA_384] = 48,
  88                 [PLAIN_SHA_512] = 64
  89 };
  90
  91 /**
  92  * Get the IPsec specified truncated length in bytes of the HMAC digest for a
  93  * specified authentication algorithm
  94  *
  95  * @Note: this function will not return a valid value for a non-valid
  96  * authentication algorithm
  97  */
  98 static inline unsigned
  99 get_truncated_digest_byte_length(JOB_HASH_ALG algo)
 100 {
 101         return auth_truncated_digest_byte_lengths[algo];
 102 }
 103
 104 static const unsigned auth_digest_byte_lengths[] = {
 105                 [MD5]           = 16,
 106                 [SHA1]          = 20,
 107                 [SHA_224]       = 28,
 108                 [SHA_256]       = 32,
 109                 [SHA_384]       = 48,
 110                 [SHA_512]       = 64,
 111                 [AES_XCBC]      = 16,
 112                 [AES_CMAC]      = 16,
 113                 [AES_CCM]       = 16,
 114                 [AES_GMAC]      = 12,
 115                 [NULL_HASH]     = 0,
 116                 [PLAIN_SHA1]    = 20,
 117                 [PLAIN_SHA_224] = 28,
 118                 [PLAIN_SHA_256] = 32,
 119                 [PLAIN_SHA_384] = 48,
 120                 [PLAIN_SHA_512] = 64
 121         /**< Vector mode dependent pointer table of the multi-buffer APIs */
 122
 123 };
 124
 125 /**
 126  * Get the full digest size in bytes for a specified authentication algorithm
 127  * (if available in the Multi-buffer library)
 128  *
 129  * @Note: this function will not return a valid value for a non-valid
 130  * authentication algorithm
 131  */
 132 static inline unsigned
 133 get_digest_byte_length(JOB_HASH_ALG algo)
 134 {
 135         return auth_digest_byte_lengths[algo];
 136 }
 137
 138 enum aesni_mb_operation {
 139         AESNI_MB_OP_HASH_CIPHER,
 140         AESNI_MB_OP_CIPHER_HASH,
 141         AESNI_MB_OP_HASH_ONLY,
 142         AESNI_MB_OP_CIPHER_ONLY,
 143         AESNI_MB_OP_AEAD_HASH_CIPHER,
 144         AESNI_MB_OP_AEAD_CIPHER_HASH,
 145         AESNI_MB_OP_NOT_SUPPORTED
 146 };
 147
 148 /** private data structure for each virtual AESNI device */
 149 struct aesni_mb_private {
 150         enum aesni_mb_vector_mode vector_mode;
 151         /**< CPU vector instruction set mode */
 152         unsigned max_nb_queue_pairs;
 153         /**< Max number of queue pairs supported by device */
 154         MB_MGR *mb_mgr;
 155         /**< Multi-buffer instance */
 156 };
 157
 158 /** AESNI Multi buffer queue pair */
 159 struct aesni_mb_qp {
 160         uint16_t id;
 161         /**< Queue Pair Identifier */
 162         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 163         /**< Unique Queue Pair Name */
 164         MB_MGR *mb_mgr;
 165         /**< Multi-buffer instance */
 166         struct rte_ring *ingress_queue;
 167         /**< Ring for placing operations ready for processing */
 168         struct rte_mempool *sess_mp;
 169         /**< Session Mempool */
 170         struct rte_mempool *sess_mp_priv;
 171         /**< Session Private Data Mempool */
 172         struct rte_cryptodev_stats stats;
 173         /**< Queue pair statistics */
 174         uint8_t digest_idx;
 175         /**< Index of the next slot to be used in temp_digests,
 176          * to store the digest for a given operation
 177          */
 178         uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];
 179         /**< Buffers used to store the digest generated
 180          * by the driver when verifying a digest provided
 181          * by the user (using authentication verify operation)
 182          */
 183 } __rte_cache_aligned;
 184
 185 /** AES-NI multi-buffer private session structure */
 186 struct aesni_mb_session {
 187         JOB_CHAIN_ORDER chain_order;
 188         struct {
 189                 uint16_t length;
 190                 uint16_t offset;
 191         } iv;
 192         /**< IV parameters */
 193
 194         /** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;
 195         /**< Vector mode dependent pointer table of the multi-buffer APIs */
 196
 197         struct {
 198                 /** Cipher direction - encrypt / decrypt */
 199                 JOB_CIPHER_DIRECTION direction;
 200                 /** Cipher mode - CBC / Counter */
 201                 JOB_CIPHER_MODE mode;
 202
 203                 uint64_t key_length_in_bytes;
 204
 205                 union {
 206                         struct {
 207                                 uint32_t encode[60] __rte_aligned(16);
 208                                 /**< encode key */
 209                                 uint32_t decode[60] __rte_aligned(16);
 210                                 /**< decode key */
 211                         } expanded_aes_keys;
 212                         struct {
 213                                 const void *ks_ptr[3];
 214                                 uint64_t key[3][16];
 215                         } exp_3des_keys;
 216
 217                         struct gcm_key_data gcm_key;
 218                 };
 219                 /**< Expanded AES keys - Allocating space to
 220                  * contain the maximum expanded key size which
 221                  * is 240 bytes for 256 bit AES, calculate by:
 222                  * ((key size (bytes)) *
 223                  * ((number of rounds) + 1))
 224                  */
 225         } cipher;
 226
 227         /** Authentication Parameters */
 228         struct {
 229                 JOB_HASH_ALG algo; /**< Authentication Algorithm */
 230                 enum rte_crypto_auth_operation operation;
 231                 /**< auth operation generate or verify */
 232                 union {
 233                         struct {
 234                                 uint8_t inner[128] __rte_aligned(16);
 235                                 /**< inner pad */
 236                                 uint8_t outer[128] __rte_aligned(16);
 237                                 /**< outer pad */
 238                         } pads;
 239                         /**< HMAC Authentication pads -
 240                          * allocating space for the maximum pad
 241                          * size supported which is 128 bytes for
 242                          * SHA512
 243                          */
 244
 245                         struct {
 246                             uint32_t k1_expanded[44] __rte_aligned(16);
 247                             /**< k1 (expanded key). */
 248                             uint8_t k2[16] __rte_aligned(16);
 249                             /**< k2. */
 250                             uint8_t k3[16] __rte_aligned(16);
 251                             /**< k3. */
 252                         } xcbc;
 253
 254                         struct {
 255                                 uint32_t expkey[60] __rte_aligned(16);
 256                                                     /**< k1 (expanded key). */
 257                                 uint32_t skey1[4] __rte_aligned(16);
 258                                                     /**< k2. */
 259                                 uint32_t skey2[4] __rte_aligned(16);
 260                                                     /**< k3. */
 261                         } cmac;
 262                         /**< Expanded XCBC authentication keys */
 263                 };
 264         /** Generated digest size by the Multi-buffer library */
 265         uint16_t gen_digest_len;
 266         /** Requested digest size from Cryptodev */
 267         uint16_t req_digest_len;
 268
 269         } auth;
 270         struct {
 271                 /** AAD data length */
 272                 uint16_t aad_len;
 273         } aead;
 274 } __rte_cache_aligned;
 275
 276 extern int
 277 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 278                 struct aesni_mb_session *sess,
 279                 const struct rte_crypto_sym_xform *xform);
 280
 281 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 282 extern int
 283 aesni_mb_set_docsis_sec_session_parameters(
 284                 __rte_unused struct rte_cryptodev *dev,
 285                 struct rte_security_session_conf *conf,
 286                 void *sess);
 287 #endif
 288
 289 /** device specific operations function pointer structures */
 290 extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;
 291 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 292 extern struct rte_security_ops *rte_aesni_mb_pmd_sec_ops;
 293 #endif
 294
 295 extern uint32_t
 296 aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
 297         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
 298         struct rte_crypto_sym_vec *vec);
 299
 300 #endif /* _AESNI_MB_PMD_PRIVATE_H_ */