drivers/crypto/aesni_mb/aesni_mb_pmd_private.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2016 Intel Corporation
   3  */
   4
   5 #ifndef _AESNI_MB_PMD_PRIVATE_H_
   6 #define _AESNI_MB_PMD_PRIVATE_H_
   7
   8 #include <intel-ipsec-mb.h>
   9
  10 #if defined(RTE_LIBRTE_SECURITY) && (IMB_VERSION_NUM) >= IMB_VERSION(0, 54, 0)
  11 #define AESNI_MB_DOCSIS_SEC_ENABLED 1
  12 #include <rte_security.h>
  13 #include <rte_security_driver.h>
  14 #endif
  15
  16 enum aesni_mb_vector_mode {
  17         RTE_AESNI_MB_NOT_SUPPORTED = 0,
  18         RTE_AESNI_MB_SSE,
  19         RTE_AESNI_MB_AVX,
  20         RTE_AESNI_MB_AVX2,
  21         RTE_AESNI_MB_AVX512
  22 };
  23
  24 #define CRYPTODEV_NAME_AESNI_MB_PMD     crypto_aesni_mb
  25 /**< AES-NI Multi buffer PMD device name */
  26
  27 /** AESNI_MB PMD LOGTYPE DRIVER */
  28 extern int aesni_mb_logtype_driver;
  29
  30 #define AESNI_MB_LOG(level, fmt, ...)  \
  31         rte_log(RTE_LOG_ ## level, aesni_mb_logtype_driver,  \
  32                         "%s() line %u: " fmt "\n", __func__, __LINE__,  \
  33                                         ## __VA_ARGS__)
  34
  35
  36 #define HMAC_IPAD_VALUE                 (0x36)
  37 #define HMAC_OPAD_VALUE                 (0x5C)
  38
  39 /* Maximum length for digest */
  40 #define DIGEST_LENGTH_MAX 64
  41 static const unsigned auth_blocksize[] = {
  42                 [NULL_HASH]                     = 0,
  43                 [MD5]                           = 64,
  44                 [SHA1]                          = 64,
  45                 [SHA_224]                       = 64,
  46                 [SHA_256]                       = 64,
  47                 [SHA_384]                       = 128,
  48                 [SHA_512]                       = 128,
  49                 [AES_XCBC]                      = 16,
  50                 [AES_CCM]                       = 16,
  51                 [AES_CMAC]                      = 16,
  52                 [AES_GMAC]                      = 16,
  53                 [PLAIN_SHA1]                    = 64,
  54                 [PLAIN_SHA_224]                 = 64,
  55                 [PLAIN_SHA_256]                 = 64,
  56                 [PLAIN_SHA_384]                 = 128,
  57                 [PLAIN_SHA_512]                 = 128,
  58 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
  59                 [IMB_AUTH_ZUC_EIA3_BITLEN]      = 16,
  60                 [IMB_AUTH_SNOW3G_UIA2_BITLEN]   = 16
  61 #endif
  62 };
  63
  64 /**
  65  * Get the blocksize in bytes for a specified authentication algorithm
  66  *
  67  * @Note: this function will not return a valid value for a non-valid
  68  * authentication algorithm
  69  */
  70 static inline unsigned
  71 get_auth_algo_blocksize(JOB_HASH_ALG algo)
  72 {
  73         return auth_blocksize[algo];
  74 }
  75
  76 static const unsigned auth_truncated_digest_byte_lengths[] = {
  77                 [MD5]                           = 12,
  78                 [SHA1]                          = 12,
  79                 [SHA_224]                       = 14,
  80                 [SHA_256]                       = 16,
  81                 [SHA_384]                       = 24,
  82                 [SHA_512]                       = 32,
  83                 [AES_XCBC]                      = 12,
  84                 [AES_CMAC]                      = 12,
  85                 [AES_CCM]                       = 8,
  86                 [NULL_HASH]                     = 0,
  87                 [AES_GMAC]                      = 16,
  88                 [PLAIN_SHA1]                    = 20,
  89                 [PLAIN_SHA_224]                 = 28,
  90                 [PLAIN_SHA_256]                 = 32,
  91                 [PLAIN_SHA_384]                 = 48,
  92                 [PLAIN_SHA_512]                 = 64,
  93 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
  94                 [IMB_AUTH_ZUC_EIA3_BITLEN]      = 4,
  95                 [IMB_AUTH_SNOW3G_UIA2_BITLEN]   = 4
  96 #endif
  97 };
  98
  99 /**
 100  * Get the IPsec specified truncated length in bytes of the HMAC digest for a
 101  * specified authentication algorithm
 102  *
 103  * @Note: this function will not return a valid value for a non-valid
 104  * authentication algorithm
 105  */
 106 static inline unsigned
 107 get_truncated_digest_byte_length(JOB_HASH_ALG algo)
 108 {
 109         return auth_truncated_digest_byte_lengths[algo];
 110 }
 111
 112 static const unsigned auth_digest_byte_lengths[] = {
 113                 [MD5]                           = 16,
 114                 [SHA1]                          = 20,
 115                 [SHA_224]                       = 28,
 116                 [SHA_256]                       = 32,
 117                 [SHA_384]                       = 48,
 118                 [SHA_512]                       = 64,
 119                 [AES_XCBC]                      = 16,
 120                 [AES_CMAC]                      = 16,
 121                 [AES_CCM]                       = 16,
 122                 [AES_GMAC]                      = 12,
 123                 [NULL_HASH]                     = 0,
 124                 [PLAIN_SHA1]                    = 20,
 125                 [PLAIN_SHA_224]                 = 28,
 126                 [PLAIN_SHA_256]                 = 32,
 127                 [PLAIN_SHA_384]                 = 48,
 128                 [PLAIN_SHA_512]                 = 64,
 129 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 130                 [IMB_AUTH_ZUC_EIA3_BITLEN]      = 4,
 131                 [IMB_AUTH_SNOW3G_UIA2_BITLEN]   = 4
 132 #endif
 133         /**< Vector mode dependent pointer table of the multi-buffer APIs */
 134
 135 };
 136
 137 /**
 138  * Get the full digest size in bytes for a specified authentication algorithm
 139  * (if available in the Multi-buffer library)
 140  *
 141  * @Note: this function will not return a valid value for a non-valid
 142  * authentication algorithm
 143  */
 144 static inline unsigned
 145 get_digest_byte_length(JOB_HASH_ALG algo)
 146 {
 147         return auth_digest_byte_lengths[algo];
 148 }
 149
 150 enum aesni_mb_operation {
 151         AESNI_MB_OP_HASH_CIPHER,
 152         AESNI_MB_OP_CIPHER_HASH,
 153         AESNI_MB_OP_HASH_ONLY,
 154         AESNI_MB_OP_CIPHER_ONLY,
 155         AESNI_MB_OP_AEAD_HASH_CIPHER,
 156         AESNI_MB_OP_AEAD_CIPHER_HASH,
 157         AESNI_MB_OP_NOT_SUPPORTED
 158 };
 159
 160 /** private data structure for each virtual AESNI device */
 161 struct aesni_mb_private {
 162         enum aesni_mb_vector_mode vector_mode;
 163         /**< CPU vector instruction set mode */
 164         unsigned max_nb_queue_pairs;
 165         /**< Max number of queue pairs supported by device */
 166         MB_MGR *mb_mgr;
 167         /**< Multi-buffer instance */
 168 };
 169
 170 /** AESNI Multi buffer queue pair */
 171 struct aesni_mb_qp {
 172         uint16_t id;
 173         /**< Queue Pair Identifier */
 174         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 175         /**< Unique Queue Pair Name */
 176         MB_MGR *mb_mgr;
 177         /**< Multi-buffer instance */
 178         struct rte_ring *ingress_queue;
 179         /**< Ring for placing operations ready for processing */
 180         struct rte_mempool *sess_mp;
 181         /**< Session Mempool */
 182         struct rte_mempool *sess_mp_priv;
 183         /**< Session Private Data Mempool */
 184         struct rte_cryptodev_stats stats;
 185         /**< Queue pair statistics */
 186         uint8_t digest_idx;
 187         /**< Index of the next slot to be used in temp_digests,
 188          * to store the digest for a given operation
 189          */
 190         uint8_t temp_digests[MAX_JOBS][DIGEST_LENGTH_MAX];
 191         /**< Buffers used to store the digest generated
 192          * by the driver when verifying a digest provided
 193          * by the user (using authentication verify operation)
 194          */
 195 } __rte_cache_aligned;
 196
 197 /** AES-NI multi-buffer private session structure */
 198 struct aesni_mb_session {
 199         JOB_CHAIN_ORDER chain_order;
 200         struct {
 201                 uint16_t length;
 202                 uint16_t offset;
 203         } iv;
 204         struct {
 205                 uint16_t length;
 206                 uint16_t offset;
 207         } auth_iv;
 208         /**< IV parameters */
 209
 210         /** Cipher Parameters */const struct aesni_mb_op_fns *op_fns;
 211         /**< Vector mode dependent pointer table of the multi-buffer APIs */
 212
 213         struct {
 214                 /** Cipher direction - encrypt / decrypt */
 215                 JOB_CIPHER_DIRECTION direction;
 216                 /** Cipher mode - CBC / Counter */
 217                 JOB_CIPHER_MODE mode;
 218
 219                 uint64_t key_length_in_bytes;
 220
 221                 union {
 222                         struct {
 223                                 uint32_t encode[60] __rte_aligned(16);
 224                                 /**< encode key */
 225                                 uint32_t decode[60] __rte_aligned(16);
 226                                 /**< decode key */
 227                         } expanded_aes_keys;
 228                         /**< Expanded AES keys - Allocating space to
 229                          * contain the maximum expanded key size which
 230                          * is 240 bytes for 256 bit AES, calculate by:
 231                          * ((key size (bytes)) *
 232                          * ((number of rounds) + 1))
 233                          */
 234                         struct {
 235                                 const void *ks_ptr[3];
 236                                 uint64_t key[3][16];
 237                         } exp_3des_keys;
 238                         /**< Expanded 3DES keys */
 239
 240                         struct gcm_key_data gcm_key;
 241                         /**< Expanded GCM key */
 242                         uint8_t zuc_cipher_key[16];
 243                         /**< ZUC cipher key */
 244 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 245                         snow3g_key_schedule_t pKeySched_snow3g_cipher;
 246                         /**< SNOW3G scheduled cipher key */
 247 #endif
 248                 };
 249         } cipher;
 250
 251         /** Authentication Parameters */
 252         struct {
 253                 JOB_HASH_ALG algo; /**< Authentication Algorithm */
 254                 enum rte_crypto_auth_operation operation;
 255                 /**< auth operation generate or verify */
 256                 union {
 257                         struct {
 258                                 uint8_t inner[128] __rte_aligned(16);
 259                                 /**< inner pad */
 260                                 uint8_t outer[128] __rte_aligned(16);
 261                                 /**< outer pad */
 262                         } pads;
 263                         /**< HMAC Authentication pads -
 264                          * allocating space for the maximum pad
 265                          * size supported which is 128 bytes for
 266                          * SHA512
 267                          */
 268
 269                         struct {
 270                             uint32_t k1_expanded[44] __rte_aligned(16);
 271                             /**< k1 (expanded key). */
 272                             uint8_t k2[16] __rte_aligned(16);
 273                             /**< k2. */
 274                             uint8_t k3[16] __rte_aligned(16);
 275                             /**< k3. */
 276                         } xcbc;
 277
 278                         struct {
 279                                 uint32_t expkey[60] __rte_aligned(16);
 280                                                     /**< k1 (expanded key). */
 281                                 uint32_t skey1[4] __rte_aligned(16);
 282                                                     /**< k2. */
 283                                 uint32_t skey2[4] __rte_aligned(16);
 284                                                     /**< k3. */
 285                         } cmac;
 286                         /**< Expanded XCBC authentication keys */
 287                         uint8_t zuc_auth_key[16];
 288                         /**< ZUC authentication key */
 289 #if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
 290                         snow3g_key_schedule_t pKeySched_snow3g_auth;
 291                         /**< SNOW3G scheduled authentication key */
 292 #endif
 293                 };
 294         /** Generated digest size by the Multi-buffer library */
 295         uint16_t gen_digest_len;
 296         /** Requested digest size from Cryptodev */
 297         uint16_t req_digest_len;
 298
 299         } auth;
 300         struct {
 301                 /** AAD data length */
 302                 uint16_t aad_len;
 303         } aead;
 304 } __rte_cache_aligned;
 305
 306 extern int
 307 aesni_mb_set_session_parameters(const MB_MGR *mb_mgr,
 308                 struct aesni_mb_session *sess,
 309                 const struct rte_crypto_sym_xform *xform);
 310
 311 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 312 extern int
 313 aesni_mb_set_docsis_sec_session_parameters(
 314                 __rte_unused struct rte_cryptodev *dev,
 315                 struct rte_security_session_conf *conf,
 316                 void *sess);
 317 #endif
 318
 319 /** device specific operations function pointer structures */
 320 extern struct rte_cryptodev_ops *rte_aesni_mb_pmd_ops;
 321 #ifdef AESNI_MB_DOCSIS_SEC_ENABLED
 322 extern struct rte_security_ops *rte_aesni_mb_pmd_sec_ops;
 323 #endif
 324
 325 extern uint32_t
 326 aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
 327         struct rte_cryptodev_sym_session *sess, union rte_crypto_sym_ofs sofs,
 328         struct rte_crypto_sym_vec *vec);
 329
 330 #endif /* _AESNI_MB_PMD_PRIVATE_H_ */