X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=drivers%2Fcrypto%2Fqat%2Fqat_crypto.c;h=62ee175ba4a760071cb7800a0581d3cf5e364f8a;hb=5dce9fcdb2308becb7de7470118af3eeccfe4fd7;hp=e0750763ca6b9e8c5cf36cbfeb4ac92323ab7ef1;hpb=b79e4c00af0e7cfb8601ab0208659d226b82bd10;p=dpdk.git diff --git a/drivers/crypto/qat/qat_crypto.c b/drivers/crypto/qat/qat_crypto.c index e0750763ca..62ee175ba4 100644 --- a/drivers/crypto/qat/qat_crypto.c +++ b/drivers/crypto/qat/qat_crypto.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include @@ -171,16 +170,19 @@ cipher_decrypt_err: /** Creates a context in either AES or DES in ECB mode * Depends on openssl libcrypto */ -static void * +static int bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, enum rte_crypto_cipher_operation direction __rte_unused, - uint8_t *key) + uint8_t *key, void **ctx) { const EVP_CIPHER *algo = NULL; - EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + int ret; + *ctx = EVP_CIPHER_CTX_new(); - if (ctx == NULL) + if (*ctx == NULL) { + ret = -ENOMEM; goto ctx_init_err; + } if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI) algo = EVP_des_ecb(); @@ -188,15 +190,17 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, algo = EVP_aes_128_ecb(); /* IV will be ECB encrypted whether direction is encrypt or decrypt*/ - if (EVP_EncryptInit_ex(ctx, algo, NULL, key, 0) != 1) + if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) { + ret = -EINVAL; goto ctx_init_err; + } - return ctx; + return 0; ctx_init_err: - if (ctx != NULL) - EVP_CIPHER_CTX_free(ctx); - return NULL; + if (*ctx != NULL) + EVP_CIPHER_CTX_free(*ctx); + return ret; } /** Frees a context previously created @@ -214,25 +218,25 @@ adf_modulo(uint32_t data, uint32_t shift); static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, - struct qat_crypto_op_cookie *qat_op_cookie); + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp); -void qat_crypto_sym_clear_session(struct rte_cryptodev *dev, - void *session) +void +qat_crypto_sym_clear_session(struct rte_cryptodev *dev, + struct rte_cryptodev_sym_session *sess) { - struct qat_session *sess = session; - phys_addr_t cd_paddr; - PMD_INIT_FUNC_TRACE(); - if (sess) { - if (sess->bpi_ctx) { - bpi_cipher_ctx_free(sess->bpi_ctx); - sess->bpi_ctx = NULL; - } - cd_paddr = sess->cd_paddr; - memset(sess, 0, qat_crypto_sym_get_session_private_size(dev)); - sess->cd_paddr = cd_paddr; - } else - PMD_DRV_LOG(ERR, "NULL session"); + uint8_t index = dev->driver_id; + void *sess_priv = get_session_private_data(sess, index); + struct qat_session *s = (struct qat_session *)sess_priv; + + if (sess_priv) { + if (s->bpi_ctx) + bpi_cipher_ctx_free(s->bpi_ctx); + memset(s, 0, qat_crypto_sym_get_session_private_size(dev)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + set_session_private_data(sess, index, NULL); + rte_mempool_put(sess_mp, sess_priv); + } } static int @@ -295,13 +299,15 @@ qat_get_cipher_xform(struct rte_crypto_sym_xform *xform) return NULL; } -void * + +int qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, - struct rte_crypto_sym_xform *xform, void *session_private) + struct rte_crypto_sym_xform *xform, + struct qat_session *session) { - struct qat_session *session = session_private; struct qat_pmd_private *internals = dev->data->dev_private; struct rte_crypto_cipher_xform *cipher_xform = NULL; + int ret; /* Get cipher xform from crypto xform chain */ cipher_xform = qat_get_cipher_xform(xform); @@ -314,6 +320,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -322,6 +329,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_aes_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; @@ -330,6 +338,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_snow3g_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; @@ -341,6 +350,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_kasumi_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE; @@ -349,6 +359,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_3des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -357,6 +368,7 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -365,38 +377,43 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_validate_3des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; break; case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: - session->bpi_ctx = bpi_cipher_ctx_init( + ret = bpi_cipher_ctx_init( cipher_xform->algo, cipher_xform->op, - cipher_xform->key.data); - if (session->bpi_ctx == NULL) { + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { PMD_DRV_LOG(ERR, "failed to create DES BPI ctx"); goto error_out; } if (qat_alg_validate_des_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid DES cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; break; case RTE_CRYPTO_CIPHER_AES_DOCSISBPI: - session->bpi_ctx = bpi_cipher_ctx_init( + ret = bpi_cipher_ctx_init( cipher_xform->algo, cipher_xform->op, - cipher_xform->key.data); - if (session->bpi_ctx == NULL) { + cipher_xform->key.data, + &session->bpi_ctx); + if (ret != 0) { PMD_DRV_LOG(ERR, "failed to create AES BPI ctx"); goto error_out; } if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE; @@ -407,11 +424,13 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, PMD_DRV_LOG(ERR, "%s not supported on this device", rte_crypto_cipher_algorithm_strings [cipher_xform->algo]); + ret = -ENOTSUP; goto error_out; } if (qat_alg_validate_zuc_key(cipher_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size"); + ret = -EINVAL; goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE; @@ -423,10 +442,12 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, case RTE_CRYPTO_CIPHER_ARC4: PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u", cipher_xform->algo); + ret = -ENOTSUP; goto error_out; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n", cipher_xform->algo); + ret = -EINVAL; goto error_out; } @@ -437,63 +458,118 @@ qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev, if (qat_alg_aead_session_create_content_desc_cipher(session, cipher_xform->key.data, - cipher_xform->key.length)) + cipher_xform->key.length)) { + ret = -EINVAL; goto error_out; + } - return session; + return 0; error_out: if (session->bpi_ctx) { bpi_cipher_ctx_free(session->bpi_ctx); session->bpi_ctx = NULL; } - return NULL; + return ret; } - -void * +int qat_crypto_sym_configure_session(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, + struct rte_cryptodev_sym_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + int ret; + + if (rte_mempool_get(mempool, &sess_private_data)) { + CDEV_LOG_ERR( + "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data); + if (ret != 0) { + PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure " + "session parameters"); + + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_session_private_data(sess, dev->driver_id, + sess_private_data); + + return 0; +} + +int +qat_crypto_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_session *session = session_private; + int ret; int qat_cmd_id; PMD_INIT_FUNC_TRACE(); + /* Set context descriptor physical address */ + session->cd_paddr = rte_mempool_virt2phy(NULL, session) + + offsetof(struct qat_session, cd); + + session->min_qat_dev_gen = QAT_GEN1; + /* Get requested QAT command id */ qat_cmd_id = qat_get_cmd_id(xform); if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) { PMD_DRV_LOG(ERR, "Unsupported xform chain requested"); - goto error_out; + return -ENOTSUP; } session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id; switch (session->qat_cmd) { case ICP_QAT_FW_LA_CMD_CIPHER: - session = qat_crypto_sym_configure_session_cipher(dev, xform, session); + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_AUTH: - session = qat_crypto_sym_configure_session_auth(dev, xform, session); + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (ret < 0) + return ret; break; case ICP_QAT_FW_LA_CMD_CIPHER_HASH: - if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) - session = qat_crypto_sym_configure_session_aead(xform, + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, session); - else { - session = qat_crypto_sym_configure_session_cipher(dev, + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); - session = qat_crypto_sym_configure_session_auth(dev, + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); + if (ret < 0) + return ret; } break; case ICP_QAT_FW_LA_CMD_HASH_CIPHER: - if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) - session = qat_crypto_sym_configure_session_aead(xform, + if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + ret = qat_crypto_sym_configure_session_aead(xform, session); - else { - session = qat_crypto_sym_configure_session_auth(dev, + if (ret < 0) + return ret; + } else { + ret = qat_crypto_sym_configure_session_auth(dev, xform, session); - session = qat_crypto_sym_configure_session_cipher(dev, + if (ret < 0) + return ret; + ret = qat_crypto_sym_configure_session_cipher(dev, xform, session); + if (ret < 0) + return ret; } break; case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM: @@ -507,26 +583,21 @@ qat_crypto_sym_configure_session(struct rte_cryptodev *dev, case ICP_QAT_FW_LA_CMD_DELIMITER: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Unsupported Service %u", session->qat_cmd); - goto error_out; + return -ENOTSUP; } - return session; - -error_out: - return NULL; + return 0; } -struct qat_session * +int qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, - struct qat_session *session_private) + struct qat_session *session) { - - struct qat_session *session = session_private; struct rte_crypto_auth_xform *auth_xform = NULL; struct qat_pmd_private *internals = dev->data->dev_private; auth_xform = qat_get_auth_xform(xform); @@ -556,7 +627,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, if (qat_alg_validate_aes_key(auth_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES key size"); - goto error_out; + return -EINVAL; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; @@ -579,7 +650,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, PMD_DRV_LOG(ERR, "%s not supported on this device", rte_crypto_auth_algorithm_strings [auth_xform->algo]); - goto error_out; + return -ENOTSUP; } session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3; break; @@ -593,11 +664,11 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, case RTE_CRYPTO_AUTH_AES_CBC_MAC: PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u", auth_xform->algo); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified", auth_xform->algo); - goto error_out; + return -EINVAL; } session->auth_iv.offset = auth_xform->iv.offset; @@ -614,7 +685,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, if (qat_alg_aead_session_create_content_desc_cipher(session, auth_xform->key.data, auth_xform->key.length)) - goto error_out; + return -EINVAL; if (qat_alg_aead_session_create_content_desc_auth(session, key_data, @@ -622,7 +693,7 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; } else { session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER; session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; @@ -636,12 +707,12 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; if (qat_alg_aead_session_create_content_desc_cipher(session, auth_xform->key.data, auth_xform->key.length)) - goto error_out; + return -EINVAL; } /* Restore to authentication only only */ session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH; @@ -652,21 +723,17 @@ qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev, 0, auth_xform->digest_length, auth_xform->op)) - goto error_out; + return -EINVAL; } session->digest_length = auth_xform->digest_length; - return session; - -error_out: - return NULL; + return 0; } -struct qat_session * +int qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, - struct qat_session *session_private) + struct qat_session *session) { - struct qat_session *session = session_private; struct rte_crypto_aead_xform *aead_xform = &xform->aead; /* @@ -681,7 +748,7 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, if (qat_alg_validate_aes_key(aead_xform->key.length, &session->qat_cipher_alg) != 0) { PMD_DRV_LOG(ERR, "Invalid AES key size"); - goto error_out; + return -EINVAL; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; @@ -689,11 +756,11 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, case RTE_CRYPTO_AEAD_AES_CCM: PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported AEAD alg %u", aead_xform->algo); - goto error_out; + return -ENOTSUP; default: PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n", aead_xform->algo); - goto error_out; + return -EINVAL; } if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { @@ -705,15 +772,15 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, if (qat_alg_aead_session_create_content_desc_cipher(session, aead_xform->key.data, aead_xform->key.length)) - goto error_out; + return -EINVAL; if (qat_alg_aead_session_create_content_desc_auth(session, aead_xform->key.data, aead_xform->key.length, - aead_xform->add_auth_data_length, + aead_xform->aad_length, aead_xform->digest_length, RTE_CRYPTO_AUTH_OP_GENERATE)) - goto error_out; + return -EINVAL; } else { session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; /* @@ -723,22 +790,19 @@ qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform, if (qat_alg_aead_session_create_content_desc_auth(session, aead_xform->key.data, aead_xform->key.length, - aead_xform->add_auth_data_length, + aead_xform->aad_length, aead_xform->digest_length, RTE_CRYPTO_AUTH_OP_VERIFY)) - goto error_out; + return -EINVAL; if (qat_alg_aead_session_create_content_desc_cipher(session, aead_xform->key.data, aead_xform->key.length)) - goto error_out; + return -EINVAL; } session->digest_length = aead_xform->digest_length; - return session; - -error_out: - return NULL; + return 0; } unsigned qat_crypto_sym_get_session_private_size( @@ -753,7 +817,8 @@ qat_bpicipher_preprocess(struct qat_session *ctx, { uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); struct rte_crypto_sym_op *sym_op = op->sym; - uint8_t last_block_len = sym_op->cipher.data.length % block_len; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) { @@ -807,7 +872,8 @@ qat_bpicipher_postprocess(struct qat_session *ctx, { uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); struct rte_crypto_sym_op *sym_op = op->sym; - uint8_t last_block_len = sym_op->cipher.data.length % block_len; + uint8_t last_block_len = block_len > 0 ? + sym_op->cipher.data.length % block_len : 0; if (last_block_len > 0 && ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) { @@ -890,7 +956,7 @@ qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops, while (nb_ops_sent != nb_ops_possible) { ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail, - tmp_qp->op_cookies[tail / queue->msg_size]); + tmp_qp->op_cookies[tail / queue->msg_size], tmp_qp); if (ret != 0) { tmp_qp->stats.enqueue_err_count++; /* @@ -946,7 +1012,10 @@ qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops, rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } else { struct qat_session *sess = (struct qat_session *) - (rx_op->sym->session->_private); + get_session_private_data( + rx_op->sym->session, + cryptodev_qat_driver_id); + if (sess->bpi_ctx) qat_bpicipher_postprocess(sess, rx_op); rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; @@ -1044,7 +1113,7 @@ set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, static inline int qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, - struct qat_crypto_op_cookie *qat_op_cookie) + struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp) { int ret = 0; struct qat_session *ctx; @@ -1072,12 +1141,20 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, return -EINVAL; } - if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) { + ctx = (struct qat_session *)get_session_private_data( + op->sym->session, cryptodev_qat_driver_id); + + if (unlikely(ctx == NULL)) { PMD_DRV_LOG(ERR, "Session was not created for this device"); return -EINVAL; } - ctx = (struct qat_session *)op->sym->session->_private; + if (unlikely(ctx->min_qat_dev_gen > qp->qat_dev_gen)) { + PMD_DRV_LOG(ERR, "Session alg not supported on this device gen"); + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return -EINVAL; + } + qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; @@ -1156,18 +1233,7 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, auth_ofs = op->sym->auth.data.offset >> 3; auth_len = op->sym->auth.data.length >> 3; - if (ctx->qat_hash_alg == - ICP_QAT_HW_AUTH_ALGO_KASUMI_F9) { - if (do_cipher) { - auth_len = auth_len + auth_ofs + 1 - - ICP_QAT_HW_KASUMI_BLK_SZ; - auth_ofs = ICP_QAT_HW_KASUMI_BLK_SZ; - } else { - auth_len = auth_len + auth_ofs + 1; - auth_ofs = 0; - } - } else - auth_param->u1.aad_adr = + auth_param->u1.aad_adr = rte_crypto_op_ctophys_offset(op, ctx->auth_iv.offset); @@ -1179,6 +1245,21 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, set_cipher_iv(ctx->auth_iv.length, ctx->auth_iv.offset, cipher_param, op, qat_req); + auth_ofs = op->sym->auth.data.offset; + auth_len = op->sym->auth.data.length; + + auth_param->u1.aad_adr = 0; + auth_param->u2.aad_sz = 0; + + /* + * If len(iv)==12B fw computes J0 + */ + if (ctx->auth_iv.length == 12) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + + } } else { auth_ofs = op->sym->auth.data.offset; auth_len = op->sym->auth.data.length; @@ -1191,6 +1272,21 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, } if (do_aead) { + if (ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || + ctx->qat_hash_alg == + ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { + /* + * If len(iv)==12B fw computes J0 + */ + if (ctx->cipher_iv.length == 12) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + qat_req->comn_hdr.serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); + } + + } + cipher_len = op->sym->aead.data.length; cipher_ofs = op->sym->aead.data.offset; auth_len = op->sym->aead.data.length; @@ -1308,30 +1404,6 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, qat_req->comn_mid.dest_data_addr = dst_buf_start; } - if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || - ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { - if (ctx->cipher_iv.length == 12 || - ctx->auth_iv.length == 12) { - /* - * For GCM a 12 byte IV is allowed, - * but we need to inform the f/w - */ - ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( - qat_req->comn_hdr.serv_specif_flags, - ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); - } - /* GMAC */ - if (!do_aead) { - qat_req->comn_mid.dst_length = - qat_req->comn_mid.src_length = - rte_pktmbuf_data_len(op->sym->m_src); - auth_param->u1.aad_adr = 0; - auth_param->auth_len = op->sym->auth.data.length; - auth_param->auth_off = op->sym->auth.data.offset; - auth_param->u2.aad_sz = 0; - } - } - #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX rte_hexdump(stdout, "qat_req:", qat_req, sizeof(struct icp_qat_fw_la_bulk_req)); @@ -1376,17 +1448,6 @@ static inline uint32_t adf_modulo(uint32_t data, uint32_t shift) return data - mult; } -void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess) -{ - struct rte_cryptodev_sym_session *sess = sym_sess; - struct qat_session *s = (void *)sess->_private; - - PMD_INIT_FUNC_TRACE(); - s->cd_paddr = rte_mempool_virt2phy(mp, sess) + - offsetof(struct qat_session, cd) + - offsetof(struct rte_cryptodev_sym_session, _private); -} - int qat_dev_config(__rte_unused struct rte_cryptodev *dev, __rte_unused struct rte_cryptodev_config *config) { @@ -1433,7 +1494,7 @@ void qat_dev_info_get(struct rte_cryptodev *dev, info->feature_flags = dev->feature_flags; info->capabilities = internals->qat_dev_capabilities; info->sym.max_nb_sessions = internals->max_nb_sessions; - info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD; + info->driver_id = cryptodev_qat_driver_id; info->pci_dev = RTE_DEV_TO_PCI(dev->device); } }