X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fqat%2Fqat_sym.c;h=e6bf1152327e7f10a13005214d9992ab101e607c;hb=b34d816363b553a6098ef4647e91d5a02b66aca4;hp=2dc0614928e99fe74ca8f056e42d96f692903bfc;hpb=40002f6c2a24547846fee73afd345c3f7848a8f1;p=dpdk.git diff --git a/drivers/crypto/qat/qat_sym.c b/drivers/crypto/qat/qat_sym.c index 2dc0614928..e6bf115232 100644 --- a/drivers/crypto/qat/qat_sym.c +++ b/drivers/crypto/qat/qat_sym.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright(c) 2015-2018 Intel Corporation + * Copyright(c) 2015-2019 Intel Corporation */ #include @@ -9,9 +9,13 @@ #include #include #include +#ifdef RTE_LIBRTE_SECURITY +#include +#endif #include "qat_sym.h" + /** Decrypt a single partial block * Depends on openssl libcrypto * Uses ECB+XOR to do CFB encryption, same result, more performant @@ -61,7 +65,8 @@ qat_bpicipher_preprocess(struct qat_sym_session *ctx, last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *, last_block_offset); - if (unlikely(sym_op->m_dst != NULL)) + if (unlikely((sym_op->m_dst != NULL) + && (sym_op->m_dst != sym_op->m_src))) /* out-of-place operation (OOP) */ dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *, last_block_offset); @@ -97,6 +102,29 @@ qat_bpicipher_preprocess(struct qat_sym_session *ctx, return sym_op->cipher.data.length - last_block_len; } +#ifdef RTE_LIBRTE_SECURITY +static inline void +qat_crc_generate(struct qat_sym_session *ctx, + struct rte_crypto_op *op) +{ + struct rte_crypto_sym_op *sym_op = op->sym; + uint32_t *crc, crc_data_len; + uint8_t *crc_data; + + if (ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT && + sym_op->auth.data.length != 0) { + + crc_data_len = sym_op->auth.data.length; + crc_data = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *, + sym_op->auth.data.offset); + crc = (uint32_t *)(crc_data + crc_data_len); + + *crc = rte_net_crc_calc(crc_data, crc_data_len, + RTE_NET_CRC32_ETH); + } +} +#endif + static inline void set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, struct icp_qat_fw_la_cipher_req_params *cipher_param, @@ -147,7 +175,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, void *op_cookie, enum qat_device_gen qat_dev_gen) { int ret = 0; - struct qat_sym_session *ctx; + struct qat_sym_session *ctx = NULL; struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; register struct icp_qat_fw_la_bulk_req *qat_req; @@ -156,9 +184,11 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, uint32_t auth_len = 0, auth_ofs = 0; uint32_t min_ofs = 0; uint64_t src_buf_start = 0, dst_buf_start = 0; - uint64_t digest_start = 0; + uint64_t auth_data_end = 0; uint8_t do_sgl = 0; uint8_t in_place = 1; + uint8_t is_docsis_sec = 0; + int alignment_adjustment = 0; struct rte_crypto_op *op = (struct rte_crypto_op *)in_op; struct qat_sym_op_cookie *cookie = (struct qat_sym_op_cookie *)op_cookie; @@ -174,11 +204,23 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, QAT_DP_LOG(ERR, "QAT PMD only supports session oriented" " requests, op (%p) is sessionless.", op); return -EINVAL; + } else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + ctx = (struct qat_sym_session *)get_sym_session_private_data( + op->sym->session, qat_sym_driver_id); +#ifdef RTE_LIBRTE_SECURITY + } else { + ctx = (struct qat_sym_session *)get_sec_session_private_data( + op->sym->sec_session); + if (ctx && ctx->bpi_ctx == NULL) { + QAT_DP_LOG(ERR, "QAT PMD only supports security" + " operation requests for DOCSIS, op" + " (%p) is not for DOCSIS.", op); + return -EINVAL; + } + is_docsis_sec = 1; +#endif } - ctx = (struct qat_sym_session *)get_sym_session_private_data( - op->sym->session, cryptodev_qat_driver_id); - if (unlikely(ctx == NULL)) { QAT_DP_LOG(ERR, "Session was not created for this device"); return -EINVAL; @@ -194,7 +236,8 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; cipher_param = (void *)&qat_req->serv_specif_rqpars; - auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param)); + auth_param = (void *)((uint8_t *)cipher_param + + ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { @@ -238,7 +281,31 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, cipher_ofs = op->sym->cipher.data.offset >> 3; } else if (ctx->bpi_ctx) { - /* DOCSIS - only send complete blocks to device + /* DOCSIS processing */ +#ifdef RTE_LIBRTE_SECURITY + if (is_docsis_sec) { + /* Check for OOP or multi-segment buffers */ + if (unlikely(((op->sym->m_dst != NULL) && + (op->sym->m_dst != + op->sym->m_src)) || + op->sym->m_src->nb_segs > 1)) { + QAT_DP_LOG(ERR, + "OOP and/or multi-segment " + "buffers are not supported for " + "DOCSIS security"); + op->status = + RTE_CRYPTO_OP_STATUS_INVALID_ARGS; + return -EINVAL; + } + + /* Calculate CRC */ + qat_crc_generate(ctx, op); + } +#else + RTE_SET_USED(is_docsis_sec); +#endif + + /* Only send complete blocks to device. * Process any partial block using CFB mode. * Even if 0 complete blocks, still send this to device * to get into rx queue for post-process and dequeuing @@ -306,9 +373,8 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, } min_ofs = auth_ofs; - if (likely(ctx->qat_hash_alg != ICP_QAT_HW_AUTH_ALGO_NULL)) - auth_param->auth_res_addr = - op->sym->auth.digest.phys_addr; + auth_param->auth_res_addr = + op->sym->auth.digest.phys_addr; } @@ -425,7 +491,8 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, min_ofs = op->sym->aead.data.offset; } - if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next)) + if (op->sym->m_src->nb_segs > 1 || + (op->sym->m_dst && op->sym->m_dst->nb_segs > 1)) do_sgl = 1; /* adjust for chain case */ @@ -435,7 +502,8 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl)) min_ofs = 0; - if (unlikely(op->sym->m_dst != NULL)) { + if (unlikely((op->sym->m_dst != NULL) && + (op->sym->m_dst != op->sym->m_src))) { /* Out-of-place operation (OOP) * Don't align DMA start. DMA the minimum data-set * so as not to overwrite data in dest buffer @@ -465,6 +533,10 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, min_ofs); } dst_buf_start = src_buf_start; + + /* remember any adjustment for later, note, can be +/- */ + alignment_adjustment = src_buf_start - + rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); } if (do_cipher || do_aead) { @@ -494,33 +566,37 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, : (auth_param->auth_off + auth_param->auth_len); if (do_auth && do_cipher) { + /* Handle digest-encrypted cases, i.e. + * auth-gen-then-cipher-encrypt and + * cipher-decrypt-then-auth-verify + */ + /* First find the end of the data */ if (do_sgl) { uint32_t remaining_off = auth_param->auth_off + - auth_param->auth_len; + auth_param->auth_len + alignment_adjustment; struct rte_mbuf *sgl_buf = (in_place ? - op->sym->m_src : op->sym->m_dst); - while (remaining_off >= rte_pktmbuf_data_len( - sgl_buf)) { - remaining_off -= rte_pktmbuf_data_len( - sgl_buf); + op->sym->m_src : op->sym->m_dst); + + while (remaining_off >= rte_pktmbuf_data_len(sgl_buf) + && sgl_buf->next != NULL) { + remaining_off -= rte_pktmbuf_data_len(sgl_buf); sgl_buf = sgl_buf->next; } - digest_start = (uint64_t)rte_pktmbuf_iova_offset( + + auth_data_end = (uint64_t)rte_pktmbuf_iova_offset( sgl_buf, remaining_off); } else { - digest_start = (in_place ? + auth_data_end = (in_place ? src_buf_start : dst_buf_start) + auth_param->auth_off + auth_param->auth_len; } - /* Handle cases of auth-gen-then-cipher and - * cipher-decrypt-then-auth-verify with digest encrypted - */ + /* Then check if digest-encrypted conditions are met */ if ((auth_param->auth_off + auth_param->auth_len < cipher_param->cipher_offset + cipher_param->cipher_length) && (op->sym->auth.digest.phys_addr == - digest_start)) { + auth_data_end)) { /* Handle partial digest encryption */ if (cipher_param->cipher_offset + cipher_param->cipher_length < @@ -555,7 +631,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, return ret; } - if (likely(op->sym->m_dst == NULL)) + if (in_place) qat_req->comn_mid.dest_data_addr = qat_req->comn_mid.src_data_addr = cookie->qat_sgl_src_phys_addr; @@ -584,6 +660,13 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg, qat_req->comn_mid.dest_data_addr = dst_buf_start; } + /* Handle Single-Pass GCM */ + if (ctx->is_single_pass) { + cipher_param->spc_aad_addr = op->sym->aead.aad.phys_addr; + cipher_param->spc_auth_res_addr = + op->sym->aead.digest.phys_addr; + } + #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req, sizeof(struct icp_qat_fw_la_bulk_req));