X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcompress%2Fisal%2Fisal_compress_pmd.c;h=7d2b05c91374ed7af84e92c825c0b9c05d369fc3;hb=02afed59258d159c3fa653f91a3faa4905c68702;hp=dbdee39b544b1e293d933e59bb39e4c441ffcb33;hpb=dc49e6aa4879a3220ebd337155571887fde61721;p=dpdk.git diff --git a/drivers/compress/isal/isal_compress_pmd.c b/drivers/compress/isal/isal_compress_pmd.c index dbdee39b54..7d2b05c913 100644 --- a/drivers/compress/isal/isal_compress_pmd.c +++ b/drivers/compress/isal/isal_compress_pmd.c @@ -16,6 +16,14 @@ #define RTE_COMP_ISAL_LEVEL_ONE 1 #define RTE_COMP_ISAL_LEVEL_TWO 2 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */ +#define CHKSUM_SZ_CRC 8 +#define CHKSUM_SZ_ADLER 4 + +#define STRINGIFY(s) #s +#define ISAL_TOSTRING(maj, min, patch) \ + STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch) +#define ISAL_VERSION_STRING \ + ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION) int isal_logtype_driver; @@ -43,12 +51,6 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, } priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE; - /* Set private xform checksum - raw deflate by default */ - if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) { - ISAL_PMD_LOG(ERR, "Checksum not supported\n"); - return -ENOTSUP; - } - /* Set private xform window size, 32K supported */ if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE) priv_xform->compress.window_size = @@ -77,6 +79,28 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, return -ENOTSUP; } + /* Set private xform checksum */ + switch (xform->compress.chksum) { + /* Raw deflate by default */ + case(RTE_COMP_CHECKSUM_NONE): + priv_xform->compress.chksum = IGZIP_DEFLATE; + break; + case(RTE_COMP_CHECKSUM_CRC32): + priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR; + break; + case(RTE_COMP_CHECKSUM_ADLER32): + priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR; + break; + case(RTE_COMP_CHECKSUM_CRC32_ADLER32): + ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" + " supported\n"); + return -ENOTSUP; + default: + ISAL_PMD_LOG(ERR, "Checksum type not supported\n"); + priv_xform->compress.chksum = IGZIP_DEFLATE; + break; + } + /* Set private xform level. * Checking compliance with compressdev API, -1 <= level => 9 */ @@ -170,10 +194,26 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, } priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE; - /* Set private xform checksum - raw deflate by default */ - if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) { - ISAL_PMD_LOG(ERR, "Checksum not supported\n"); + /* Set private xform checksum */ + switch (xform->decompress.chksum) { + /* Raw deflate by default */ + case(RTE_COMP_CHECKSUM_NONE): + priv_xform->decompress.chksum = ISAL_DEFLATE; + break; + case(RTE_COMP_CHECKSUM_CRC32): + priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR; + break; + case(RTE_COMP_CHECKSUM_ADLER32): + priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR; + break; + case(RTE_COMP_CHECKSUM_CRC32_ADLER32): + ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not" + " supported\n"); return -ENOTSUP; + default: + ISAL_PMD_LOG(ERR, "Checksum type not supported\n"); + priv_xform->decompress.chksum = ISAL_DEFLATE; + break; } /* Set private xform window size, 32K supported */ @@ -188,6 +228,186 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform, return 0; } +/* Compression using chained mbufs for input/output data */ +static int +chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp) +{ + int ret; + uint32_t remaining_offset; + uint32_t remaining_data = op->src.length; + struct rte_mbuf *src = op->m_src; + struct rte_mbuf *dst = op->m_dst; + + /* check for source/destination offset passing multiple segments + * and point compression stream to input/output buffer. + */ + remaining_offset = op->src.offset; + while (remaining_offset >= src->data_len) { + remaining_offset -= src->data_len; + src = src->next; + } + qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset, + op->src.length); + qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, + remaining_offset); + + remaining_offset = op->dst.offset; + while (remaining_offset >= dst->data_len) { + remaining_offset -= dst->data_len; + dst = dst->next; + } + qp->stream->avail_out = dst->data_len - remaining_offset; + qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, + remaining_offset); + + if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { + ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + while (qp->stream->internal_state.state != ZSTATE_END) { + /* Last segment of data */ + if (remaining_data <= src->data_len) + qp->stream->end_of_stream = 1; + + /* Execute compression operation */ + ret = isal_deflate(qp->stream); + + remaining_data = op->src.length - qp->stream->total_in; + + if (ret != COMP_OK) { + ISAL_PMD_LOG(ERR, "Compression operation failed\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return ret; + } + + if (qp->stream->avail_in == 0 && + qp->stream->total_in != op->src.length) { + if (src->next != NULL) { + src = src->next; + qp->stream->next_in = + rte_pktmbuf_mtod(src, uint8_t *); + qp->stream->avail_in = + RTE_MIN(remaining_data, src->data_len); + } else { + ISAL_PMD_LOG(ERR, + "Not enough input buffer segments\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + } + + if (qp->stream->avail_out == 0 && + qp->stream->internal_state.state != ZSTATE_END) { + if (dst->next != NULL) { + dst = dst->next; + qp->stream->next_out = + rte_pktmbuf_mtod(dst, uint8_t *); + qp->stream->avail_out = dst->data_len; + } else { + ISAL_PMD_LOG(ERR, + "Not enough output buffer segments\n"); + op->status = + RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + return -1; + } + } + } + + return 0; +} + +/* Decompression using chained mbufs for input/output data */ +static int +chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp) +{ + int ret; + uint32_t consumed_data, src_remaining_offset, dst_remaining_offset; + uint32_t remaining_data = op->src.length; + struct rte_mbuf *src = op->m_src; + struct rte_mbuf *dst = op->m_dst; + + /* check for offset passing multiple segments + * and point decompression state to input/output buffer + */ + src_remaining_offset = op->src.offset; + while (src_remaining_offset >= src->data_len) { + src_remaining_offset -= src->data_len; + src = src->next; + } + qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset, + op->src.length); + qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *, + src_remaining_offset); + + dst_remaining_offset = op->dst.offset; + while (dst_remaining_offset >= dst->data_len) { + dst_remaining_offset -= dst->data_len; + dst = dst->next; + } + qp->state->avail_out = dst->data_len - dst_remaining_offset; + qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *, + dst_remaining_offset); + + while (qp->state->block_state != ISAL_BLOCK_FINISH) { + + ret = isal_inflate(qp->state); + + /* Check for first segment, offset needs to be accounted for */ + if (remaining_data == op->src.length) { + consumed_data = src->data_len - src_remaining_offset; + } else + consumed_data = src->data_len; + + if (qp->state->avail_in == 0 + && op->consumed != op->src.length) { + op->consumed += consumed_data; + remaining_data -= consumed_data; + + if (src->next != NULL) { + src = src->next; + qp->state->next_in = + rte_pktmbuf_mtod(src, uint8_t *); + qp->state->avail_in = + RTE_MIN(remaining_data, src->data_len); + } + } + + if (ret == ISAL_OUT_OVERFLOW) { + ISAL_PMD_LOG(ERR, "Decompression operation ran " + "out of space, but can be recovered.\n%d bytes " + "consumed\t%d bytes produced\n", + consumed_data, qp->state->total_out); + op->status = + RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE; + return ret; + } else if (ret < 0) { + ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return ret; + } + + if (qp->state->avail_out == 0 && + qp->state->block_state != ISAL_BLOCK_FINISH) { + if (dst->next != NULL) { + dst = dst->next; + qp->state->next_out = + rte_pktmbuf_mtod(dst, uint8_t *); + qp->state->avail_out = dst->data_len; + } else { + ISAL_PMD_LOG(ERR, + "Not enough output buffer segments\n"); + op->status = + RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + return -1; + } + } + } + + return 0; +} + /* Stateless Compression Function */ static int process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, @@ -200,30 +420,20 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, uint8_t *temp_level_buf = qp->stream->level_buf; /* Initialize compression stream */ - isal_deflate_stateless_init(qp->stream); + isal_deflate_init(qp->stream); qp->stream->level_buf = temp_level_buf; + /* Set Checksum flag */ + qp->stream->gzip_flag = priv_xform->compress.chksum; + /* Stateless operation, input will be consumed in one go */ qp->stream->flush = NO_FLUSH; - /* set op level & intermediate level buffer */ + /* set compression level & intermediate level buffer size */ qp->stream->level = priv_xform->compress.level; qp->stream->level_buf_size = priv_xform->level_buffer_size; - /* Point compression stream structure to input/output buffers */ - qp->stream->avail_in = op->src.length; - qp->stream->next_in = rte_pktmbuf_mtod(op->m_src, uint8_t *); - qp->stream->avail_out = op->m_dst->data_len; - qp->stream->next_out = rte_pktmbuf_mtod(op->m_dst, uint8_t *); - qp->stream->end_of_stream = 1; /* All input consumed in one go */ - - if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { - ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n"); - op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; - return -1; - } - /* Set op huffman code */ if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED) isal_deflate_set_hufftables(qp->stream, NULL, @@ -238,30 +448,161 @@ process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp, isal_deflate_set_hufftables(qp->stream, NULL, IGZIP_HUFFTABLE_DEFAULT); - /* Execute compression operation */ - ret = isal_deflate_stateless(qp->stream); - - /* Check that output buffer did not run out of space */ - if (ret == STATELESS_OVERFLOW) { - ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); - op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; - return ret; + if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { + ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; } - /* Check that input buffer has been fully consumed */ - if (qp->stream->avail_in != (uint32_t)0) { - ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n"); - op->status = RTE_COMP_OP_STATUS_ERROR; + if (op->dst.offset >= op->m_dst->pkt_len) { + ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough" + " for offset provided.\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; return -1; } - if (ret != COMP_OK) { - op->status = RTE_COMP_OP_STATUS_ERROR; - return ret; + /* Chained mbufs */ + if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { + ret = chained_mbuf_compression(op, qp); + if (ret < 0) + return ret; + } else { + /* Linear buffer */ + qp->stream->end_of_stream = 1; /* All input consumed in one */ + /* Point compression stream to input buffer */ + qp->stream->avail_in = op->src.length; + qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src, + uint8_t *, op->src.offset); + + /* Point compression stream to output buffer */ + qp->stream->avail_out = op->m_dst->data_len - op->dst.offset; + qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst, + uint8_t *, op->dst.offset); + + if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) { + ISAL_PMD_LOG(ERR, "Invalid source or destination" + " buffers\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + /* Execute compression operation */ + ret = isal_deflate_stateless(qp->stream); + + /* Check that output buffer did not run out of space */ + if (ret == STATELESS_OVERFLOW) { + ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); + op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + return ret; + } + + /* Check that input buffer has been fully consumed */ + if (qp->stream->avail_in != (uint32_t)0) { + ISAL_PMD_LOG(ERR, "Input buffer could not be read" + " entirely\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return -1; + } + + if (ret != COMP_OK) { + ISAL_PMD_LOG(ERR, "Compression operation failed\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return ret; + } } op->consumed = qp->stream->total_in; - op->produced = qp->stream->total_out; + if (qp->stream->gzip_flag == IGZIP_DEFLATE) { + op->produced = qp->stream->total_out; + } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) { + op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER; + op->output_chksum = qp->stream->internal_state.crc + 1; + } else { + op->produced = qp->stream->total_out - CHKSUM_SZ_CRC; + op->output_chksum = qp->stream->internal_state.crc; + } + + return ret; +} + +/* Stateless Decompression Function */ +static int +process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp, + struct isal_priv_xform *priv_xform) +{ + int ret = 0; + + op->status = RTE_COMP_OP_STATUS_SUCCESS; + + /* Initialize decompression state */ + isal_inflate_init(qp->state); + + /* Set Checksum flag */ + qp->state->crc_flag = priv_xform->decompress.chksum; + + if (op->m_src->pkt_len < (op->src.length + op->src.offset)) { + ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + if (op->dst.offset >= op->m_dst->pkt_len) { + ISAL_PMD_LOG(ERR, "Output mbuf not big enough for " + "offset provided.\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + /* Chained mbufs */ + if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) { + ret = chained_mbuf_decompression(op, qp); + if (ret != 0) + return ret; + } else { + /* Linear buffer */ + /* Point decompression state to input buffer */ + qp->state->avail_in = op->src.length; + qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src, + uint8_t *, op->src.offset); + + /* Point decompression state to output buffer */ + qp->state->avail_out = op->m_dst->data_len - op->dst.offset; + qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst, + uint8_t *, op->dst.offset); + + if (unlikely(!qp->state->next_in || !qp->state->next_out)) { + ISAL_PMD_LOG(ERR, "Invalid source or destination" + " buffers\n"); + op->status = RTE_COMP_OP_STATUS_INVALID_ARGS; + return -1; + } + + /* Execute decompression operation */ + ret = isal_inflate_stateless(qp->state); + + if (ret == ISAL_OUT_OVERFLOW) { + ISAL_PMD_LOG(ERR, "Output buffer not big enough\n"); + op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED; + return ret; + } + + /* Check that input buffer has been fully consumed */ + if (qp->state->avail_in != (uint32_t)0) { + ISAL_PMD_LOG(ERR, "Input buffer could not be read" + " entirely\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return -1; + } + + if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) { + ISAL_PMD_LOG(ERR, "Decompression operation failed\n"); + op->status = RTE_COMP_OP_STATUS_ERROR; + return ret; + } + op->consumed = op->src.length - qp->state->avail_in; + } + op->produced = qp->state->total_out; + op->output_chksum = qp->state->crc; return ret; } @@ -276,6 +617,7 @@ process_op(struct isal_comp_qp *qp, struct rte_comp_op *op, process_isal_deflate(op, qp, priv_xform); break; case RTE_COMP_DECOMPRESS: + process_isal_inflate(op, qp, priv_xform); break; default: ISAL_PMD_LOG(ERR, "Operation Not Supported\n"); @@ -352,6 +694,8 @@ compdev_isal_create(const char *name, struct rte_vdev_device *vdev, dev->dequeue_burst = isal_comp_pmd_dequeue_burst; dev->enqueue_burst = isal_comp_pmd_enqueue_burst; + ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING); + return 0; } @@ -409,12 +753,9 @@ RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv); RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD, "socket_id="); -RTE_INIT(isal_init_log); - -static void -isal_init_log(void) +RTE_INIT(isal_init_log) { - isal_logtype_driver = rte_log_register("comp_isal"); + isal_logtype_driver = rte_log_register("pmd.compress.isal"); if (isal_logtype_driver >= 0) rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO); }