#define RTE_COMP_ISAL_LEVEL_ONE 1
#define RTE_COMP_ISAL_LEVEL_TWO 2
#define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
+#define CHKSUM_SZ_CRC 8
+#define CHKSUM_SZ_ADLER 4
+
+#define STRINGIFY(s) #s
+#define ISAL_TOSTRING(maj, min, patch) \
+ STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch)
+#define ISAL_VERSION_STRING \
+ ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION)
int isal_logtype_driver;
}
priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
- /* Set private xform checksum - raw deflate by default */
- if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) {
- ISAL_PMD_LOG(ERR, "Checksum not supported\n");
- return -ENOTSUP;
- }
-
/* Set private xform window size, 32K supported */
if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
priv_xform->compress.window_size =
return -ENOTSUP;
}
+ /* Set private xform checksum */
+ switch (xform->compress.chksum) {
+ /* Raw deflate by default */
+ case(RTE_COMP_CHECKSUM_NONE):
+ priv_xform->compress.chksum = IGZIP_DEFLATE;
+ break;
+ case(RTE_COMP_CHECKSUM_CRC32):
+ priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
+ break;
+ case(RTE_COMP_CHECKSUM_ADLER32):
+ priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
+ break;
+ case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+ ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
+ " supported\n");
+ return -ENOTSUP;
+ default:
+ ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
+ priv_xform->compress.chksum = IGZIP_DEFLATE;
+ break;
+ }
+
/* Set private xform level.
* Checking compliance with compressdev API, -1 <= level => 9
*/
}
priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
- /* Set private xform checksum - raw deflate by default */
- if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) {
- ISAL_PMD_LOG(ERR, "Checksum not supported\n");
+ /* Set private xform checksum */
+ switch (xform->decompress.chksum) {
+ /* Raw deflate by default */
+ case(RTE_COMP_CHECKSUM_NONE):
+ priv_xform->decompress.chksum = ISAL_DEFLATE;
+ break;
+ case(RTE_COMP_CHECKSUM_CRC32):
+ priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
+ break;
+ case(RTE_COMP_CHECKSUM_ADLER32):
+ priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
+ break;
+ case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+ ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
+ " supported\n");
return -ENOTSUP;
+ default:
+ ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
+ priv_xform->decompress.chksum = ISAL_DEFLATE;
+ break;
}
/* Set private xform window size, 32K supported */
return 0;
}
+/* Compression using chained mbufs for input/output data */
+static int
+chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
+{
+ int ret;
+ uint32_t remaining_offset;
+ uint32_t remaining_data = op->src.length;
+ struct rte_mbuf *src = op->m_src;
+ struct rte_mbuf *dst = op->m_dst;
+
+ /* check for source/destination offset passing multiple segments
+ * and point compression stream to input/output buffer.
+ */
+ remaining_offset = op->src.offset;
+ while (remaining_offset >= src->data_len) {
+ remaining_offset -= src->data_len;
+ src = src->next;
+ }
+ qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
+ op->src.length);
+ qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
+ remaining_offset);
+
+ remaining_offset = op->dst.offset;
+ while (remaining_offset >= dst->data_len) {
+ remaining_offset -= dst->data_len;
+ dst = dst->next;
+ }
+ qp->stream->avail_out = dst->data_len - remaining_offset;
+ qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
+ remaining_offset);
+
+ if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
+ ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
+ op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+ return -1;
+ }
+
+ while (qp->stream->internal_state.state != ZSTATE_END) {
+ /* Last segment of data */
+ if (remaining_data <= src->data_len)
+ qp->stream->end_of_stream = 1;
+
+ /* Execute compression operation */
+ ret = isal_deflate(qp->stream);
+
+ remaining_data = op->src.length - qp->stream->total_in;
+
+ if (ret != COMP_OK) {
+ ISAL_PMD_LOG(ERR, "Compression operation failed\n");
+ op->status = RTE_COMP_OP_STATUS_ERROR;
+ return ret;
+ }
+
+ if (qp->stream->avail_in == 0 &&
+ qp->stream->total_in != op->src.length) {
+ if (src->next != NULL) {
+ src = src->next;
+ qp->stream->next_in =
+ rte_pktmbuf_mtod(src, uint8_t *);
+ qp->stream->avail_in =
+ RTE_MIN(remaining_data, src->data_len);
+ } else {
+ ISAL_PMD_LOG(ERR,
+ "Not enough input buffer segments\n");
+ op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+ return -1;
+ }
+ }
+
+ if (qp->stream->avail_out == 0 &&
+ qp->stream->internal_state.state != ZSTATE_END) {
+ if (dst->next != NULL) {
+ dst = dst->next;
+ qp->stream->next_out =
+ rte_pktmbuf_mtod(dst, uint8_t *);
+ qp->stream->avail_out = dst->data_len;
+ } else {
+ ISAL_PMD_LOG(ERR,
+ "Not enough output buffer segments\n");
+ op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+ return -1;
+ }
+ }
+ }
+
+ return 0;
+}
+
+/* Decompression using chained mbufs for input/output data */
+static int
+chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
+{
+ int ret;
+ uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
+ uint32_t remaining_data = op->src.length;
+ struct rte_mbuf *src = op->m_src;
+ struct rte_mbuf *dst = op->m_dst;
+
+ /* check for offset passing multiple segments
+ * and point decompression state to input/output buffer
+ */
+ src_remaining_offset = op->src.offset;
+ while (src_remaining_offset >= src->data_len) {
+ src_remaining_offset -= src->data_len;
+ src = src->next;
+ }
+ qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
+ op->src.length);
+ qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
+ src_remaining_offset);
+
+ dst_remaining_offset = op->dst.offset;
+ while (dst_remaining_offset >= dst->data_len) {
+ dst_remaining_offset -= dst->data_len;
+ dst = dst->next;
+ }
+ qp->state->avail_out = dst->data_len - dst_remaining_offset;
+ qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
+ dst_remaining_offset);
+
+ while (qp->state->block_state != ISAL_BLOCK_FINISH) {
+
+ ret = isal_inflate(qp->state);
+
+ /* Check for first segment, offset needs to be accounted for */
+ if (remaining_data == op->src.length) {
+ consumed_data = src->data_len - src_remaining_offset;
+ } else
+ consumed_data = src->data_len;
+
+ if (qp->state->avail_in == 0
+ && op->consumed != op->src.length) {
+ op->consumed += consumed_data;
+ remaining_data -= consumed_data;
+
+ if (src->next != NULL) {
+ src = src->next;
+ qp->state->next_in =
+ rte_pktmbuf_mtod(src, uint8_t *);
+ qp->state->avail_in =
+ RTE_MIN(remaining_data, src->data_len);
+ }
+ }
+
+ if (ret == ISAL_OUT_OVERFLOW) {
+ ISAL_PMD_LOG(ERR, "Decompression operation ran "
+ "out of space, but can be recovered.\n%d bytes "
+ "consumed\t%d bytes produced\n",
+ consumed_data, qp->state->total_out);
+ op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+ return ret;
+ } else if (ret < 0) {
+ ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
+ op->status = RTE_COMP_OP_STATUS_ERROR;
+ return ret;
+ }
+
+ if (qp->state->avail_out == 0 &&
+ qp->state->block_state != ISAL_BLOCK_FINISH) {
+ if (dst->next != NULL) {
+ dst = dst->next;
+ qp->state->next_out =
+ rte_pktmbuf_mtod(dst, uint8_t *);
+ qp->state->avail_out = dst->data_len;
+ } else {
+ ISAL_PMD_LOG(ERR,
+ "Not enough output buffer segments\n");
+ op->status =
+ RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+ return -1;
+ }
+ }
+ }
+
+ return 0;
+}
+
/* Stateless Compression Function */
static int
process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
uint8_t *temp_level_buf = qp->stream->level_buf;
/* Initialize compression stream */
- isal_deflate_stateless_init(qp->stream);
+ isal_deflate_init(qp->stream);
qp->stream->level_buf = temp_level_buf;
+ /* Set Checksum flag */
+ qp->stream->gzip_flag = priv_xform->compress.chksum;
+
/* Stateless operation, input will be consumed in one go */
qp->stream->flush = NO_FLUSH;
- /* set op level & intermediate level buffer */
+ /* set compression level & intermediate level buffer size */
qp->stream->level = priv_xform->compress.level;
qp->stream->level_buf_size = priv_xform->level_buffer_size;
isal_deflate_set_hufftables(qp->stream, NULL,
IGZIP_HUFFTABLE_DEFAULT);
- qp->stream->end_of_stream = 1; /* All input consumed in one go */
- if ((op->src.length + op->src.offset) > op->m_src->data_len) {
- ISAL_PMD_LOG(ERR, "Input mbuf not big enough for the length and"
- " offset provided.\n");
+ if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
+ ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
return -1;
}
- /* Point compression stream to input buffer */
- qp->stream->avail_in = op->src.length;
- qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src, uint8_t *,
- op->src.offset);
-
- if (op->dst.offset > op->m_dst->data_len) {
- ISAL_PMD_LOG(ERR, "Output mbuf not big enough for the length"
- " and offset provided.\n");
- op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
- return -1;
- }
- /* Point compression stream to output buffer */
- qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
- qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst, uint8_t *,
- op->dst.offset);
- if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
- ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n");
+ if (op->dst.offset >= op->m_dst->pkt_len) {
+ ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
+ " for offset provided.\n");
op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
return -1;
}
- /* Execute compression operation */
- ret = isal_deflate_stateless(qp->stream);
+ /* Chained mbufs */
+ if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
+ ret = chained_mbuf_compression(op, qp);
+ if (ret < 0)
+ return ret;
+ } else {
+ /* Linear buffer */
+ qp->stream->end_of_stream = 1; /* All input consumed in one */
+ /* Point compression stream to input buffer */
+ qp->stream->avail_in = op->src.length;
+ qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
+ uint8_t *, op->src.offset);
+
+ /* Point compression stream to output buffer */
+ qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
+ qp->stream->next_out = rte_pktmbuf_mtod_offset(op->m_dst,
+ uint8_t *, op->dst.offset);
+
+ if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
+ ISAL_PMD_LOG(ERR, "Invalid source or destination"
+ " buffers\n");
+ op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+ return -1;
+ }
- /* Check that output buffer did not run out of space */
- if (ret == STATELESS_OVERFLOW) {
- ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
- op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
- return ret;
- }
+ /* Execute compression operation */
+ ret = isal_deflate_stateless(qp->stream);
- /* Check that input buffer has been fully consumed */
- if (qp->stream->avail_in != (uint32_t)0) {
- ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n");
- op->status = RTE_COMP_OP_STATUS_ERROR;
- return -1;
- }
+ /* Check that output buffer did not run out of space */
+ if (ret == STATELESS_OVERFLOW) {
+ ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
+ op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+ return ret;
+ }
+
+ /* Check that input buffer has been fully consumed */
+ if (qp->stream->avail_in != (uint32_t)0) {
+ ISAL_PMD_LOG(ERR, "Input buffer could not be read"
+ " entirely\n");
+ op->status = RTE_COMP_OP_STATUS_ERROR;
+ return -1;
+ }
- if (ret != COMP_OK) {
- op->status = RTE_COMP_OP_STATUS_ERROR;
- return ret;
+ if (ret != COMP_OK) {
+ ISAL_PMD_LOG(ERR, "Compression operation failed\n");
+ op->status = RTE_COMP_OP_STATUS_ERROR;
+ return ret;
+ }
}
op->consumed = qp->stream->total_in;
- op->produced = qp->stream->total_out;
+ if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
+ op->produced = qp->stream->total_out;
+ } else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
+ op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
+ op->output_chksum = qp->stream->internal_state.crc + 1;
+ } else {
+ op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
+ op->output_chksum = qp->stream->internal_state.crc;
+ }
return ret;
}
/* Stateless Decompression Function */
static int
-process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp)
+process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
+ struct isal_priv_xform *priv_xform)
{
int ret = 0;
/* Initialize decompression state */
isal_inflate_init(qp->state);
- if ((op->src.length + op->src.offset) > op->m_src->data_len) {
- ISAL_PMD_LOG(ERR, "Input mbuf not big enough for the length and"
- " offset provided.\n");
- op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
- return -1;
- }
- /* Point decompression state to input buffer */
- qp->state->avail_in = op->src.length;
- qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src, uint8_t *,
- op->src.offset);
-
- if (op->dst.offset > op->m_dst->data_len) {
- ISAL_PMD_LOG(ERR, "Output mbuf not big enough for the length "
- "and offset provided.\n");
+ /* Set Checksum flag */
+ qp->state->crc_flag = priv_xform->decompress.chksum;
+
+ if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
+ ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
return -1;
}
- /* Point decompression state to output buffer */
- qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
- qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst, uint8_t *,
- op->dst.offset);
- if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
- ISAL_PMD_LOG(ERR, "Invalid source or destination buffers\n");
+ if (op->dst.offset >= op->m_dst->pkt_len) {
+ ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
+ "offset provided.\n");
op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
return -1;
}
- /* Execute decompression operation */
- ret = isal_inflate_stateless(qp->state);
+ /* Chained mbufs */
+ if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
+ ret = chained_mbuf_decompression(op, qp);
+ if (ret != 0)
+ return ret;
+ } else {
+ /* Linear buffer */
+ /* Point decompression state to input buffer */
+ qp->state->avail_in = op->src.length;
+ qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
+ uint8_t *, op->src.offset);
+
+ /* Point decompression state to output buffer */
+ qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
+ qp->state->next_out = rte_pktmbuf_mtod_offset(op->m_dst,
+ uint8_t *, op->dst.offset);
+
+ if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
+ ISAL_PMD_LOG(ERR, "Invalid source or destination"
+ " buffers\n");
+ op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+ return -1;
+ }
- if (ret == ISAL_OUT_OVERFLOW) {
- ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
- op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
- return ret;
- }
+ /* Execute decompression operation */
+ ret = isal_inflate_stateless(qp->state);
- /* Check that input buffer has been fully consumed */
- if (qp->state->avail_in != (uint32_t)0) {
- ISAL_PMD_LOG(ERR, "Input buffer could not be read entirely\n");
- op->status = RTE_COMP_OP_STATUS_ERROR;
- return -1;
- }
+ if (ret == ISAL_OUT_OVERFLOW) {
+ ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
+ op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+ return ret;
+ }
- if (ret != ISAL_DECOMP_OK) {
- op->status = RTE_COMP_OP_STATUS_ERROR;
- return ret;
- }
+ /* Check that input buffer has been fully consumed */
+ if (qp->state->avail_in != (uint32_t)0) {
+ ISAL_PMD_LOG(ERR, "Input buffer could not be read"
+ " entirely\n");
+ op->status = RTE_COMP_OP_STATUS_ERROR;
+ return -1;
+ }
- op->consumed = op->src.length - qp->state->avail_in;
+ if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
+ ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
+ op->status = RTE_COMP_OP_STATUS_ERROR;
+ return ret;
+ }
+ op->consumed = op->src.length - qp->state->avail_in;
+ }
op->produced = qp->state->total_out;
+ op->output_chksum = qp->state->crc;
-return ret;
+ return ret;
}
/* Process compression/decompression operation */
process_isal_deflate(op, qp, priv_xform);
break;
case RTE_COMP_DECOMPRESS:
- process_isal_inflate(op, qp);
+ process_isal_inflate(op, qp, priv_xform);
break;
default:
ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
+ ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING);
+
return 0;
}
RTE_INIT(isal_init_log)
{
- isal_logtype_driver = rte_log_register("comp_isal");
+ isal_logtype_driver = rte_log_register("pmd.compress.isal");
if (isal_logtype_driver >= 0)
rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO);
}
git.droids-corp.org / dpdk.git / blobdiff