X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcompress%2Fisal%2Fisal_compress_pmd.c;h=7d2b05c91374ed7af84e92c825c0b9c05d369fc3;hb=727758576714ea4de4df01bca53978f321ae323a;hp=1224dd0d3c3b246a9ddd076eb56350d5f1da4785;hpb=c720cd2b35faedbed17df151b3cb1fcf48e1a0bf;p=dpdk.git

diff --git a/drivers/compress/isal/isal_compress_pmd.c b/drivers/compress/isal/isal_compress_pmd.c
index 1224dd0d3c..7d2b05c913 100644
--- a/drivers/compress/isal/isal_compress_pmd.c
+++ b/drivers/compress/isal/isal_compress_pmd.c
@@ -6,6 +6,7 @@
 #include <rte_bus_vdev.h>
 #include <rte_common.h>
 #include <rte_malloc.h>
+#include <rte_mbuf.h>
 #include <rte_compressdev_pmd.h>
 
 #include "isal_compress_pmd_private.h"
@@ -15,6 +16,14 @@
 #define RTE_COMP_ISAL_LEVEL_ONE 1
 #define RTE_COMP_ISAL_LEVEL_TWO 2
 #define RTE_COMP_ISAL_LEVEL_THREE 3 /* Optimised for AVX512 & AVX2 only */
+#define CHKSUM_SZ_CRC 8
+#define CHKSUM_SZ_ADLER 4
+
+#define STRINGIFY(s) #s
+#define ISAL_TOSTRING(maj, min, patch) \
+	STRINGIFY(maj)"."STRINGIFY(min)"."STRINGIFY(patch)
+#define ISAL_VERSION_STRING \
+	ISAL_TOSTRING(ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION)
 
 int isal_logtype_driver;
 
@@ -42,12 +51,6 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 		}
 		priv_xform->compress.algo = RTE_COMP_ALGO_DEFLATE;
 
-		/* Set private xform checksum - raw deflate by default */
-		if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) {
-			ISAL_PMD_LOG(ERR, "Checksum not supported\n");
-			return -ENOTSUP;
-		}
-
 		/* Set private xform window size, 32K supported */
 		if (xform->compress.window_size == RTE_COMP_ISAL_WINDOW_SIZE)
 			priv_xform->compress.window_size =
@@ -76,6 +79,28 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 			return -ENOTSUP;
 		}
 
+		/* Set private xform checksum */
+		switch (xform->compress.chksum) {
+		/* Raw deflate by default */
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->compress.chksum = IGZIP_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->compress.chksum = IGZIP_GZIP_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->compress.chksum = IGZIP_ZLIB_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
+					" supported\n");
+			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
+			priv_xform->compress.chksum = IGZIP_DEFLATE;
+			break;
+		}
+
 		/* Set private xform level.
 		 * Checking compliance with compressdev API, -1 <= level => 9
 		 */
@@ -169,10 +194,26 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 		}
 		priv_xform->decompress.algo = RTE_COMP_ALGO_DEFLATE;
 
-		/* Set private xform checksum - raw deflate by default */
-		if (xform->compress.chksum != RTE_COMP_CHECKSUM_NONE) {
-			ISAL_PMD_LOG(ERR, "Checksum not supported\n");
+		/* Set private xform checksum */
+		switch (xform->decompress.chksum) {
+		/* Raw deflate by default */
+		case(RTE_COMP_CHECKSUM_NONE):
+			priv_xform->decompress.chksum = ISAL_DEFLATE;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32):
+			priv_xform->decompress.chksum = ISAL_GZIP_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_ADLER32):
+			priv_xform->decompress.chksum = ISAL_ZLIB_NO_HDR;
+			break;
+		case(RTE_COMP_CHECKSUM_CRC32_ADLER32):
+			ISAL_PMD_LOG(ERR, "Combined CRC and ADLER checksum not"
+					" supported\n");
 			return -ENOTSUP;
+		default:
+			ISAL_PMD_LOG(ERR, "Checksum type not supported\n");
+			priv_xform->decompress.chksum = ISAL_DEFLATE;
+			break;
 		}
 
 		/* Set private xform window size, 32K supported */
@@ -187,6 +228,452 @@ isal_comp_set_priv_xform_parameters(struct isal_priv_xform *priv_xform,
 	return 0;
 }
 
+/* Compression using chained mbufs for input/output data */
+static int
+chained_mbuf_compression(struct rte_comp_op *op, struct isal_comp_qp *qp)
+{
+	int ret;
+	uint32_t remaining_offset;
+	uint32_t remaining_data = op->src.length;
+	struct rte_mbuf *src = op->m_src;
+	struct rte_mbuf *dst = op->m_dst;
+
+	/* check for source/destination offset passing multiple segments
+	 * and point compression stream to input/output buffer.
+	 */
+	remaining_offset = op->src.offset;
+	while (remaining_offset >= src->data_len) {
+		remaining_offset -= src->data_len;
+		src = src->next;
+	}
+	qp->stream->avail_in = RTE_MIN(src->data_len - remaining_offset,
+			op->src.length);
+	qp->stream->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
+			remaining_offset);
+
+	remaining_offset = op->dst.offset;
+	while (remaining_offset >= dst->data_len) {
+		remaining_offset -= dst->data_len;
+		dst = dst->next;
+	}
+	qp->stream->avail_out = dst->data_len - remaining_offset;
+	qp->stream->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
+			remaining_offset);
+
+	if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
+		ISAL_PMD_LOG(ERR, "Invalid source or destination buffer\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
+
+	while (qp->stream->internal_state.state != ZSTATE_END) {
+		/* Last segment of data */
+		if (remaining_data <= src->data_len)
+			qp->stream->end_of_stream = 1;
+
+		/* Execute compression operation */
+		ret = isal_deflate(qp->stream);
+
+		remaining_data = op->src.length - qp->stream->total_in;
+
+		if (ret != COMP_OK) {
+			ISAL_PMD_LOG(ERR, "Compression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+
+		if (qp->stream->avail_in == 0 &&
+				qp->stream->total_in != op->src.length) {
+			if (src->next != NULL) {
+				src = src->next;
+				qp->stream->next_in =
+						rte_pktmbuf_mtod(src, uint8_t *);
+				qp->stream->avail_in =
+					RTE_MIN(remaining_data, src->data_len);
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough input buffer segments\n");
+				op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+				return -1;
+			}
+		}
+
+		if (qp->stream->avail_out == 0 &&
+				qp->stream->internal_state.state != ZSTATE_END) {
+			if (dst->next != NULL) {
+				dst = dst->next;
+				qp->stream->next_out =
+						rte_pktmbuf_mtod(dst, uint8_t *);
+				qp->stream->avail_out = dst->data_len;
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough output buffer segments\n");
+				op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/* Decompression using chained mbufs for input/output data */
+static int
+chained_mbuf_decompression(struct rte_comp_op *op, struct isal_comp_qp *qp)
+{
+	int ret;
+	uint32_t consumed_data, src_remaining_offset, dst_remaining_offset;
+	uint32_t remaining_data = op->src.length;
+	struct rte_mbuf *src = op->m_src;
+	struct rte_mbuf *dst = op->m_dst;
+
+	/* check for offset passing multiple segments
+	 * and point decompression state to input/output buffer
+	 */
+	src_remaining_offset = op->src.offset;
+	while (src_remaining_offset >= src->data_len) {
+		src_remaining_offset -= src->data_len;
+		src = src->next;
+	}
+	qp->state->avail_in = RTE_MIN(src->data_len - src_remaining_offset,
+			op->src.length);
+	qp->state->next_in = rte_pktmbuf_mtod_offset(src, uint8_t *,
+			src_remaining_offset);
+
+	dst_remaining_offset = op->dst.offset;
+	while (dst_remaining_offset >= dst->data_len) {
+		dst_remaining_offset -= dst->data_len;
+		dst = dst->next;
+	}
+	qp->state->avail_out = dst->data_len - dst_remaining_offset;
+	qp->state->next_out = rte_pktmbuf_mtod_offset(dst, uint8_t *,
+			dst_remaining_offset);
+
+	while (qp->state->block_state != ISAL_BLOCK_FINISH) {
+
+		ret = isal_inflate(qp->state);
+
+		/* Check for first segment, offset needs to be accounted for */
+		if (remaining_data == op->src.length) {
+			consumed_data = src->data_len - src_remaining_offset;
+		} else
+			consumed_data = src->data_len;
+
+		if (qp->state->avail_in == 0
+				&& op->consumed != op->src.length) {
+			op->consumed += consumed_data;
+			remaining_data -= consumed_data;
+
+			if (src->next != NULL) {
+				src = src->next;
+				qp->state->next_in =
+						rte_pktmbuf_mtod(src, uint8_t *);
+				qp->state->avail_in =
+					RTE_MIN(remaining_data, src->data_len);
+			}
+		}
+
+		if (ret == ISAL_OUT_OVERFLOW) {
+			ISAL_PMD_LOG(ERR, "Decompression operation ran "
+				"out of space, but can be recovered.\n%d bytes "
+				"consumed\t%d bytes produced\n",
+				consumed_data, qp->state->total_out);
+				op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE;
+			return ret;
+		} else if (ret < 0) {
+			ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+
+		if (qp->state->avail_out == 0 &&
+				qp->state->block_state != ISAL_BLOCK_FINISH) {
+			if (dst->next != NULL) {
+				dst = dst->next;
+				qp->state->next_out =
+						rte_pktmbuf_mtod(dst, uint8_t *);
+				qp->state->avail_out = dst->data_len;
+			} else {
+				ISAL_PMD_LOG(ERR,
+				"Not enough output buffer segments\n");
+				op->status =
+				RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+				return -1;
+			}
+		}
+	}
+
+	return 0;
+}
+
+/* Stateless Compression Function */
+static int
+process_isal_deflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
+		struct isal_priv_xform *priv_xform)
+{
+	int ret = 0;
+	op->status = RTE_COMP_OP_STATUS_SUCCESS;
+
+	/* Required due to init clearing level_buf */
+	uint8_t *temp_level_buf = qp->stream->level_buf;
+
+	/* Initialize compression stream */
+	isal_deflate_init(qp->stream);
+
+	qp->stream->level_buf = temp_level_buf;
+
+	/* Set Checksum flag */
+	qp->stream->gzip_flag = priv_xform->compress.chksum;
+
+	/* Stateless operation, input will be consumed in one go */
+	qp->stream->flush = NO_FLUSH;
+
+	/* set compression level & intermediate level buffer size */
+	qp->stream->level = priv_xform->compress.level;
+	qp->stream->level_buf_size = priv_xform->level_buffer_size;
+
+	/* Set op huffman code */
+	if (priv_xform->compress.deflate.huffman == RTE_COMP_HUFFMAN_FIXED)
+		isal_deflate_set_hufftables(qp->stream, NULL,
+				IGZIP_HUFFTABLE_STATIC);
+	else if (priv_xform->compress.deflate.huffman ==
+			RTE_COMP_HUFFMAN_DEFAULT)
+		isal_deflate_set_hufftables(qp->stream, NULL,
+			IGZIP_HUFFTABLE_DEFAULT);
+	/* Dynamically change the huffman code to suit the input data */
+	else if (priv_xform->compress.deflate.huffman ==
+			RTE_COMP_HUFFMAN_DYNAMIC)
+		isal_deflate_set_hufftables(qp->stream, NULL,
+				IGZIP_HUFFTABLE_DEFAULT);
+
+	if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
+		ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
+
+	if (op->dst.offset >= op->m_dst->pkt_len) {
+		ISAL_PMD_LOG(ERR, "Output mbuf(s) not big enough"
+				" for offset provided.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
+
+	/* Chained mbufs */
+	if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
+		ret = chained_mbuf_compression(op, qp);
+		if (ret < 0)
+			return ret;
+	} else {
+		/* Linear buffer */
+		qp->stream->end_of_stream = 1; /* All input consumed in one */
+		/* Point compression stream to input buffer */
+		qp->stream->avail_in = op->src.length;
+		qp->stream->next_in = rte_pktmbuf_mtod_offset(op->m_src,
+				uint8_t *, op->src.offset);
+
+		/*  Point compression stream to output buffer */
+		qp->stream->avail_out = op->m_dst->data_len - op->dst.offset;
+		qp->stream->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
+				uint8_t *, op->dst.offset);
+
+		if (unlikely(!qp->stream->next_in || !qp->stream->next_out)) {
+			ISAL_PMD_LOG(ERR, "Invalid source or destination"
+					" buffers\n");
+			op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+			return -1;
+		}
+
+		/* Execute compression operation */
+		ret =  isal_deflate_stateless(qp->stream);
+
+		/* Check that output buffer did not run out of space */
+		if (ret == STATELESS_OVERFLOW) {
+			ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			return ret;
+		}
+
+		/* Check that input buffer has been fully consumed */
+		if (qp->stream->avail_in != (uint32_t)0) {
+			ISAL_PMD_LOG(ERR, "Input buffer could not be read"
+					" entirely\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return -1;
+		}
+
+		if (ret != COMP_OK) {
+			ISAL_PMD_LOG(ERR, "Compression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+	}
+
+	op->consumed = qp->stream->total_in;
+	if (qp->stream->gzip_flag == IGZIP_DEFLATE) {
+		op->produced = qp->stream->total_out;
+	} else if (qp->stream->gzip_flag == IGZIP_ZLIB_NO_HDR) {
+		op->produced = qp->stream->total_out - CHKSUM_SZ_ADLER;
+		op->output_chksum = qp->stream->internal_state.crc + 1;
+	} else {
+		op->produced = qp->stream->total_out - CHKSUM_SZ_CRC;
+		op->output_chksum = qp->stream->internal_state.crc;
+	}
+
+	return ret;
+}
+
+/* Stateless Decompression Function */
+static int
+process_isal_inflate(struct rte_comp_op *op, struct isal_comp_qp *qp,
+		struct isal_priv_xform *priv_xform)
+{
+	int ret = 0;
+
+	op->status = RTE_COMP_OP_STATUS_SUCCESS;
+
+	/* Initialize decompression state */
+	isal_inflate_init(qp->state);
+
+	/* Set Checksum flag */
+	qp->state->crc_flag = priv_xform->decompress.chksum;
+
+	if (op->m_src->pkt_len < (op->src.length + op->src.offset)) {
+		ISAL_PMD_LOG(ERR, "Input mbuf(s) not big enough.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
+
+	if (op->dst.offset >= op->m_dst->pkt_len) {
+		ISAL_PMD_LOG(ERR, "Output mbuf not big enough for "
+				"offset provided.\n");
+		op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+		return -1;
+	}
+
+	/* Chained mbufs */
+	if (op->m_src->nb_segs > 1 || op->m_dst->nb_segs > 1) {
+		ret = chained_mbuf_decompression(op, qp);
+		if (ret !=  0)
+			return ret;
+	} else {
+		/* Linear buffer */
+		/* Point decompression state to input buffer */
+		qp->state->avail_in = op->src.length;
+		qp->state->next_in = rte_pktmbuf_mtod_offset(op->m_src,
+				uint8_t *, op->src.offset);
+
+		/* Point decompression state to output buffer */
+		qp->state->avail_out = op->m_dst->data_len - op->dst.offset;
+		qp->state->next_out  = rte_pktmbuf_mtod_offset(op->m_dst,
+				uint8_t *, op->dst.offset);
+
+		if (unlikely(!qp->state->next_in || !qp->state->next_out)) {
+			ISAL_PMD_LOG(ERR, "Invalid source or destination"
+					" buffers\n");
+			op->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+			return -1;
+		}
+
+		/* Execute decompression operation */
+		ret = isal_inflate_stateless(qp->state);
+
+		if (ret == ISAL_OUT_OVERFLOW) {
+			ISAL_PMD_LOG(ERR, "Output buffer not big enough\n");
+			op->status = RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED;
+			return ret;
+		}
+
+		/* Check that input buffer has been fully consumed */
+		if (qp->state->avail_in != (uint32_t)0) {
+			ISAL_PMD_LOG(ERR, "Input buffer could not be read"
+					" entirely\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return -1;
+		}
+
+		if (ret != ISAL_DECOMP_OK && ret != ISAL_END_INPUT) {
+			ISAL_PMD_LOG(ERR, "Decompression operation failed\n");
+			op->status = RTE_COMP_OP_STATUS_ERROR;
+			return ret;
+		}
+		op->consumed = op->src.length - qp->state->avail_in;
+	}
+	op->produced = qp->state->total_out;
+	op->output_chksum = qp->state->crc;
+
+	return ret;
+}
+
+/* Process compression/decompression operation */
+static int
+process_op(struct isal_comp_qp *qp, struct rte_comp_op *op,
+		struct isal_priv_xform *priv_xform)
+{
+	switch (priv_xform->type) {
+	case RTE_COMP_COMPRESS:
+		process_isal_deflate(op, qp, priv_xform);
+		break;
+	case RTE_COMP_DECOMPRESS:
+		process_isal_inflate(op, qp, priv_xform);
+		break;
+	default:
+		ISAL_PMD_LOG(ERR, "Operation Not Supported\n");
+		return -ENOTSUP;
+	}
+	return 0;
+}
+
+/* Enqueue burst */
+static uint16_t
+isal_comp_pmd_enqueue_burst(void *queue_pair, struct rte_comp_op **ops,
+			uint16_t nb_ops)
+{
+	struct isal_comp_qp *qp = queue_pair;
+	uint16_t i;
+	int retval;
+	int16_t num_enq = RTE_MIN(qp->num_free_elements, nb_ops);
+
+	for (i = 0; i < num_enq; i++) {
+		if (unlikely(ops[i]->op_type != RTE_COMP_OP_STATELESS)) {
+			ops[i]->status = RTE_COMP_OP_STATUS_INVALID_ARGS;
+			ISAL_PMD_LOG(ERR, "Stateful operation not Supported\n");
+			qp->qp_stats.enqueue_err_count++;
+			continue;
+		}
+		retval = process_op(qp, ops[i], ops[i]->private_xform);
+		if (unlikely(retval < 0) ||
+				ops[i]->status != RTE_COMP_OP_STATUS_SUCCESS) {
+			qp->qp_stats.enqueue_err_count++;
+		}
+	}
+
+	retval = rte_ring_enqueue_burst(qp->processed_pkts, (void *)ops,
+			num_enq, NULL);
+	qp->num_free_elements -= retval;
+	qp->qp_stats.enqueued_count += retval;
+
+	return retval;
+}
+
+/* Dequeue burst */
+static uint16_t
+isal_comp_pmd_dequeue_burst(void *queue_pair, struct rte_comp_op **ops,
+		uint16_t nb_ops)
+{
+	struct isal_comp_qp *qp = queue_pair;
+	uint16_t nb_dequeued;
+
+	nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts, (void **)ops,
+			nb_ops, NULL);
+	qp->num_free_elements += nb_dequeued;
+	qp->qp_stats.dequeued_count += nb_dequeued;
+
+	return nb_dequeued;
+}
+
 /* Create ISA-L compression device */
 static int
 compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
@@ -203,6 +690,12 @@ compdev_isal_create(const char *name, struct rte_vdev_device *vdev,
 
 	dev->dev_ops = isal_compress_pmd_ops;
 
+	/* register rx/tx burst functions for data path */
+	dev->dequeue_burst = isal_comp_pmd_dequeue_burst;
+	dev->enqueue_burst = isal_comp_pmd_enqueue_burst;
+
+	ISAL_PMD_LOG(INFO, "\nISA-L library version used: "ISAL_VERSION_STRING);
+
 	return 0;
 }
 
@@ -260,12 +753,9 @@ RTE_PMD_REGISTER_VDEV(COMPDEV_NAME_ISAL_PMD, compdev_isal_pmd_drv);
 RTE_PMD_REGISTER_PARAM_STRING(COMPDEV_NAME_ISAL_PMD,
 	"socket_id=<int>");
 
-RTE_INIT(isal_init_log);
-
-static void
-isal_init_log(void)
+RTE_INIT(isal_init_log)
 {
-	isal_logtype_driver = rte_log_register("comp_isal");
+	isal_logtype_driver = rte_log_register("pmd.compress.isal");
 	if (isal_logtype_driver >= 0)
 		rte_log_set_level(isal_logtype_driver, RTE_LOG_INFO);
 }