+static inline uint32_t adf_modulo(uint32_t data, uint32_t modulo_mask)
+{
+ return data & modulo_mask;
+}
+
+static inline void
+qat_comp_mbuf_skip(struct rte_mbuf **mbuf, uint32_t *offset, uint32_t len)
+{
+ while (*offset + len >= rte_pktmbuf_data_len(*mbuf)) {
+ len -= (rte_pktmbuf_data_len(*mbuf) - *offset);
+ *mbuf = (*mbuf)->next;
+ *offset = 0;
+ }
+ *offset = len;
+}
+
+int
+qat_comp_build_multiple_requests(void *in_op, struct qat_qp *qp,
+ uint32_t parent_tail, int nb_descr)
+{
+ struct rte_comp_op op_backup;
+ struct rte_mbuf dst_mbuf;
+ struct rte_comp_op *op = in_op;
+ struct qat_queue *txq = &(qp->tx_q);
+ uint8_t *base_addr = (uint8_t *)txq->base_addr;
+ uint8_t *out_msg = base_addr + parent_tail;
+ uint32_t tail = parent_tail;
+ struct icp_qat_fw_comp_req *comp_req =
+ (struct icp_qat_fw_comp_req *)out_msg;
+ struct qat_comp_op_cookie *parent_cookie =
+ (struct qat_comp_op_cookie *)
+ qp->op_cookies[parent_tail / txq->msg_size];
+ struct qat_comp_op_cookie *child_cookie;
+ uint16_t dst_data_size =
+ RTE_MIN(RTE_PMD_QAT_COMP_IM_BUFFER_SIZE, 65535);
+ uint32_t data_to_enqueue = op->src.length - QAT_FALLBACK_THLD;
+ int num_descriptors_built = 1;
+ int ret;
+
+ QAT_DP_LOG(DEBUG, "op %p, parent_cookie %p", op, parent_cookie);
+
+ /* copy original op to the local variable for restoring later */
+ rte_memcpy(&op_backup, op, sizeof(op_backup));
+
+ parent_cookie->nb_child_responses = 0;
+ parent_cookie->nb_children = 0;
+ parent_cookie->split_op = 1;
+ parent_cookie->dst_data = op->m_dst;
+ parent_cookie->dst_data_offset = op->dst.offset;
+
+ op->src.length = QAT_FALLBACK_THLD;
+ op->flush_flag = RTE_COMP_FLUSH_FULL;
+
+ QAT_DP_LOG(DEBUG, "parent op src len %u dst len %u",
+ op->src.length, op->m_dst->pkt_len);
+
+ ret = qat_comp_build_request(in_op, out_msg, parent_cookie,
+ qp->qat_dev_gen);
+ if (ret != 0) {
+ /* restore op and clear cookie */
+ QAT_DP_LOG(WARNING, "Failed to build parent descriptor");
+ op->src.length = op_backup.src.length;
+ op->flush_flag = op_backup.flush_flag;
+ parent_cookie->split_op = 0;
+ return ret;
+ }
+
+ /* prepare local dst mbuf */
+ rte_memcpy(&dst_mbuf, op->m_dst, sizeof(dst_mbuf));
+ rte_pktmbuf_reset(&dst_mbuf);
+ dst_mbuf.buf_len = dst_data_size;
+ dst_mbuf.data_len = dst_data_size;
+ dst_mbuf.pkt_len = dst_data_size;
+ dst_mbuf.data_off = 0;
+
+ /* update op for the child operations */
+ op->m_dst = &dst_mbuf;
+ op->dst.offset = 0;
+
+ while (data_to_enqueue) {
+ const struct rte_memzone *mz =
+ parent_cookie->dst_memzones[num_descriptors_built - 1];
+ uint32_t src_data_size = RTE_MIN(data_to_enqueue,
+ QAT_FALLBACK_THLD);
+ uint32_t cookie_index;
+
+ /* update params for the next op */
+ op->src.offset += QAT_FALLBACK_THLD;
+ op->src.length = src_data_size;
+ op->flush_flag = (src_data_size == data_to_enqueue) ?
+ op_backup.flush_flag : RTE_COMP_FLUSH_FULL;
+
+ /* update dst mbuf for the next op (use memzone for dst data) */
+ dst_mbuf.buf_addr = mz->addr;
+ dst_mbuf.buf_iova = mz->iova;
+
+ /* move the tail and calculate next cookie index */
+ tail = adf_modulo(tail + txq->msg_size, txq->modulo_mask);
+ cookie_index = tail / txq->msg_size;
+ child_cookie = (struct qat_comp_op_cookie *)
+ qp->op_cookies[cookie_index];
+ comp_req = (struct icp_qat_fw_comp_req *)(base_addr + tail);
+
+ /* update child cookie */
+ child_cookie->split_op = 1; /* must be set for child as well */
+ child_cookie->parent_cookie = parent_cookie; /* same as above */
+ child_cookie->nb_children = 0;
+ child_cookie->dest_buffer = mz->addr;
+
+ QAT_DP_LOG(DEBUG,
+ "cookie_index %u, child_cookie %p, comp_req %p",
+ cookie_index, child_cookie, comp_req);
+ QAT_DP_LOG(DEBUG,
+ "data_to_enqueue %u, num_descriptors_built %d",
+ data_to_enqueue, num_descriptors_built);
+ QAT_DP_LOG(DEBUG, "child op src len %u dst len %u",
+ op->src.length, op->m_dst->pkt_len);
+
+ /* build the request */
+ ret = qat_comp_build_request(op, (uint8_t *)comp_req,
+ child_cookie, qp->qat_dev_gen);
+ if (ret < 0) {
+ QAT_DP_LOG(WARNING, "Failed to build child descriptor");
+ /* restore op and clear cookie */
+ rte_memcpy(op, &op_backup, sizeof(op_backup));
+ parent_cookie->split_op = 0;
+ parent_cookie->nb_children = 0;
+ return ret;
+ }
+
+ data_to_enqueue -= src_data_size;
+ num_descriptors_built++;
+ }
+
+ /* restore backed up original op */
+ rte_memcpy(op, &op_backup, sizeof(op_backup));
+
+ if (nb_descr != num_descriptors_built)
+ QAT_DP_LOG(ERR, "split op. expected %d, built %d",
+ nb_descr, num_descriptors_built);
+
+ parent_cookie->nb_children = num_descriptors_built - 1;
+ return num_descriptors_built;
+}
+
+static inline void
+qat_comp_response_data_copy(struct qat_comp_op_cookie *cookie,
+ struct rte_comp_op *rx_op)
+{
+ struct qat_comp_op_cookie *pc = cookie->parent_cookie;
+ struct rte_mbuf *sgl_buf = pc->dst_data;
+ void *op_dst_addr = rte_pktmbuf_mtod_offset(sgl_buf, uint8_t *,
+ pc->dst_data_offset);
+
+ /* number of bytes left in the current segment */
+ uint32_t left_in_current = rte_pktmbuf_data_len(sgl_buf) -
+ pc->dst_data_offset;
+
+ uint32_t prod, sent;
+
+ if (rx_op->produced <= left_in_current) {
+ rte_memcpy(op_dst_addr, cookie->dest_buffer,
+ rx_op->produced);
+ /* calculate dst mbuf and offset for the next child op */
+ if (rx_op->produced == left_in_current) {
+ pc->dst_data = sgl_buf->next;
+ pc->dst_data_offset = 0;
+ } else
+ pc->dst_data_offset += rx_op->produced;
+ } else {
+ rte_memcpy(op_dst_addr, cookie->dest_buffer,
+ left_in_current);
+ sgl_buf = sgl_buf->next;
+ prod = rx_op->produced - left_in_current;
+ sent = left_in_current;
+ while (prod > rte_pktmbuf_data_len(sgl_buf)) {
+ op_dst_addr = rte_pktmbuf_mtod_offset(sgl_buf,
+ uint8_t *, 0);
+
+ rte_memcpy(op_dst_addr,
+ ((uint8_t *)cookie->dest_buffer) +
+ sent,
+ rte_pktmbuf_data_len(sgl_buf));
+
+ prod -= rte_pktmbuf_data_len(sgl_buf);
+ sent += rte_pktmbuf_data_len(sgl_buf);
+
+ sgl_buf = sgl_buf->next;
+ }
+
+ op_dst_addr = rte_pktmbuf_mtod_offset(sgl_buf, uint8_t *, 0);
+
+ rte_memcpy(op_dst_addr,
+ ((uint8_t *)cookie->dest_buffer) + sent,
+ prod);
+
+ /* calculate dst mbuf and offset for the next child op */
+ if (prod == rte_pktmbuf_data_len(sgl_buf)) {
+ pc->dst_data = sgl_buf->next;
+ pc->dst_data_offset = 0;
+ } else {
+ pc->dst_data = sgl_buf;
+ pc->dst_data_offset = prod;
+ }
+ }
+}
+