+ if (rte_ioat_completed_ops(dev_id, COMP_BURST_SZ, status, ¬_ok,
+ (void *)completed_src, (void *)completed_dst) != COMP_BURST_SZ) {
+ PRINT_ERR("Error with rte_ioat_completed_ops\n");
+ rte_rawdev_dump(dev_id, stdout);
+ return -1;
+ }
+ if (not_ok != expected_failures) {
+ unsigned int j;
+ PRINT_ERR("Error, missing expected failed copy, got %u, not %u\n",
+ not_ok, expected_failures);
+ for (j = 0; j < COMP_BURST_SZ; j++)
+ printf("%u ", status[j]);
+ printf("<-- Statuses\n");
+ return -1;
+ }
+ for (j = 0; j < COMP_BURST_SZ; j++) {
+ rte_pktmbuf_free(completed_src[j]);
+ rte_pktmbuf_free(completed_dst[j]);
+ }
+ }
+
+ /* Test gathering status for half batch at a time */
+ for (i = 0; i < RTE_DIM(fail_copy); i++) {
+ uint32_t status[COMP_BURST_SZ] = {0};
+ unsigned int j;
+
+ for (j = 0; j < COMP_BURST_SZ; j++) {
+ srcs[j] = rte_pktmbuf_alloc(pool);
+ dsts[j] = rte_pktmbuf_alloc(pool);
+
+ if (rte_ioat_enqueue_copy(dev_id,
+ (j == fail_copy[i] ? (phys_addr_t)NULL :
+ (srcs[j]->buf_iova + srcs[j]->data_off)),
+ dsts[j]->buf_iova + dsts[j]->data_off,
+ length,
+ (uintptr_t)srcs[j],
+ (uintptr_t)dsts[j]) != 1) {
+ PRINT_ERR("Error with rte_ioat_enqueue_copy for buffer %u\n", j);
+ return -1;
+ }
+ }
+ rte_ioat_perform_ops(dev_id);
+ usleep(100);
+
+ if (rte_ioat_completed_ops(dev_id, COMP_BURST_SZ / 2, status, ¬_ok,
+ (void *)completed_src,
+ (void *)completed_dst) != (COMP_BURST_SZ / 2)) {
+ PRINT_ERR("Error with rte_ioat_completed_ops\n");
+ rte_rawdev_dump(dev_id, stdout);
+ return -1;
+ }
+ if (fail_copy[i] < COMP_BURST_SZ / 2 &&
+ (not_ok != 1 || status[fail_copy[i]] == RTE_IOAT_OP_SUCCESS)) {
+ PRINT_ERR("Missing expected failure in first half-batch\n");
+ rte_rawdev_dump(dev_id, stdout);
+ return -1;
+ }
+ if (rte_ioat_completed_ops(dev_id, COMP_BURST_SZ / 2, status, ¬_ok,
+ (void *)&completed_src[COMP_BURST_SZ / 2],
+ (void *)&completed_dst[COMP_BURST_SZ / 2]) != (COMP_BURST_SZ / 2)) {
+ PRINT_ERR("Error with rte_ioat_completed_ops\n");
+ rte_rawdev_dump(dev_id, stdout);
+ return -1;
+ }
+ if (fail_copy[i] >= COMP_BURST_SZ / 2 && (not_ok != 1 ||
+ status[fail_copy[i] - (COMP_BURST_SZ / 2)]
+ == RTE_IOAT_OP_SUCCESS)) {
+ PRINT_ERR("Missing expected failure in second half-batch\n");
+ rte_rawdev_dump(dev_id, stdout);
+ return -1;
+ }
+
+ for (j = 0; j < COMP_BURST_SZ; j++) {
+ rte_pktmbuf_free(completed_src[j]);
+ rte_pktmbuf_free(completed_dst[j]);
+ }
+ }
+
+ /* Test gathering statuses with fence */
+ for (i = 1; i < RTE_DIM(fail_copy); i++) {
+ uint32_t status[COMP_BURST_SZ * 2] = {0};
+ unsigned int j;
+ uint16_t count;
+
+ for (j = 0; j < COMP_BURST_SZ; j++) {
+ srcs[j] = rte_pktmbuf_alloc(pool);
+ dsts[j] = rte_pktmbuf_alloc(pool);
+
+ /* always fail the first copy */
+ if (rte_ioat_enqueue_copy(dev_id,
+ (j == 0 ? (phys_addr_t)NULL :
+ (srcs[j]->buf_iova + srcs[j]->data_off)),
+ dsts[j]->buf_iova + dsts[j]->data_off,
+ length,
+ (uintptr_t)srcs[j],
+ (uintptr_t)dsts[j]) != 1) {
+ PRINT_ERR("Error with rte_ioat_enqueue_copy for buffer %u\n", j);
+ return -1;
+ }
+ /* put in a fence which will stop any further transactions
+ * because we had a previous failure.
+ */
+ if (j == fail_copy[i])
+ rte_ioat_fence(dev_id);
+ }
+ rte_ioat_perform_ops(dev_id);
+ usleep(100);
+
+ count = rte_ioat_completed_ops(dev_id, COMP_BURST_SZ * 2, status, ¬_ok,
+ (void *)completed_src, (void *)completed_dst);
+ if (count != COMP_BURST_SZ) {
+ PRINT_ERR("Error with rte_ioat_completed_ops, got %u not %u\n",
+ count, COMP_BURST_SZ);
+ for (j = 0; j < count; j++)
+ printf("%u ", status[j]);
+ printf("<-- Statuses\n");
+ return -1;
+ }
+ if (not_ok != COMP_BURST_SZ - fail_copy[i]) {
+ PRINT_ERR("Unexpected failed copy count, got %u, expected %u\n",
+ not_ok, COMP_BURST_SZ - fail_copy[i]);
+ for (j = 0; j < COMP_BURST_SZ; j++)
+ printf("%u ", status[j]);
+ printf("<-- Statuses\n");
+ return -1;
+ }
+ if (status[0] == RTE_IOAT_OP_SUCCESS || status[0] == RTE_IOAT_OP_SKIPPED) {
+ PRINT_ERR("Error, op 0 unexpectedly did not fail.\n");
+ return -1;
+ }
+ for (j = 1; j <= fail_copy[i]; j++) {
+ if (status[j] != RTE_IOAT_OP_SUCCESS) {
+ PRINT_ERR("Error, op %u unexpectedly failed\n", j);
+ return -1;
+ }
+ }
+ for (j = fail_copy[i] + 1; j < COMP_BURST_SZ; j++) {
+ if (status[j] != RTE_IOAT_OP_SKIPPED) {
+ PRINT_ERR("Error, all descriptors after fence should be invalid\n");
+ return -1;
+ }
+ }
+ for (j = 0; j < COMP_BURST_SZ; j++) {
+ rte_pktmbuf_free(completed_src[j]);
+ rte_pktmbuf_free(completed_dst[j]);
+ }
+ }