X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=app%2Ftest-eventdev%2Ftest_perf_common.c;h=9d1f4a4567ae60e1eb6cb806c4d3defd091dd6a8;hb=a538d1d2d01e85429e7dd011806b52250d37c5dc;hp=d00f918026631e3c63ae421434d4d17d27c1016d;hpb=d9a42a69febf453cdb735e77fc0e01463ddf4acc;p=dpdk.git diff --git a/app/test-eventdev/test_perf_common.c b/app/test-eventdev/test_perf_common.c index d00f918026..9d1f4a4567 100644 --- a/app/test-eventdev/test_perf_common.c +++ b/app/test-eventdev/test_perf_common.c @@ -2,8 +2,12 @@ * Copyright(c) 2017 Cavium, Inc */ +#include + #include "test_perf_common.h" +#define NB_CRYPTODEV_DESCRIPTORS 128 + int perf_test_result(struct evt_test *test, struct evt_options *opt) { @@ -17,7 +21,7 @@ perf_test_result(struct evt_test *test, struct evt_options *opt) total += t->worker[i].processed_pkts; for (i = 0; i < t->nb_workers; i++) printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:" - CLGRN" %3.2f\n"CLNRM, i, + CLGRN" %3.2f"CLNRM"\n", i, t->worker[i].processed_pkts, (((double)t->worker[i].processed_pkts)/total) * 100); @@ -28,6 +32,7 @@ perf_test_result(struct evt_test *test, struct evt_options *opt) static inline int perf_producer(void *arg) { + int i; struct prod_data *p = arg; struct test_perf *t = p->t; struct evt_options *opt = t->opt; @@ -38,7 +43,7 @@ perf_producer(void *arg) const uint32_t nb_flows = t->nb_flows; uint32_t flow_counter = 0; uint64_t count = 0; - struct perf_elt *m; + struct perf_elt *m[BURST_SIZE + 1] = {NULL}; struct rte_event ev; if (opt->verbose_level > 1) @@ -54,27 +59,95 @@ perf_producer(void *arg) ev.sub_event_type = 0; /* stage 0 */ while (count < nb_pkts && t->done == false) { - if (rte_mempool_get(pool, (void **)&m) < 0) + if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) continue; + for (i = 0; i < BURST_SIZE; i++) { + ev.flow_id = flow_counter++ % nb_flows; + ev.event_ptr = m[i]; + m[i]->timestamp = rte_get_timer_cycles(); + while (rte_event_enqueue_burst(dev_id, + port, &ev, 1) != 1) { + if (t->done) + break; + rte_pause(); + m[i]->timestamp = rte_get_timer_cycles(); + } + } + count += BURST_SIZE; + } + + return 0; +} + +static inline int +perf_producer_burst(void *arg) +{ + uint32_t i; + uint64_t timestamp; + struct rte_event_dev_info dev_info; + struct prod_data *p = arg; + struct test_perf *t = p->t; + struct evt_options *opt = t->opt; + const uint8_t dev_id = p->dev_id; + const uint8_t port = p->port_id; + struct rte_mempool *pool = t->pool; + const uint64_t nb_pkts = t->nb_pkts; + const uint32_t nb_flows = t->nb_flows; + uint32_t flow_counter = 0; + uint16_t enq = 0; + uint64_t count = 0; + struct perf_elt *m[MAX_PROD_ENQ_BURST_SIZE + 1]; + struct rte_event ev[MAX_PROD_ENQ_BURST_SIZE + 1]; + uint32_t burst_size = opt->prod_enq_burst_sz; + + memset(m, 0, sizeof(*m) * (MAX_PROD_ENQ_BURST_SIZE + 1)); + rte_event_dev_info_get(dev_id, &dev_info); + if (dev_info.max_event_port_enqueue_depth < burst_size) + burst_size = dev_info.max_event_port_enqueue_depth; + + if (opt->verbose_level > 1) + printf("%s(): lcore %d dev_id %d port=%d queue %d\n", __func__, + rte_lcore_id(), dev_id, port, p->queue_id); + + for (i = 0; i < burst_size; i++) { + ev[i].op = RTE_EVENT_OP_NEW; + ev[i].queue_id = p->queue_id; + ev[i].sched_type = t->opt->sched_type_list[0]; + ev[i].priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + ev[i].event_type = RTE_EVENT_TYPE_CPU; + ev[i].sub_event_type = 0; /* stage 0 */ + } - ev.flow_id = flow_counter++ % nb_flows; - ev.event_ptr = m; - m->timestamp = rte_get_timer_cycles(); - while (rte_event_enqueue_burst(dev_id, port, &ev, 1) != 1) { + while (count < nb_pkts && t->done == false) { + if (rte_mempool_get_bulk(pool, (void **)m, burst_size) < 0) + continue; + timestamp = rte_get_timer_cycles(); + for (i = 0; i < burst_size; i++) { + ev[i].flow_id = flow_counter++ % nb_flows; + ev[i].event_ptr = m[i]; + m[i]->timestamp = timestamp; + } + enq = rte_event_enqueue_burst(dev_id, port, ev, burst_size); + while (enq < burst_size) { + enq += rte_event_enqueue_burst(dev_id, port, + ev + enq, + burst_size - enq); if (t->done) break; rte_pause(); - m->timestamp = rte_get_timer_cycles(); + timestamp = rte_get_timer_cycles(); + for (i = enq; i < burst_size; i++) + m[i]->timestamp = timestamp; } - count++; + count += burst_size; } - return 0; } static inline int perf_event_timer_producer(void *arg) { + int i; struct prod_data *p = arg; struct test_perf *t = p->t; struct evt_options *opt = t->opt; @@ -85,17 +158,19 @@ perf_event_timer_producer(void *arg) const uint32_t nb_flows = t->nb_flows; const uint64_t nb_timers = opt->nb_timers; struct rte_mempool *pool = t->pool; - struct perf_elt *m; + struct perf_elt *m[BURST_SIZE + 1] = {NULL}; struct rte_event_timer_adapter **adptr = t->timer_adptr; struct rte_event_timer tim; uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; memset(&tim, 0, sizeof(struct rte_event_timer)); - timeout_ticks = opt->optm_timer_tick_nsec ? - (timeout_ticks * opt->timer_tick_nsec) - / opt->optm_timer_tick_nsec : timeout_ticks; + timeout_ticks = + opt->optm_timer_tick_nsec + ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / + opt->optm_timer_tick_nsec) + : timeout_ticks; timeout_ticks += timeout_ticks ? 0 : 1; - tim.ev.event_type = RTE_EVENT_TYPE_TIMER; + tim.ev.event_type = RTE_EVENT_TYPE_TIMER; tim.ev.op = RTE_EVENT_OP_NEW; tim.ev.sched_type = t->opt->sched_type_list[0]; tim.ev.queue_id = p->queue_id; @@ -107,29 +182,31 @@ perf_event_timer_producer(void *arg) printf("%s(): lcore %d\n", __func__, rte_lcore_id()); while (count < nb_timers && t->done == false) { - if (rte_mempool_get(pool, (void **)&m) < 0) + if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) continue; - - m->tim = tim; - m->tim.ev.flow_id = flow_counter++ % nb_flows; - m->tim.ev.event_ptr = m; - m->timestamp = rte_get_timer_cycles(); - while (rte_event_timer_arm_burst( - adptr[flow_counter % nb_timer_adptrs], - (struct rte_event_timer **)&m, 1) != 1) { - if (t->done) - break; - rte_pause(); - m->timestamp = rte_get_timer_cycles(); + for (i = 0; i < BURST_SIZE; i++) { + rte_prefetch0(m[i + 1]); + m[i]->tim = tim; + m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; + m[i]->tim.ev.event_ptr = m[i]; + m[i]->timestamp = rte_get_timer_cycles(); + while (rte_event_timer_arm_burst( + adptr[flow_counter % nb_timer_adptrs], + (struct rte_event_timer **)&m[i], 1) != 1) { + if (t->done) + break; + m[i]->timestamp = rte_get_timer_cycles(); + } + arm_latency += rte_get_timer_cycles() - m[i]->timestamp; } - arm_latency += rte_get_timer_cycles() - m->timestamp; - count++; + count += BURST_SIZE; } fflush(stdout); rte_delay_ms(1000); printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", - __func__, rte_lcore_id(), (float)(arm_latency / count) / - (rte_get_timer_hz() / 1000000)); + __func__, rte_lcore_id(), + count ? (float)(arm_latency / count) / + (rte_get_timer_hz() / 1000000) : 0); return 0; } @@ -153,11 +230,13 @@ perf_event_timer_producer_burst(void *arg) uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; memset(&tim, 0, sizeof(struct rte_event_timer)); - timeout_ticks = opt->optm_timer_tick_nsec ? - (timeout_ticks * opt->timer_tick_nsec) - / opt->optm_timer_tick_nsec : timeout_ticks; + timeout_ticks = + opt->optm_timer_tick_nsec + ? ceil((double)(timeout_ticks * opt->timer_tick_nsec) / + opt->optm_timer_tick_nsec) + : timeout_ticks; timeout_ticks += timeout_ticks ? 0 : 1; - tim.ev.event_type = RTE_EVENT_TYPE_TIMER; + tim.ev.event_type = RTE_EVENT_TYPE_TIMER; tim.ev.op = RTE_EVENT_OP_NEW; tim.ev.sched_type = t->opt->sched_type_list[0]; tim.ev.queue_id = p->queue_id; @@ -189,8 +268,126 @@ perf_event_timer_producer_burst(void *arg) fflush(stdout); rte_delay_ms(1000); printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", - __func__, rte_lcore_id(), (float)(arm_latency / count) / - (rte_get_timer_hz() / 1000000)); + __func__, rte_lcore_id(), + count ? (float)(arm_latency / count) / + (rte_get_timer_hz() / 1000000) : 0); + return 0; +} + +static inline void +crypto_adapter_enq_op_new(struct prod_data *p) +{ + struct rte_cryptodev_sym_session **crypto_sess = p->ca.crypto_sess; + struct test_perf *t = p->t; + const uint32_t nb_flows = t->nb_flows; + const uint64_t nb_pkts = t->nb_pkts; + struct rte_mempool *pool = t->pool; + struct rte_crypto_sym_op *sym_op; + struct evt_options *opt = t->opt; + uint16_t qp_id = p->ca.cdev_qp_id; + uint8_t cdev_id = p->ca.cdev_id; + uint32_t flow_counter = 0; + struct rte_crypto_op *op; + struct rte_mbuf *m; + uint64_t count = 0; + uint16_t len; + + if (opt->verbose_level > 1) + printf("%s(): lcore %d queue %d cdev_id %u cdev_qp_id %u\n", + __func__, rte_lcore_id(), p->queue_id, p->ca.cdev_id, + p->ca.cdev_qp_id); + + len = opt->mbuf_sz ? opt->mbuf_sz : RTE_ETHER_MIN_LEN; + + while (count < nb_pkts && t->done == false) { + m = rte_pktmbuf_alloc(pool); + if (m == NULL) + continue; + + rte_pktmbuf_append(m, len); + op = rte_crypto_op_alloc(t->ca_op_pool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + sym_op = op->sym; + sym_op->m_src = m; + sym_op->cipher.data.offset = 0; + sym_op->cipher.data.length = len; + rte_crypto_op_attach_sym_session( + op, crypto_sess[flow_counter++ % nb_flows]); + + while (rte_cryptodev_enqueue_burst(cdev_id, qp_id, &op, 1) != 1 && + t->done == false) + rte_pause(); + + count++; + } +} + +static inline void +crypto_adapter_enq_op_fwd(struct prod_data *p) +{ + struct rte_cryptodev_sym_session **crypto_sess = p->ca.crypto_sess; + const uint8_t dev_id = p->dev_id; + const uint8_t port = p->port_id; + struct test_perf *t = p->t; + const uint32_t nb_flows = t->nb_flows; + const uint64_t nb_pkts = t->nb_pkts; + struct rte_mempool *pool = t->pool; + struct evt_options *opt = t->opt; + struct rte_crypto_sym_op *sym_op; + uint32_t flow_counter = 0; + struct rte_crypto_op *op; + struct rte_event ev; + struct rte_mbuf *m; + uint64_t count = 0; + uint16_t len; + + if (opt->verbose_level > 1) + printf("%s(): lcore %d port %d queue %d cdev_id %u cdev_qp_id %u\n", + __func__, rte_lcore_id(), port, p->queue_id, + p->ca.cdev_id, p->ca.cdev_qp_id); + + ev.event = 0; + ev.op = RTE_EVENT_OP_NEW; + ev.queue_id = p->queue_id; + ev.sched_type = RTE_SCHED_TYPE_ATOMIC; + ev.event_type = RTE_EVENT_TYPE_CPU; + len = opt->mbuf_sz ? opt->mbuf_sz : RTE_ETHER_MIN_LEN; + + while (count < nb_pkts && t->done == false) { + m = rte_pktmbuf_alloc(pool); + if (m == NULL) + continue; + + rte_pktmbuf_append(m, len); + op = rte_crypto_op_alloc(t->ca_op_pool, + RTE_CRYPTO_OP_TYPE_SYMMETRIC); + sym_op = op->sym; + sym_op->m_src = m; + sym_op->cipher.data.offset = 0; + sym_op->cipher.data.length = len; + rte_crypto_op_attach_sym_session( + op, crypto_sess[flow_counter++ % nb_flows]); + ev.event_ptr = op; + + while (rte_event_crypto_adapter_enqueue(dev_id, port, &ev, 1) != 1 && + t->done == false) + rte_pause(); + + count++; + } +} + +static inline int +perf_event_crypto_producer(void *arg) +{ + struct prod_data *p = arg; + struct evt_options *opt = p->t->opt; + + if (opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) + crypto_adapter_enq_op_new(p); + else + crypto_adapter_enq_op_fwd(p); + return 0; } @@ -199,15 +396,29 @@ perf_producer_wrapper(void *arg) { struct prod_data *p = arg; struct test_perf *t = p->t; - /* Launch the producer function only in case of synthetic producer. */ - if (t->opt->prod_type == EVT_PROD_TYPE_SYNT) + bool burst = evt_has_burst_mode(p->dev_id); + + /* In case of synthetic producer, launch perf_producer or + * perf_producer_burst depending on producer enqueue burst size + */ + if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && + t->opt->prod_enq_burst_sz == 1) return perf_producer(arg); + else if (t->opt->prod_type == EVT_PROD_TYPE_SYNT && + t->opt->prod_enq_burst_sz > 1) { + if (!burst) + evt_err("This event device does not support burst mode"); + else + return perf_producer_burst(arg); + } else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && !t->opt->timdev_use_burst) return perf_event_timer_producer(arg); else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && t->opt->timdev_use_burst) return perf_event_timer_producer_burst(arg); + else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) + return perf_event_crypto_producer(arg); return 0; } @@ -217,7 +428,6 @@ processed_pkts(struct test_perf *t) uint8_t i; uint64_t total = 0; - rte_smp_rmb(); for (i = 0; i < t->nb_workers; i++) total += t->worker[i].processed_pkts; @@ -230,7 +440,6 @@ total_latency(struct test_perf *t) uint8_t i; uint64_t total = 0; - rte_smp_rmb(); for (i = 0; i < t->nb_workers; i++) total += t->worker[i].latency; @@ -247,7 +456,7 @@ perf_launch_lcores(struct evt_test *test, struct evt_options *opt, int port_idx = 0; /* launch workers */ - RTE_LCORE_FOREACH_SLAVE(lcore_id) { + RTE_LCORE_FOREACH_WORKER(lcore_id) { if (!(opt->wlcores[lcore_id])) continue; @@ -261,7 +470,7 @@ perf_launch_lcores(struct evt_test *test, struct evt_options *opt, } /* launch producers */ - RTE_LCORE_FOREACH_SLAVE(lcore_id) { + RTE_LCORE_FOREACH_WORKER(lcore_id) { if (!(opt->plcores[lcore_id])) continue; @@ -317,23 +526,25 @@ perf_launch_lcores(struct evt_test *test, struct evt_options *opt, if (remaining <= 0) { t->result = EVT_TEST_SUCCESS; if (opt->prod_type == EVT_PROD_TYPE_SYNT || - opt->prod_type == - EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { + opt->prod_type == + EVT_PROD_TYPE_EVENT_TIMER_ADPTR || + opt->prod_type == + EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { t->done = true; - rte_smp_wmb(); break; } } } if (new_cycles - dead_lock_cycles > dead_lock_sample && - opt->prod_type == EVT_PROD_TYPE_SYNT) { + (opt->prod_type == EVT_PROD_TYPE_SYNT || + opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR || + opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR)) { remaining = t->outstand_pkts - processed_pkts(t); if (dead_lock_remaining == remaining) { rte_event_dev_dump(opt->dev_id, stdout); evt_err("No schedules for seconds, deadlock"); t->done = true; - rte_smp_wmb(); break; } dead_lock_remaining = remaining; @@ -392,21 +603,6 @@ perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, return ret; } } - - ret = rte_eth_dev_start(prod); - if (ret) { - evt_err("Ethernet dev [%d] failed to start." - " Using synthetic producer", prod); - return ret; - } - - ret = rte_event_eth_rx_adapter_start(prod); - if (ret) { - evt_err("Rx adapter[%d] start failed", prod); - return ret; - } - printf("%s: Port[%d] using Rx adapter[%d] started\n", __func__, - prod, prod); } return ret; @@ -431,7 +627,7 @@ perf_event_timer_adapter_setup(struct test_perf *t) .timer_adapter_id = i, .timer_tick_ns = t->opt->timer_tick_nsec, .max_tmo_ns = t->opt->max_tmo_nsec, - .nb_timers = 2 * 1024 * 1024, + .nb_timers = t->opt->pool_sz, .flags = flags, }; @@ -448,7 +644,7 @@ perf_event_timer_adapter_setup(struct test_perf *t) if (!(adapter_info.caps & RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) { - uint32_t service_id; + uint32_t service_id = -1U; rte_event_timer_adapter_service_id_get(wl, &service_id); @@ -460,17 +656,79 @@ perf_event_timer_adapter_setup(struct test_perf *t) } rte_service_runstate_set(service_id, 1); } - - ret = rte_event_timer_adapter_start(wl); - if (ret) { - evt_err("failed to Start event timer adapter %d", i); - return ret; - } t->timer_adptr[i] = wl; } return 0; } +static int +perf_event_crypto_adapter_setup(struct test_perf *t, struct prod_data *p) +{ + struct evt_options *opt = t->opt; + uint32_t cap; + int ret; + + ret = rte_event_crypto_adapter_caps_get(p->dev_id, p->ca.cdev_id, &cap); + if (ret) { + evt_err("Failed to get crypto adapter capabilities"); + return ret; + } + + if (((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) && + !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW)) || + ((opt->crypto_adptr_mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) && + !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD))) { + evt_err("crypto adapter %s mode unsupported\n", + opt->crypto_adptr_mode ? "OP_FORWARD" : "OP_NEW"); + return -ENOTSUP; + } else if (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA)) { + evt_err("Storing crypto session not supported"); + return -ENOTSUP; + } + + if (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) { + struct rte_event response_info; + + response_info.event = 0; + response_info.sched_type = RTE_SCHED_TYPE_ATOMIC; + response_info.queue_id = p->queue_id; + ret = rte_event_crypto_adapter_queue_pair_add( + TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, + &response_info); + } else { + ret = rte_event_crypto_adapter_queue_pair_add( + TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id, NULL); + } + + return ret; +} + +static struct rte_cryptodev_sym_session * +cryptodev_sym_sess_create(struct prod_data *p, struct test_perf *t) +{ + struct rte_crypto_sym_xform cipher_xform; + struct rte_cryptodev_sym_session *sess; + + cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER; + cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_NULL; + cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + cipher_xform.next = NULL; + + sess = rte_cryptodev_sym_session_create(t->ca_sess_pool); + if (sess == NULL) { + evt_err("Failed to create sym session"); + return NULL; + } + + if (rte_cryptodev_sym_session_init(p->ca.cdev_id, sess, &cipher_xform, + t->ca_sess_priv_pool)) { + evt_err("Failed to init session"); + return NULL; + } + + return sess; +} + int perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, uint8_t stride, uint8_t nb_queues, @@ -491,7 +749,10 @@ perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, w->processed_pkts = 0; w->latency = 0; - ret = rte_event_port_setup(opt->dev_id, port, port_conf); + struct rte_event_port_conf conf = *port_conf; + conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_WORKER; + + ret = rte_event_port_setup(opt->dev_id, port, &conf); if (ret) { evt_err("failed to setup port %d", port); return ret; @@ -511,7 +772,10 @@ perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, p->t = t; } - ret = perf_event_rx_adapter_setup(opt, stride, *port_conf); + struct rte_event_port_conf conf = *port_conf; + conf.event_port_cfg |= RTE_EVENT_PORT_CFG_HINT_PRODUCER; + + ret = perf_event_rx_adapter_setup(opt, stride, conf); if (ret) return ret; } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { @@ -526,6 +790,73 @@ perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, ret = perf_event_timer_adapter_setup(t); if (ret) return ret; + } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { + struct rte_event_port_conf conf = *port_conf; + uint8_t cdev_id = 0; + uint16_t qp_id = 0; + + ret = rte_event_crypto_adapter_create(TEST_PERF_CA_ID, + opt->dev_id, &conf, 0); + if (ret) { + evt_err("Failed to create crypto adapter"); + return ret; + } + + prod = 0; + for (; port < perf_nb_event_ports(opt); port++) { + struct rte_cryptodev_sym_session *crypto_sess; + union rte_event_crypto_metadata m_data; + struct prod_data *p = &t->prod[port]; + uint32_t flow_id; + + if (qp_id == rte_cryptodev_queue_pair_count(cdev_id)) { + cdev_id++; + qp_id = 0; + } + + p->dev_id = opt->dev_id; + p->port_id = port; + p->queue_id = prod * stride; + p->ca.cdev_id = cdev_id; + p->ca.cdev_qp_id = qp_id; + p->ca.crypto_sess = rte_zmalloc_socket( + NULL, sizeof(crypto_sess) * t->nb_flows, + RTE_CACHE_LINE_SIZE, opt->socket_id); + p->t = t; + + m_data.request_info.cdev_id = p->ca.cdev_id; + m_data.request_info.queue_pair_id = p->ca.cdev_qp_id; + m_data.response_info.sched_type = RTE_SCHED_TYPE_ATOMIC; + m_data.response_info.queue_id = p->queue_id; + + for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { + crypto_sess = cryptodev_sym_sess_create(p, t); + if (crypto_sess == NULL) + return -ENOMEM; + + m_data.response_info.flow_id = flow_id; + rte_cryptodev_sym_session_set_user_data( + crypto_sess, &m_data, sizeof(m_data)); + p->ca.crypto_sess[flow_id] = crypto_sess; + } + + conf.event_port_cfg |= + RTE_EVENT_PORT_CFG_HINT_PRODUCER | + RTE_EVENT_PORT_CFG_HINT_CONSUMER; + + ret = rte_event_port_setup(opt->dev_id, port, &conf); + if (ret) { + evt_err("failed to setup port %d", port); + return ret; + } + + ret = perf_event_crypto_adapter_setup(t, p); + if (ret) + return ret; + + qp_id++; + prod++; + } } else { prod = 0; for ( ; port < perf_nb_event_ports(opt); port++) { @@ -536,8 +867,12 @@ perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, p->queue_id = prod * stride; p->t = t; - ret = rte_event_port_setup(opt->dev_id, port, - port_conf); + struct rte_event_port_conf conf = *port_conf; + conf.event_port_cfg |= + RTE_EVENT_PORT_CFG_HINT_PRODUCER | + RTE_EVENT_PORT_CFG_HINT_CONSUMER; + + ret = rte_event_port_setup(opt->dev_id, port, &conf); if (ret) { evt_err("failed to setup port %d", port); return ret; @@ -554,8 +889,8 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues) { unsigned int lcores; - /* N producer + N worker + 1 master when producer cores are used - * Else N worker + 1 master when Rx adapter is used + /* N producer + N worker + main when producer cores are used + * Else N worker + main when Rx adapter is used */ lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2; @@ -565,8 +900,8 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues) } /* Validate worker lcores */ - if (evt_lcores_has_overlap(opt->wlcores, rte_get_master_lcore())) { - evt_err("worker lcores overlaps with master lcore"); + if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) { + evt_err("worker lcores overlaps with main lcore"); return -1; } if (evt_lcores_has_overlap_multi(opt->wlcores, opt->plcores)) { @@ -582,11 +917,13 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues) return -1; } - if (opt->prod_type == EVT_PROD_TYPE_SYNT) { + if (opt->prod_type == EVT_PROD_TYPE_SYNT || + opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR || + opt->prod_type == EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) { /* Validate producer lcores */ if (evt_lcores_has_overlap(opt->plcores, - rte_get_master_lcore())) { - evt_err("producer lcores overlaps with master lcore"); + rte_get_main_lcore())) { + evt_err("producer lcores overlaps with main lcore"); return -1; } if (evt_has_disabled_lcore(opt->plcores)) { @@ -645,6 +982,7 @@ perf_opt_dump(struct evt_options *opt, uint8_t nb_queues) evt_dump_queue_priority(opt); evt_dump_sched_type_list(opt); evt_dump_producer_type(opt); + evt_dump("prod_enq_burst_sz", "%d", opt->prod_enq_burst_sz); } void @@ -674,30 +1012,22 @@ int perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) { uint16_t i; + int ret; struct test_perf *t = evt_test_priv(test); struct rte_eth_conf port_conf = { .rxmode = { - .mq_mode = ETH_MQ_RX_RSS, - .max_rx_pkt_len = ETHER_MAX_LEN, + .mq_mode = RTE_ETH_MQ_RX_RSS, .split_hdr_size = 0, - .header_split = 0, - .hw_ip_checksum = 0, - .hw_vlan_filter = 0, - .hw_vlan_strip = 0, - .hw_vlan_extend = 0, - .jumbo_frame = 0, - .hw_strip_crc = 1, }, .rx_adv_conf = { .rss_conf = { .rss_key = NULL, - .rss_hf = ETH_RSS_IP, + .rss_hf = RTE_ETH_RSS_IP, }, }, }; - if (opt->prod_type == EVT_PROD_TYPE_SYNT || - opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) + if (opt->prod_type != EVT_PROD_TYPE_ETH_RX_ADPTR) return 0; if (!rte_eth_dev_count_avail()) { @@ -706,10 +1036,28 @@ perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) } RTE_ETH_FOREACH_DEV(i) { + struct rte_eth_dev_info dev_info; + struct rte_eth_conf local_port_conf = port_conf; + + ret = rte_eth_dev_info_get(i, &dev_info); + if (ret != 0) { + evt_err("Error during getting device (port %u) info: %s\n", + i, strerror(-ret)); + return ret; + } + + local_port_conf.rx_adv_conf.rss_conf.rss_hf &= + dev_info.flow_type_rss_offloads; + if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != + port_conf.rx_adv_conf.rss_conf.rss_hf) { + evt_info("Port %u modified RSS hash function based on hardware support," + "requested:%#"PRIx64" configured:%#"PRIx64"\n", + i, + port_conf.rx_adv_conf.rss_conf.rss_hf, + local_port_conf.rx_adv_conf.rss_conf.rss_hf); + } - if (rte_eth_dev_configure(i, 1, 1, - &port_conf) - < 0) { + if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) { evt_err("Failed to configure eth port [%d]", i); return -EINVAL; } @@ -728,7 +1076,12 @@ perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) return -EINVAL; } - rte_eth_promiscuous_enable(i); + ret = rte_eth_promiscuous_enable(i); + if (ret != 0) { + evt_err("Failed to enable promiscuous mode for eth port [%d]: %s", + i, rte_strerror(-ret)); + return ret; + } } return 0; @@ -743,11 +1096,166 @@ void perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt) RTE_ETH_FOREACH_DEV(i) { rte_event_eth_rx_adapter_stop(i); rte_eth_dev_stop(i); - rte_eth_dev_close(i); } } } +int +perf_cryptodev_setup(struct evt_test *test, struct evt_options *opt) +{ + uint8_t cdev_count, cdev_id, nb_plcores, nb_qps; + struct test_perf *t = evt_test_priv(test); + unsigned int max_session_size; + uint32_t nb_sessions; + int ret; + + if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) + return 0; + + cdev_count = rte_cryptodev_count(); + if (cdev_count == 0) { + evt_err("No crypto devices available\n"); + return -ENODEV; + } + + t->ca_op_pool = rte_crypto_op_pool_create( + "crypto_op_pool", RTE_CRYPTO_OP_TYPE_SYMMETRIC, opt->pool_sz, + 128, 0, rte_socket_id()); + if (t->ca_op_pool == NULL) { + evt_err("Failed to create crypto op pool"); + return -ENOMEM; + } + + nb_sessions = evt_nr_active_lcores(opt->plcores) * t->nb_flows; + t->ca_sess_pool = rte_cryptodev_sym_session_pool_create( + "ca_sess_pool", nb_sessions, 0, 0, + sizeof(union rte_event_crypto_metadata), SOCKET_ID_ANY); + if (t->ca_sess_pool == NULL) { + evt_err("Failed to create sym session pool"); + ret = -ENOMEM; + goto err; + } + + max_session_size = 0; + for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { + unsigned int session_size; + + session_size = + rte_cryptodev_sym_get_private_session_size(cdev_id); + if (session_size > max_session_size) + max_session_size = session_size; + } + + max_session_size += sizeof(union rte_event_crypto_metadata); + t->ca_sess_priv_pool = rte_mempool_create( + "ca_sess_priv_pool", nb_sessions, max_session_size, 0, 0, NULL, + NULL, NULL, NULL, SOCKET_ID_ANY, 0); + if (t->ca_sess_priv_pool == NULL) { + evt_err("failed to create sym session private pool"); + ret = -ENOMEM; + goto err; + } + + /* + * Calculate number of needed queue pairs, based on the amount of + * available number of logical cores and crypto devices. For instance, + * if there are 4 cores and 2 crypto devices, 2 queue pairs will be set + * up per device. + */ + nb_plcores = evt_nr_active_lcores(opt->plcores); + nb_qps = (nb_plcores % cdev_count) ? (nb_plcores / cdev_count) + 1 : + nb_plcores / cdev_count; + for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { + struct rte_cryptodev_qp_conf qp_conf; + struct rte_cryptodev_config conf; + struct rte_cryptodev_info info; + int qp_id; + + rte_cryptodev_info_get(cdev_id, &info); + if (nb_qps > info.max_nb_queue_pairs) { + evt_err("Not enough queue pairs per cryptodev (%u)", + nb_qps); + ret = -EINVAL; + goto err; + } + + conf.nb_queue_pairs = nb_qps; + conf.socket_id = SOCKET_ID_ANY; + conf.ff_disable = RTE_CRYPTODEV_FF_SECURITY; + + ret = rte_cryptodev_configure(cdev_id, &conf); + if (ret) { + evt_err("Failed to configure cryptodev (%u)", cdev_id); + goto err; + } + + qp_conf.nb_descriptors = NB_CRYPTODEV_DESCRIPTORS; + qp_conf.mp_session = t->ca_sess_pool; + qp_conf.mp_session_private = t->ca_sess_priv_pool; + + for (qp_id = 0; qp_id < conf.nb_queue_pairs; qp_id++) { + ret = rte_cryptodev_queue_pair_setup( + cdev_id, qp_id, &qp_conf, + rte_cryptodev_socket_id(cdev_id)); + if (ret) { + evt_err("Failed to setup queue pairs on cryptodev %u\n", + cdev_id); + goto err; + } + } + } + + return 0; +err: + for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) + rte_cryptodev_close(cdev_id); + + rte_mempool_free(t->ca_op_pool); + rte_mempool_free(t->ca_sess_pool); + rte_mempool_free(t->ca_sess_priv_pool); + + return ret; +} + +void +perf_cryptodev_destroy(struct evt_test *test, struct evt_options *opt) +{ + uint8_t cdev_id, cdev_count = rte_cryptodev_count(); + struct test_perf *t = evt_test_priv(test); + uint16_t port; + + if (opt->prod_type != EVT_PROD_TYPE_EVENT_CRYPTO_ADPTR) + return; + + for (port = t->nb_workers; port < perf_nb_event_ports(opt); port++) { + struct rte_cryptodev_sym_session *sess; + struct prod_data *p = &t->prod[port]; + uint32_t flow_id; + uint8_t cdev_id; + + for (flow_id = 0; flow_id < t->nb_flows; flow_id++) { + sess = p->ca.crypto_sess[flow_id]; + cdev_id = p->ca.cdev_id; + rte_cryptodev_sym_session_clear(cdev_id, sess); + rte_cryptodev_sym_session_free(sess); + } + + rte_event_crypto_adapter_queue_pair_del( + TEST_PERF_CA_ID, p->ca.cdev_id, p->ca.cdev_qp_id); + } + + rte_event_crypto_adapter_free(TEST_PERF_CA_ID); + + for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { + rte_cryptodev_stop(cdev_id); + rte_cryptodev_close(cdev_id); + } + + rte_mempool_free(t->ca_op_pool); + rte_mempool_free(t->ca_sess_pool); + rte_mempool_free(t->ca_sess_priv_pool); +} + int perf_mempool_setup(struct evt_test *test, struct evt_options *opt) {