X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=examples%2Fl2fwd-crypto%2Fmain.c;h=66d1491bf76d178e71e9c126668a61eaeaced8ea;hb=a7db3afce75346832059d8bfe54a8f81945fb213;hp=7dfcba421f0a3454d161748668c4b60974cb6f70;hpb=47523597ff6c6ecf5e26e9bd149ab74cca5ec598;p=dpdk.git diff --git a/examples/l2fwd-crypto/main.c b/examples/l2fwd-crypto/main.c index 7dfcba421f..66d1491bf7 100644 --- a/examples/l2fwd-crypto/main.c +++ b/examples/l2fwd-crypto/main.c @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2015-2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2015-2016 Intel Corporation */ #include @@ -48,6 +19,7 @@ #include #include +#include #include #include #include @@ -67,12 +39,13 @@ #include #include #include -#include -#include #include #include #include #include +#ifdef RTE_CRYPTO_SCHEDULER +#include +#endif enum cdev_type { CDEV_TYPE_ANY, @@ -90,7 +63,6 @@ enum cdev_type { #define MAX_AAD_SIZE 65535 #define MAX_PKT_BURST 32 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */ -#define MAX_SESSIONS 32 #define SESSION_POOL_CACHE_SIZE 0 #define MAXIMUM_IV_LENGTH 16 @@ -100,21 +72,21 @@ enum cdev_type { /* * Configurable number of RX/TX ring descriptors */ -#define RTE_TEST_RX_DESC_DEFAULT 128 -#define RTE_TEST_TX_DESC_DEFAULT 512 +#define RTE_TEST_RX_DESC_DEFAULT 1024 +#define RTE_TEST_TX_DESC_DEFAULT 1024 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /* ethernet addresses of ports */ -static struct ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS]; +static struct rte_ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS]; /* mask of enabled ports */ static uint64_t l2fwd_enabled_port_mask; static uint64_t l2fwd_enabled_crypto_mask; /* list of enabled ports */ -static uint32_t l2fwd_dst_ports[RTE_MAX_ETHPORTS]; +static uint16_t l2fwd_dst_ports[RTE_MAX_ETHPORTS]; struct pkt_buffer { @@ -141,7 +113,7 @@ enum l2fwd_crypto_xform_chain { struct l2fwd_key { uint8_t *data; uint32_t length; - phys_addr_t phys_addr; + rte_iova_t phys_addr; }; struct l2fwd_iv { @@ -164,6 +136,7 @@ struct l2fwd_crypto_options { struct rte_crypto_sym_xform cipher_xform; unsigned ckey_param; int ckey_random_size; + uint8_t cipher_key[MAX_KEY_SIZE]; struct l2fwd_iv cipher_iv; unsigned int cipher_iv_param; @@ -172,6 +145,7 @@ struct l2fwd_crypto_options { struct rte_crypto_sym_xform auth_xform; uint8_t akey_param; int akey_random_size; + uint8_t auth_key[MAX_KEY_SIZE]; struct l2fwd_iv auth_iv; unsigned int auth_iv_param; @@ -180,6 +154,7 @@ struct l2fwd_crypto_options { struct rte_crypto_sym_xform aead_xform; unsigned int aead_key_param; int aead_key_random_size; + uint8_t aead_key[MAX_KEY_SIZE]; struct l2fwd_iv aead_iv; unsigned int aead_iv_param; @@ -207,6 +182,8 @@ struct l2fwd_crypto_params { unsigned digest_length; unsigned block_size; + uint16_t cipher_dataunit_len; + struct l2fwd_iv cipher_iv; struct l2fwd_iv auth_iv; struct l2fwd_iv aead_iv; @@ -226,7 +203,7 @@ struct l2fwd_crypto_params { /** lcore configuration */ struct lcore_queue_conf { unsigned nb_rx_ports; - unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE]; + uint16_t rx_port_list[MAX_RX_QUEUE_PER_LCORE]; unsigned nb_crypto_devs; unsigned cryptodev_list[MAX_RX_QUEUE_PER_LCORE]; @@ -237,16 +214,11 @@ struct lcore_queue_conf { struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE]; -static const struct rte_eth_conf port_conf = { +static struct rte_eth_conf port_conf = { .rxmode = { .mq_mode = ETH_MQ_RX_NONE, - .max_rx_pkt_len = ETHER_MAX_LEN, + .max_rx_pkt_len = RTE_ETHER_MAX_LEN, .split_hdr_size = 0, - .header_split = 0, /**< Header Split disabled */ - .hw_ip_checksum = 0, /**< IP checksum offload disabled */ - .hw_vlan_filter = 0, /**< VLAN filtering disabled */ - .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 1, /**< CRC stripped by hardware */ }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, @@ -255,7 +227,10 @@ static const struct rte_eth_conf port_conf = { struct rte_mempool *l2fwd_pktmbuf_pool; struct rte_mempool *l2fwd_crypto_op_pool; -struct rte_mempool *session_pool_socket[RTE_MAX_NUMA_NODES] = { 0 }; +static struct { + struct rte_mempool *sess_mp; + struct rte_mempool *priv_mp; +} session_pool_socket[RTE_MAX_NUMA_NODES]; /* Per-port statistics struct */ struct l2fwd_port_statistics { @@ -292,7 +267,7 @@ print_stats(void) uint64_t total_packets_dropped, total_packets_tx, total_packets_rx; uint64_t total_packets_enqueued, total_packets_dequeued, total_packets_errors; - unsigned portid; + uint16_t portid; uint64_t cdevid; total_packets_dropped = 0; @@ -361,8 +336,11 @@ print_stats(void) total_packets_dropped, total_packets_errors); printf("\n====================================================\n"); + + fflush(stdout); } +/* l2fwd_crypto_send_burst 8< */ static int l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n, struct l2fwd_crypto_params *cparams) @@ -387,7 +365,9 @@ l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n, return 0; } +/* >8 End of l2fwd_crypto_send_burst. */ +/* Crypto enqueue. 8< */ static int l2fwd_crypto_enqueue(struct rte_crypto_op *op, struct l2fwd_crypto_params *cparams) @@ -411,37 +391,38 @@ l2fwd_crypto_enqueue(struct rte_crypto_op *op, qconf->op_buf[cparams->dev_id].len = len; return 0; } +/* >8 End of crypto enqueue. */ static int l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, struct rte_crypto_op *op, struct l2fwd_crypto_params *cparams) { - struct ether_hdr *eth_hdr; - struct ipv4_hdr *ip_hdr; + struct rte_ether_hdr *eth_hdr; + struct rte_ipv4_hdr *ip_hdr; uint32_t ipdata_offset, data_len; uint32_t pad_len = 0; char *padding; - eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); - if (eth_hdr->ether_type != rte_cpu_to_be_16(ETHER_TYPE_IPv4)) + if (eth_hdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) return -1; - ipdata_offset = sizeof(struct ether_hdr); + ipdata_offset = sizeof(struct rte_ether_hdr); - ip_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + + ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + ipdata_offset); - ipdata_offset += (ip_hdr->version_ihl & IPV4_HDR_IHL_MASK) - * IPV4_IHL_MULTIPLIER; + ipdata_offset += (ip_hdr->version_ihl & RTE_IPV4_HDR_IHL_MASK) + * RTE_IPV4_IHL_MULTIPLIER; /* Zero pad data to be crypto'd so it is block aligned */ data_len = rte_pktmbuf_data_len(m) - ipdata_offset; - if (cparams->do_hash && cparams->hash_verify) + if ((cparams->do_hash || cparams->do_aead) && cparams->hash_verify) data_len -= cparams->digest_length; if (cparams->do_cipher) { @@ -459,6 +440,12 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, pad_len = cparams->block_size - (data_len % cparams->block_size); break; + case RTE_CRYPTO_CIPHER_AES_XTS: + if (cparams->cipher_dataunit_len != 0 && + (data_len % cparams->cipher_dataunit_len)) + pad_len = cparams->cipher_dataunit_len - + (data_len % cparams->cipher_dataunit_len); + break; default: pad_len = 0; } @@ -498,7 +485,7 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, uint8_t *) + ipdata_offset + data_len; } - op->sym->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(m, + op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m, rte_pktmbuf_pkt_len(m) - cparams->digest_length); /* For wireless algorithms, offset/length must be in bits */ @@ -559,7 +546,7 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, uint8_t *) + ipdata_offset + data_len; } - op->sym->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(m, + op->sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m, rte_pktmbuf_pkt_len(m) - cparams->digest_length); if (cparams->aad.length) { @@ -596,7 +583,7 @@ l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n, return 0; } -/* Enqueue packets for TX and prepare them to be sent */ +/* Enqueue packets for TX and prepare them to be sent. 8< */ static int l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) { @@ -619,35 +606,50 @@ l2fwd_send_packet(struct rte_mbuf *m, uint16_t port) qconf->pkt_buf[port].len = len; return 0; } +/* >8 End of Enqueuing packets for TX. */ static void -l2fwd_mac_updating(struct rte_mbuf *m, unsigned int dest_portid) +l2fwd_mac_updating(struct rte_mbuf *m, uint16_t dest_portid) { - struct ether_hdr *eth; + struct rte_ether_hdr *eth; void *tmp; - eth = rte_pktmbuf_mtod(m, struct ether_hdr *); + eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); /* 02:00:00:00:00:xx */ tmp = ð->d_addr.addr_bytes[0]; *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40); /* src addr */ - ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], ð->s_addr); + rte_ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], ð->s_addr); } static void -l2fwd_simple_forward(struct rte_mbuf *m, unsigned int portid, +l2fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, struct l2fwd_crypto_options *options) { - unsigned int dst_port; + uint16_t dst_port; + uint32_t pad_len; + struct rte_ipv4_hdr *ip_hdr; + uint32_t ipdata_offset = sizeof(struct rte_ether_hdr); + ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) + + ipdata_offset); dst_port = l2fwd_dst_ports[portid]; if (options->mac_updating) l2fwd_mac_updating(m, dst_port); - l2fwd_send_packet(m, (uint8_t) dst_port); + if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY) + rte_pktmbuf_trim(m, options->auth_xform.auth.digest_length); + + if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { + pad_len = m->pkt_len - rte_be_to_cpu_16(ip_hdr->total_length) - + ipdata_offset; + rte_pktmbuf_trim(m, pad_len); + } + + l2fwd_send_packet(m, dst_port); } /** Generate random key */ @@ -668,6 +670,7 @@ generate_random_key(uint8_t *key, unsigned length) rte_exit(EXIT_FAILURE, "Failed to generate random key\n"); } +/* Session is created and is later attached to the crypto operation. 8< */ static struct rte_cryptodev_sym_session * initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) { @@ -679,7 +682,6 @@ initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) return NULL; uint8_t socket_id = (uint8_t) retval; - struct rte_mempool *sess_mp = session_pool_socket[socket_id]; if (options->xform_chain == L2FWD_CRYPTO_AEAD) { first_xform = &options->aead_xform; @@ -695,17 +697,19 @@ initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id) first_xform = &options->auth_xform; } - session = rte_cryptodev_sym_session_create(sess_mp); - + session = rte_cryptodev_sym_session_create( + session_pool_socket[socket_id].sess_mp); if (session == NULL) return NULL; if (rte_cryptodev_sym_session_init(cdev_id, session, - first_xform, sess_mp) < 0) + first_xform, + session_pool_socket[socket_id].priv_mp) < 0) return NULL; return session; } +/* >8 End of creation of session. */ static void l2fwd_crypto_options_print(struct l2fwd_crypto_options *options); @@ -719,7 +723,8 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) unsigned lcore_id = rte_lcore_id(); uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; - unsigned i, j, portid, nb_rx, len; + unsigned int i, j, nb_rx, len; + uint16_t portid; struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id]; const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; @@ -837,6 +842,8 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) port_cparams[i].cipher_iv.length); port_cparams[i].cipher_algo = options->cipher_xform.cipher.algo; + port_cparams[i].cipher_dataunit_len = + options->cipher_xform.cipher.dataunit_len; /* Set IV parameters */ options->cipher_xform.cipher.iv.offset = IV_OFFSET; options->cipher_xform.cipher.iv.length = @@ -884,7 +891,7 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) continue; l2fwd_send_burst(&lcore_queue_conf[lcore_id], qconf->pkt_buf[portid].len, - (uint8_t) portid); + portid); qconf->pkt_buf[portid].len = 0; } @@ -898,8 +905,8 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) if (unlikely(timer_tsc >= (uint64_t)timer_period)) { - /* do this only on master core */ - if (lcore_id == rte_get_master_lcore() + /* do this only on main core */ + if (lcore_id == rte_get_main_lcore() && options->refresh_period) { print_stats(); timer_tsc = 0; @@ -918,11 +925,12 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) cparams = &port_cparams[i]; - nb_rx = rte_eth_rx_burst((uint8_t) portid, 0, + nb_rx = rte_eth_rx_burst(portid, 0, pkts_burst, MAX_PKT_BURST); port_statistics[portid].rx += nb_rx; + /* Allocate and fillcrypto operations. 8< */ if (nb_rx) { /* * If we can't allocate a crypto_ops, then drop @@ -939,6 +947,7 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) nb_rx = 0; } + /* >8 End of crypto operation allocated and filled. */ /* Enqueue packets from Crypto device*/ for (j = 0; j < nb_rx; j++) { @@ -949,7 +958,7 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) } } - /* Dequeue packets from Crypto device */ + /* Dequeue packets from Crypto device. 8< */ do { nb_rx = rte_cryptodev_dequeue_burst( cparams->dev_id, cparams->qp_id, @@ -967,6 +976,7 @@ l2fwd_main_loop(struct l2fwd_crypto_options *options) options); } } while (nb_rx == MAX_PKT_BURST); + /* >8 End of dequeue packets from crypto device. */ } } } @@ -999,6 +1009,7 @@ l2fwd_crypto_usage(const char *prgname) " --cipher_key_random_size SIZE: size of cipher key when generated randomly\n" " --cipher_iv IV (bytes separated with \":\")\n" " --cipher_iv_random_size SIZE: size of cipher IV when generated randomly\n" + " --cipher_dataunit_len SIZE: length of the algorithm data-unit\n" " --auth_algo ALGO\n" " --auth_op GENERATE / VERIFY\n" @@ -1225,12 +1236,12 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, struct option *lgopts, int option_index) { int retval; + int val; if (strcmp(lgopts[option_index].name, "cdev_type") == 0) { retval = parse_cryptodev_type(&options->type, optarg); if (retval == 0) - snprintf(options->string_type, MAX_STR_LEN, - "%s", optarg); + strlcpy(options->string_type, optarg, MAX_STR_LEN); return retval; } @@ -1249,14 +1260,23 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, else if (strcmp(lgopts[option_index].name, "cipher_key") == 0) { options->ckey_param = 1; options->cipher_xform.cipher.key.length = - parse_bytes(options->cipher_xform.cipher.key.data, optarg, - MAX_KEY_SIZE); + parse_bytes(options->cipher_key, optarg, MAX_KEY_SIZE); if (options->cipher_xform.cipher.key.length > 0) return 0; else return -1; } + else if (strcmp(lgopts[option_index].name, "cipher_dataunit_len") == 0) { + retval = parse_size(&val, optarg); + if (retval == 0 && val >= 0 && val <= UINT16_MAX) { + options->cipher_xform.cipher.dataunit_len = + (uint16_t)val; + return 0; + } else + return -1; + } + else if (strcmp(lgopts[option_index].name, "cipher_key_random_size") == 0) return parse_size(&options->ckey_random_size, optarg); @@ -1286,8 +1306,7 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, else if (strcmp(lgopts[option_index].name, "auth_key") == 0) { options->akey_param = 1; options->auth_xform.auth.key.length = - parse_bytes(options->auth_xform.auth.key.data, optarg, - MAX_KEY_SIZE); + parse_bytes(options->auth_key, optarg, MAX_KEY_SIZE); if (options->auth_xform.auth.key.length > 0) return 0; else @@ -1324,8 +1343,7 @@ l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options, else if (strcmp(lgopts[option_index].name, "aead_key") == 0) { options->aead_key_param = 1; options->aead_xform.aead.key.length = - parse_bytes(options->aead_xform.aead.key.data, optarg, - MAX_KEY_SIZE); + parse_bytes(options->aead_key, optarg, MAX_KEY_SIZE); if (options->aead_xform.aead.key.length > 0) return 0; else @@ -1483,6 +1501,7 @@ l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) options->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC; options->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT; + options->cipher_xform.cipher.dataunit_len = 0; /* Authentication Data */ options->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH; @@ -1507,8 +1526,8 @@ l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) options->aead_iv_random_size = -1; options->aead_iv.length = 0; - options->auth_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; - options->auth_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; + options->aead_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; + options->aead_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; options->aad_param = 0; options->aad_random_size = -1; @@ -1658,6 +1677,7 @@ l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options, { "cipher_key_random_size", required_argument, 0, 0 }, { "cipher_iv", required_argument, 0, 0 }, { "cipher_iv_random_size", required_argument, 0, 0 }, + { "cipher_dataunit_len", required_argument, 0, 0}, { "auth_algo", required_argument, 0, 0 }, { "auth_op", required_argument, 0, 0 }, @@ -1754,33 +1774,38 @@ l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options, /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint16_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ uint16_t portid; uint8_t count, all_ports_up, print_flag = 0; struct rte_eth_link link; + int ret; + char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; printf("\nChecking link status"); fflush(stdout); for (count = 0; count <= MAX_CHECK_TIME; count++) { all_ports_up = 1; - for (portid = 0; portid < port_num; portid++) { + RTE_ETH_FOREACH_DEV(portid) { if ((port_mask & (1 << portid)) == 0) continue; memset(&link, 0, sizeof(link)); - rte_eth_link_get_nowait(portid, &link); + ret = rte_eth_link_get_nowait(portid, &link); + if (ret < 0) { + all_ports_up = 0; + if (print_flag == 1) + printf("Port %u link get failed: %s\n", + portid, rte_strerror(-ret)); + continue; + } /* print link status if flag set */ if (print_flag == 1) { - if (link.link_status) - printf( - "Port%d Link Up. Speed %u Mbps - %s\n", - portid, link.link_speed, - (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? - ("full-duplex") : ("half-duplex\n")); - else - printf("Port %d Link Down\n", portid); + rte_eth_link_to_str(link_status_text, + sizeof(link_status_text), &link); + printf("Port %d %s\n", portid, + link_status_text); continue; } /* clear all_ports_up flag if any link down */ @@ -1963,21 +1988,19 @@ check_supported_size(uint16_t length, uint16_t min, uint16_t max, static int check_iv_param(const struct rte_crypto_param_range *iv_range_size, unsigned int iv_param, int iv_random_size, - uint16_t *iv_length) + uint16_t iv_length) { /* * Check if length of provided IV is supported * by the algorithm chosen. */ if (iv_param) { - if (check_supported_size(*iv_length, + if (check_supported_size(iv_length, iv_range_size->min, iv_range_size->max, iv_range_size->increment) - != 0) { - printf("Unsupported IV length\n"); + != 0) return -1; - } /* * Check if length of IV to be randomly generated * is supported by the algorithm chosen. @@ -1987,14 +2010,301 @@ check_iv_param(const struct rte_crypto_param_range *iv_range_size, iv_range_size->min, iv_range_size->max, iv_range_size->increment) - != 0) { - printf("Unsupported IV length\n"); + != 0) + return -1; + } + + return 0; +} + +static int +check_capabilities(struct l2fwd_crypto_options *options, uint8_t cdev_id) +{ + struct rte_cryptodev_info dev_info; + const struct rte_cryptodev_capabilities *cap; + + rte_cryptodev_info_get(cdev_id, &dev_info); + + /* Set AEAD parameters */ + if (options->xform_chain == L2FWD_CRYPTO_AEAD) { + /* Check if device supports AEAD algo */ + cap = check_device_support_aead_algo(options, &dev_info, + cdev_id); + if (cap == NULL) + return -1; + + if (check_iv_param(&cap->sym.aead.iv_size, + options->aead_iv_param, + options->aead_iv_random_size, + options->aead_iv.length) != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support IV length\n", + cdev_id); return -1; } - *iv_length = iv_random_size; - /* No size provided, use minimum size. */ - } else - *iv_length = iv_range_size->min; + + /* + * Check if length of provided AEAD key is supported + * by the algorithm chosen. + */ + if (options->aead_key_param) { + if (check_supported_size( + options->aead_xform.aead.key.length, + cap->sym.aead.key_size.min, + cap->sym.aead.key_size.max, + cap->sym.aead.key_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support " + "AEAD key length\n", + cdev_id); + return -1; + } + /* + * Check if length of the aead key to be randomly generated + * is supported by the algorithm chosen. + */ + } else if (options->aead_key_random_size != -1) { + if (check_supported_size(options->aead_key_random_size, + cap->sym.aead.key_size.min, + cap->sym.aead.key_size.max, + cap->sym.aead.key_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support " + "AEAD key length\n", + cdev_id); + return -1; + } + } + + + /* + * Check if length of provided AAD is supported + * by the algorithm chosen. + */ + if (options->aad_param) { + if (check_supported_size(options->aad.length, + cap->sym.aead.aad_size.min, + cap->sym.aead.aad_size.max, + cap->sym.aead.aad_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support " + "AAD length\n", + cdev_id); + return -1; + } + /* + * Check if length of AAD to be randomly generated + * is supported by the algorithm chosen. + */ + } else if (options->aad_random_size != -1) { + if (check_supported_size(options->aad_random_size, + cap->sym.aead.aad_size.min, + cap->sym.aead.aad_size.max, + cap->sym.aead.aad_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support " + "AAD length\n", + cdev_id); + return -1; + } + } + + /* Check if digest size is supported by the algorithm. */ + if (options->digest_size != -1) { + if (check_supported_size(options->digest_size, + cap->sym.aead.digest_size.min, + cap->sym.aead.digest_size.max, + cap->sym.aead.digest_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support " + "digest length\n", + cdev_id); + return -1; + } + } + } + + /* Set cipher parameters */ + if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || + options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || + options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { + + /* Check if device supports cipher algo. 8< */ + cap = check_device_support_cipher_algo(options, &dev_info, + cdev_id); + if (cap == NULL) + return -1; + + if (check_iv_param(&cap->sym.cipher.iv_size, + options->cipher_iv_param, + options->cipher_iv_random_size, + options->cipher_iv.length) != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support IV length\n", + cdev_id); + return -1; + } + /* >8 End of check if device supports cipher algo. */ + + /* Check if capable cipher is supported. 8< */ + + /* + * Check if length of provided cipher key is supported + * by the algorithm chosen. + */ + if (options->ckey_param) { + if (check_supported_size( + options->cipher_xform.cipher.key.length, + cap->sym.cipher.key_size.min, + cap->sym.cipher.key_size.max, + cap->sym.cipher.key_size.increment) + != 0) { + if (dev_info.feature_flags & + RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) { + RTE_LOG(DEBUG, USER1, + "Key length does not match the device " + "%u capability. Key may be wrapped\n", + cdev_id); + } else { + RTE_LOG(DEBUG, USER1, + "Key length does not match the device " + "%u capability\n", + cdev_id); + return -1; + } + } + + /* + * Check if length of the cipher key to be randomly generated + * is supported by the algorithm chosen. + */ + } else if (options->ckey_random_size != -1) { + if (check_supported_size(options->ckey_random_size, + cap->sym.cipher.key_size.min, + cap->sym.cipher.key_size.max, + cap->sym.cipher.key_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support cipher " + "key length\n", + cdev_id); + return -1; + } + } + + if (options->cipher_xform.cipher.dataunit_len > 0) { + if (!(dev_info.feature_flags & + RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support " + "cipher multiple data units\n", + cdev_id); + return -1; + } + if (cap->sym.cipher.dataunit_set != 0) { + int ret = 0; + + switch (options->cipher_xform.cipher.dataunit_len) { + case 512: + if (!(cap->sym.cipher.dataunit_set & + RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES)) + ret = -1; + break; + case 4096: + if (!(cap->sym.cipher.dataunit_set & + RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES)) + ret = -1; + break; + default: + ret = -1; + } + if (ret == -1) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support " + "data-unit length %u\n", + cdev_id, + options->cipher_xform.cipher.dataunit_len); + return -1; + } + } + } + /* >8 End of checking if cipher is supported. */ + } + + /* Set auth parameters */ + if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || + options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || + options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { + /* Check if device supports auth algo */ + cap = check_device_support_auth_algo(options, &dev_info, + cdev_id); + if (cap == NULL) + return -1; + + if (check_iv_param(&cap->sym.auth.iv_size, + options->auth_iv_param, + options->auth_iv_random_size, + options->auth_iv.length) != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support IV length\n", + cdev_id); + return -1; + } + /* + * Check if length of provided auth key is supported + * by the algorithm chosen. + */ + if (options->akey_param) { + if (check_supported_size( + options->auth_xform.auth.key.length, + cap->sym.auth.key_size.min, + cap->sym.auth.key_size.max, + cap->sym.auth.key_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support auth " + "key length\n", + cdev_id); + return -1; + } + /* + * Check if length of the auth key to be randomly generated + * is supported by the algorithm chosen. + */ + } else if (options->akey_random_size != -1) { + if (check_supported_size(options->akey_random_size, + cap->sym.auth.key_size.min, + cap->sym.auth.key_size.max, + cap->sym.auth.key_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support auth " + "key length\n", + cdev_id); + return -1; + } + } + + /* Check if digest size is supported by the algorithm. */ + if (options->digest_size != -1) { + if (check_supported_size(options->digest_size, + cap->sym.auth.digest_size.min, + cap->sym.auth.digest_size.max, + cap->sym.auth.digest_size.increment) + != 0) { + RTE_LOG(DEBUG, USER1, + "Device %u does not support " + "digest length\n", + cdev_id); + return -1; + } + } + } return 0; } @@ -2003,9 +2313,10 @@ static int initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, uint8_t *enabled_cdevs) { - unsigned int cdev_id, cdev_count, enabled_cdev_count = 0; + uint8_t cdev_id, cdev_count, enabled_cdev_count = 0; const struct rte_cryptodev_capabilities *cap; unsigned int sess_sz, max_sess_sz = 0; + uint32_t sessions_needed = 0; int retval; cdev_count = rte_cryptodev_count(); @@ -2014,16 +2325,37 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, return -1; } - for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { - sess_sz = rte_cryptodev_get_private_session_size(cdev_id); + for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports; + cdev_id++) { + if (check_cryptodev_mask(options, cdev_id) < 0) + continue; + + if (check_capabilities(options, cdev_id) < 0) + continue; + + sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id); if (sess_sz > max_sess_sz) max_sess_sz = sess_sz; + + l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id); + + enabled_cdevs[cdev_id] = 1; + enabled_cdev_count++; } - for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports; - cdev_id++) { + for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) { struct rte_cryptodev_qp_conf qp_conf; struct rte_cryptodev_info dev_info; + + if (enabled_cdevs[cdev_id] == 0) + continue; + + if (check_cryptodev_mask(options, cdev_id) < 0) + continue; + + if (check_capabilities(options, cdev_id) < 0) + continue; + retval = rte_cryptodev_socket_id(cdev_id); if (retval < 0) { @@ -2036,144 +2368,123 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, struct rte_cryptodev_config conf = { .nb_queue_pairs = 1, .socket_id = socket_id, + .ff_disable = RTE_CRYPTODEV_FF_SECURITY, }; - if (check_cryptodev_mask(options, (uint8_t)cdev_id)) - continue; - rte_cryptodev_info_get(cdev_id, &dev_info); - if (session_pool_socket[socket_id] == NULL) { + /* + * Two sessions objects are required for each session + * (one for the header, one for the private data) + */ + if (!strcmp(dev_info.driver_name, "crypto_scheduler")) { +#ifdef RTE_CRYPTO_SCHEDULER + uint32_t nb_workers = + rte_cryptodev_scheduler_workers_get(cdev_id, + NULL); + + sessions_needed = enabled_cdev_count * nb_workers; +#endif + } else + sessions_needed = enabled_cdev_count; + + if (session_pool_socket[socket_id].priv_mp == NULL) { char mp_name[RTE_MEMPOOL_NAMESIZE]; - struct rte_mempool *sess_mp; snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, - "sess_mp_%u", socket_id); + "priv_sess_mp_%u", socket_id); - /* - * Create enough objects for session headers and - * device private data - */ - sess_mp = rte_mempool_create(mp_name, - MAX_SESSIONS * 2, + session_pool_socket[socket_id].priv_mp = + rte_mempool_create(mp_name, + sessions_needed, max_sess_sz, - SESSION_POOL_CACHE_SIZE, - 0, NULL, NULL, NULL, + 0, 0, NULL, NULL, NULL, NULL, socket_id, 0); - if (sess_mp == NULL) { - printf("Cannot create session pool on socket %d\n", + if (session_pool_socket[socket_id].priv_mp == NULL) { + printf("Cannot create pool on socket %d\n", socket_id); return -ENOMEM; } - printf("Allocated session pool on socket %d\n", socket_id); - session_pool_socket[socket_id] = sess_mp; + printf("Allocated pool \"%s\" on socket %d\n", + mp_name, socket_id); + } + + if (session_pool_socket[socket_id].sess_mp == NULL) { + char mp_name[RTE_MEMPOOL_NAMESIZE]; + snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, + "sess_mp_%u", socket_id); + + session_pool_socket[socket_id].sess_mp = + rte_cryptodev_sym_session_pool_create( + mp_name, + sessions_needed, + 0, 0, 0, socket_id); + + if (session_pool_socket[socket_id].sess_mp == NULL) { + printf("Cannot create pool on socket %d\n", + socket_id); + return -ENOMEM; + } + + printf("Allocated pool \"%s\" on socket %d\n", + mp_name, socket_id); } /* Set AEAD parameters */ if (options->xform_chain == L2FWD_CRYPTO_AEAD) { - /* Check if device supports AEAD algo */ cap = check_device_support_aead_algo(options, &dev_info, cdev_id); - if (cap == NULL) - continue; options->block_size = cap->sym.aead.block_size; - check_iv_param(&cap->sym.aead.iv_size, - options->aead_iv_param, - options->aead_iv_random_size, - &options->aead_iv.length); + /* Set IV if not provided from command line */ + if (options->aead_iv_param == 0) { + if (options->aead_iv_random_size != -1) + options->aead_iv.length = + options->aead_iv_random_size; + /* No size provided, use minimum size. */ + else + options->aead_iv.length = + cap->sym.aead.iv_size.min; + } - /* - * Check if length of provided AEAD key is supported - * by the algorithm chosen. - */ - if (options->aead_key_param) { - if (check_supported_size( - options->aead_xform.aead.key.length, - cap->sym.aead.key_size.min, - cap->sym.aead.key_size.max, - cap->sym.aead.key_size.increment) - != 0) { - printf("Unsupported aead key length\n"); - return -1; - } - /* - * Check if length of the aead key to be randomly generated - * is supported by the algorithm chosen. - */ - } else if (options->aead_key_random_size != -1) { - if (check_supported_size(options->aead_key_random_size, - cap->sym.aead.key_size.min, - cap->sym.aead.key_size.max, - cap->sym.aead.key_size.increment) - != 0) { - printf("Unsupported aead key length\n"); - return -1; - } - options->aead_xform.aead.key.length = - options->aead_key_random_size; - /* No size provided, use minimum size. */ - } else - options->aead_xform.aead.key.length = + /* Set key if not provided from command line */ + if (options->aead_key_param == 0) { + if (options->aead_key_random_size != -1) + options->aead_xform.aead.key.length = + options->aead_key_random_size; + /* No size provided, use minimum size. */ + else + options->aead_xform.aead.key.length = cap->sym.aead.key_size.min; - if (!options->aead_key_param) - generate_random_key( - options->aead_xform.aead.key.data, + generate_random_key(options->aead_key, options->aead_xform.aead.key.length); + } - /* - * Check if length of provided AAD is supported - * by the algorithm chosen. - */ - if (options->aad_param) { - if (check_supported_size(options->aad.length, - cap->sym.aead.aad_size.min, - cap->sym.aead.aad_size.max, - cap->sym.aead.aad_size.increment) - != 0) { - printf("Unsupported AAD length\n"); - return -1; - } - /* - * Check if length of AAD to be randomly generated - * is supported by the algorithm chosen. - */ - } else if (options->aad_random_size != -1) { - if (check_supported_size(options->aad_random_size, - cap->sym.aead.aad_size.min, - cap->sym.aead.aad_size.max, - cap->sym.aead.aad_size.increment) - != 0) { - printf("Unsupported AAD length\n"); - return -1; - } - options->aad.length = options->aad_random_size; - /* No size provided, use minimum size. */ - } else - options->aad.length = cap->sym.auth.aad_size.min; + /* Set AAD if not provided from command line */ + if (options->aad_param == 0) { + if (options->aad_random_size != -1) + options->aad.length = + options->aad_random_size; + /* No size provided, use minimum size. */ + else + options->aad.length = + cap->sym.auth.aad_size.min; + } options->aead_xform.aead.aad_length = options->aad.length; - /* Check if digest size is supported by the algorithm. */ - if (options->digest_size != -1) { - if (check_supported_size(options->digest_size, - cap->sym.aead.digest_size.min, - cap->sym.aead.digest_size.max, - cap->sym.aead.digest_size.increment) - != 0) { - printf("Unsupported digest length\n"); - return -1; - } + /* Set digest size if not provided from command line */ + if (options->digest_size != -1) options->aead_xform.aead.digest_length = options->digest_size; - /* No size provided, use minimum size. */ - } else + /* No size provided, use minimum size. */ + else options->aead_xform.aead.digest_length = cap->sym.aead.digest_size.min; } @@ -2182,127 +2493,74 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) { - /* Check if device supports cipher algo */ cap = check_device_support_cipher_algo(options, &dev_info, cdev_id); - if (cap == NULL) - continue; - options->block_size = cap->sym.cipher.block_size; - check_iv_param(&cap->sym.cipher.iv_size, - options->cipher_iv_param, - options->cipher_iv_random_size, - &options->cipher_iv.length); + /* Set IV if not provided from command line */ + if (options->cipher_iv_param == 0) { + if (options->cipher_iv_random_size != -1) + options->cipher_iv.length = + options->cipher_iv_random_size; + /* No size provided, use minimum size. */ + else + options->cipher_iv.length = + cap->sym.cipher.iv_size.min; + } - /* - * Check if length of provided cipher key is supported - * by the algorithm chosen. - */ - if (options->ckey_param) { - if (check_supported_size( - options->cipher_xform.cipher.key.length, - cap->sym.cipher.key_size.min, - cap->sym.cipher.key_size.max, - cap->sym.cipher.key_size.increment) - != 0) { - printf("Unsupported cipher key length\n"); - return -1; - } - /* - * Check if length of the cipher key to be randomly generated - * is supported by the algorithm chosen. - */ - } else if (options->ckey_random_size != -1) { - if (check_supported_size(options->ckey_random_size, - cap->sym.cipher.key_size.min, - cap->sym.cipher.key_size.max, - cap->sym.cipher.key_size.increment) - != 0) { - printf("Unsupported cipher key length\n"); - return -1; - } - options->cipher_xform.cipher.key.length = - options->ckey_random_size; - /* No size provided, use minimum size. */ - } else - options->cipher_xform.cipher.key.length = + /* Set key if not provided from command line */ + if (options->ckey_param == 0) { + if (options->ckey_random_size != -1) + options->cipher_xform.cipher.key.length = + options->ckey_random_size; + /* No size provided, use minimum size. */ + else + options->cipher_xform.cipher.key.length = cap->sym.cipher.key_size.min; - if (!options->ckey_param) - generate_random_key( - options->cipher_xform.cipher.key.data, + generate_random_key(options->cipher_key, options->cipher_xform.cipher.key.length); - + } } /* Set auth parameters */ if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH || options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER || options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) { - /* Check if device supports auth algo */ cap = check_device_support_auth_algo(options, &dev_info, cdev_id); - if (cap == NULL) - continue; - check_iv_param(&cap->sym.auth.iv_size, - options->auth_iv_param, - options->auth_iv_random_size, - &options->auth_iv.length); - /* - * Check if length of provided auth key is supported - * by the algorithm chosen. - */ - if (options->akey_param) { - if (check_supported_size( - options->auth_xform.auth.key.length, - cap->sym.auth.key_size.min, - cap->sym.auth.key_size.max, - cap->sym.auth.key_size.increment) - != 0) { - printf("Unsupported auth key length\n"); - return -1; - } - /* - * Check if length of the auth key to be randomly generated - * is supported by the algorithm chosen. - */ - } else if (options->akey_random_size != -1) { - if (check_supported_size(options->akey_random_size, - cap->sym.auth.key_size.min, - cap->sym.auth.key_size.max, - cap->sym.auth.key_size.increment) - != 0) { - printf("Unsupported auth key length\n"); - return -1; - } - options->auth_xform.auth.key.length = - options->akey_random_size; - /* No size provided, use minimum size. */ - } else - options->auth_xform.auth.key.length = + /* Set IV if not provided from command line */ + if (options->auth_iv_param == 0) { + if (options->auth_iv_random_size != -1) + options->auth_iv.length = + options->auth_iv_random_size; + /* No size provided, use minimum size. */ + else + options->auth_iv.length = + cap->sym.auth.iv_size.min; + } + + /* Set key if not provided from command line */ + if (options->akey_param == 0) { + if (options->akey_random_size != -1) + options->auth_xform.auth.key.length = + options->akey_random_size; + /* No size provided, use minimum size. */ + else + options->auth_xform.auth.key.length = cap->sym.auth.key_size.min; - if (!options->akey_param) - generate_random_key( - options->auth_xform.auth.key.data, + generate_random_key(options->auth_key, options->auth_xform.auth.key.length); + } - /* Check if digest size is supported by the algorithm. */ - if (options->digest_size != -1) { - if (check_supported_size(options->digest_size, - cap->sym.auth.digest_size.min, - cap->sym.auth.digest_size.max, - cap->sym.auth.digest_size.increment) - != 0) { - printf("Unsupported digest length\n"); - return -1; - } + /* Set digest size if not provided from command line */ + if (options->digest_size != -1) options->auth_xform.auth.digest_length = options->digest_size; - /* No size provided, use minimum size. */ - } else + /* No size provided, use minimum size. */ + else options->auth_xform.auth.digest_length = cap->sym.auth.digest_size.min; } @@ -2314,9 +2572,12 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, } qp_conf.nb_descriptors = 2048; + qp_conf.mp_session = session_pool_socket[socket_id].sess_mp; + qp_conf.mp_session_private = + session_pool_socket[socket_id].priv_mp; retval = rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf, - socket_id, session_pool_socket[socket_id]); + socket_id); if (retval < 0) { printf("Failed to setup queue pair %u on cryptodev %u", 0, cdev_id); @@ -2329,11 +2590,6 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cdev_id, retval); return -1; } - - l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id); - - enabled_cdevs[cdev_id] = 1; - enabled_cdev_count++; } return enabled_cdev_count; @@ -2342,9 +2598,9 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, static int initialize_ports(struct l2fwd_crypto_options *options) { - uint8_t last_portid, portid; + uint16_t last_portid = 0, portid; unsigned enabled_portcount = 0; - unsigned nb_ports = rte_eth_dev_count(); + unsigned nb_ports = rte_eth_dev_count_avail(); if (nb_ports == 0) { printf("No Ethernet ports - bye\n"); @@ -2355,20 +2611,35 @@ initialize_ports(struct l2fwd_crypto_options *options) for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) l2fwd_dst_ports[portid] = 0; - for (last_portid = 0, portid = 0; portid < nb_ports; portid++) { + RTE_ETH_FOREACH_DEV(portid) { int retval; + struct rte_eth_dev_info dev_info; + struct rte_eth_rxconf rxq_conf; + struct rte_eth_txconf txq_conf; + struct rte_eth_conf local_port_conf = port_conf; /* Skip ports that are not enabled */ if ((options->portmask & (1 << portid)) == 0) continue; /* init port */ - printf("Initializing port %u... ", (unsigned) portid); + printf("Initializing port %u... ", portid); fflush(stdout); - retval = rte_eth_dev_configure(portid, 1, 1, &port_conf); + + retval = rte_eth_dev_info_get(portid, &dev_info); + if (retval != 0) { + printf("Error during getting device (port %u) info: %s\n", + portid, strerror(-retval)); + return retval; + } + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) + local_port_conf.txmode.offloads |= + DEV_TX_OFFLOAD_MBUF_FAST_FREE; + retval = rte_eth_dev_configure(portid, 1, 1, &local_port_conf); if (retval < 0) { printf("Cannot configure device: err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); return -1; } @@ -2376,29 +2647,33 @@ initialize_ports(struct l2fwd_crypto_options *options) &nb_txd); if (retval < 0) { printf("Cannot adjust number of descriptors: err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); return -1; } /* init one RX queue */ fflush(stdout); + rxq_conf = dev_info.default_rxconf; + rxq_conf.offloads = local_port_conf.rxmode.offloads; retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd, rte_eth_dev_socket_id(portid), - NULL, l2fwd_pktmbuf_pool); + &rxq_conf, l2fwd_pktmbuf_pool); if (retval < 0) { printf("rte_eth_rx_queue_setup:err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); return -1; } /* init one TX queue on each port */ fflush(stdout); + txq_conf = dev_info.default_txconf; + txq_conf.offloads = local_port_conf.txmode.offloads; retval = rte_eth_tx_queue_setup(portid, 0, nb_txd, rte_eth_dev_socket_id(portid), - NULL); + &txq_conf); if (retval < 0) { printf("rte_eth_tx_queue_setup:err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); return -1; } @@ -2407,22 +2682,28 @@ initialize_ports(struct l2fwd_crypto_options *options) retval = rte_eth_dev_start(portid); if (retval < 0) { printf("rte_eth_dev_start:err=%d, port=%u\n", - retval, (unsigned) portid); + retval, portid); return -1; } - rte_eth_promiscuous_enable(portid); + retval = rte_eth_promiscuous_enable(portid); + if (retval != 0) { + printf("rte_eth_promiscuous_enable:err=%s, port=%u\n", + rte_strerror(-retval), portid); + return -1; + } - rte_eth_macaddr_get(portid, &l2fwd_ports_eth_addr[portid]); + retval = rte_eth_macaddr_get(portid, + &l2fwd_ports_eth_addr[portid]); + if (retval < 0) { + printf("rte_eth_macaddr_get :err=%d, port=%u\n", + retval, portid); + return -1; + } - printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n", - (unsigned) portid, - l2fwd_ports_eth_addr[portid].addr_bytes[0], - l2fwd_ports_eth_addr[portid].addr_bytes[1], - l2fwd_ports_eth_addr[portid].addr_bytes[2], - l2fwd_ports_eth_addr[portid].addr_bytes[3], - l2fwd_ports_eth_addr[portid].addr_bytes[4], - l2fwd_ports_eth_addr[portid].addr_bytes[5]); + printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n", + portid, + RTE_ETHER_ADDR_BYTES(&l2fwd_ports_eth_addr[portid])); /* initialize port stats */ memset(&port_statistics, 0, sizeof(port_statistics)); @@ -2446,7 +2727,7 @@ initialize_ports(struct l2fwd_crypto_options *options) return -1; } - check_all_ports_link_status(nb_ports, l2fwd_enabled_port_mask); + check_all_ports_link_status(l2fwd_enabled_port_mask); return enabled_portcount; } @@ -2454,20 +2735,11 @@ initialize_ports(struct l2fwd_crypto_options *options) static void reserve_key_memory(struct l2fwd_crypto_options *options) { - options->cipher_xform.cipher.key.data = rte_malloc("crypto key", - MAX_KEY_SIZE, 0); - if (options->cipher_xform.cipher.key.data == NULL) - rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher key"); + options->cipher_xform.cipher.key.data = options->cipher_key; - options->auth_xform.auth.key.data = rte_malloc("auth key", - MAX_KEY_SIZE, 0); - if (options->auth_xform.auth.key.data == NULL) - rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth key"); + options->auth_xform.auth.key.data = options->auth_key; - options->aead_xform.aead.key.data = rte_malloc("aead key", - MAX_KEY_SIZE, 0); - if (options->aead_xform.aead.key.data == NULL) - rte_exit(EXIT_FAILURE, "Failed to allocate memory for AEAD key"); + options->aead_xform.aead.key.data = options->aead_key; options->cipher_iv.data = rte_malloc("cipher iv", MAX_KEY_SIZE, 0); if (options->cipher_iv.data == NULL) @@ -2484,17 +2756,18 @@ reserve_key_memory(struct l2fwd_crypto_options *options) options->aad.data = rte_malloc("aad", MAX_KEY_SIZE, 0); if (options->aad.data == NULL) rte_exit(EXIT_FAILURE, "Failed to allocate memory for AAD"); - options->aad.phys_addr = rte_malloc_virt2phy(options->aad.data); + options->aad.phys_addr = rte_malloc_virt2iova(options->aad.data); } int main(int argc, char **argv) { - struct lcore_queue_conf *qconf; + struct lcore_queue_conf *qconf = NULL; struct l2fwd_crypto_options options; - uint8_t nb_ports, nb_cryptodevs, portid, cdev_id; - unsigned lcore_id, rx_lcore_id; + uint8_t nb_cryptodevs, cdev_id; + uint16_t portid; + unsigned lcore_id, rx_lcore_id = 0; int ret, enabled_cdevcount, enabled_portcount; uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = {0}; @@ -2518,7 +2791,8 @@ main(int argc, char **argv) /* create the mbuf pool */ l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 512, - sizeof(struct rte_crypto_op), + RTE_ALIGN(sizeof(struct rte_crypto_op), + RTE_CACHE_LINE_SIZE), RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); if (l2fwd_pktmbuf_pool == NULL) rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); @@ -2535,10 +2809,8 @@ main(int argc, char **argv) if (enabled_portcount < 1) rte_exit(EXIT_FAILURE, "Failed to initial Ethernet ports\n"); - nb_ports = rte_eth_dev_count(); /* Initialize the port/queue configuration of each logical core */ - for (rx_lcore_id = 0, qconf = NULL, portid = 0; - portid < nb_ports; portid++) { + RTE_ETH_FOREACH_DEV(portid) { /* skip ports that are not enabled */ if ((options.portmask & (1 << portid)) == 0) @@ -2570,7 +2842,7 @@ main(int argc, char **argv) qconf->rx_port_list[qconf->nb_rx_ports] = portid; qconf->nb_rx_ports++; - printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned)portid); + printf("Lcore %u: RX port %u\n", rx_lcore_id, portid); } /* Enable Crypto devices */ @@ -2628,11 +2900,14 @@ main(int argc, char **argv) /* launch per-lcore init on every lcore */ rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options, - CALL_MASTER); - RTE_LCORE_FOREACH_SLAVE(lcore_id) { + CALL_MAIN); + RTE_LCORE_FOREACH_WORKER(lcore_id) { if (rte_eal_wait_lcore(lcore_id) < 0) return -1; } + /* clean up the EAL */ + rte_eal_cleanup(); + return 0; }