4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
39 #include <rte_ethdev.h>
40 #include <rte_ether.h>
42 #include <rte_byteorder.h>
44 #include <rte_port_ring.h>
45 #include <rte_table_hash.h>
47 #include <rte_pipeline.h>
52 translate_options(uint32_t *special, uint32_t *ext, uint32_t *key_size)
54 switch (app.pipeline_type) {
55 case e_APP_PIPELINE_HASH_KEY8_EXT:
56 *special = 0; *ext = 1; *key_size = 8; return;
57 case e_APP_PIPELINE_HASH_KEY8_LRU:
58 *special = 0; *ext = 0; *key_size = 8; return;
59 case e_APP_PIPELINE_HASH_KEY16_EXT:
60 *special = 0; *ext = 1; *key_size = 16; return;
61 case e_APP_PIPELINE_HASH_KEY16_LRU:
62 *special = 0; *ext = 0; *key_size = 16; return;
63 case e_APP_PIPELINE_HASH_KEY32_EXT:
64 *special = 0; *ext = 1; *key_size = 32; return;
65 case e_APP_PIPELINE_HASH_KEY32_LRU:
66 *special = 0; *ext = 0; *key_size = 32; return;
68 case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT:
69 *special = 1; *ext = 1; *key_size = 8; return;
70 case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU:
71 *special = 1; *ext = 0; *key_size = 8; return;
72 case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT:
73 *special = 1; *ext = 1; *key_size = 16; return;
74 case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU:
75 *special = 1; *ext = 0; *key_size = 16; return;
76 case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT:
77 *special = 1; *ext = 1; *key_size = 32; return;
78 case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU:
79 *special = 1; *ext = 0; *key_size = 32; return;
81 case e_APP_PIPELINE_HASH_CUCKOO_KEY8:
82 *special = 0; *ext = 0; *key_size = 8; return;
83 case e_APP_PIPELINE_HASH_CUCKOO_KEY16:
84 *special = 0; *ext = 0; *key_size = 16; return;
85 case e_APP_PIPELINE_HASH_CUCKOO_KEY32:
86 *special = 0; *ext = 0; *key_size = 32; return;
87 case e_APP_PIPELINE_HASH_CUCKOO_KEY48:
88 *special = 0; *ext = 0; *key_size = 48; return;
89 case e_APP_PIPELINE_HASH_CUCKOO_KEY64:
90 *special = 0; *ext = 0; *key_size = 64; return;
91 case e_APP_PIPELINE_HASH_CUCKOO_KEY80:
92 *special = 0; *ext = 0; *key_size = 80; return;
93 case e_APP_PIPELINE_HASH_CUCKOO_KEY96:
94 *special = 0; *ext = 0; *key_size = 96; return;
95 case e_APP_PIPELINE_HASH_CUCKOO_KEY112:
96 *special = 0; *ext = 0; *key_size = 112; return;
97 case e_APP_PIPELINE_HASH_CUCKOO_KEY128:
98 *special = 0; *ext = 0; *key_size = 128; return;
101 rte_panic("Invalid hash table type or key size\n");
105 app_main_loop_worker_pipeline_hash(void) {
106 struct rte_pipeline_params pipeline_params = {
108 .socket_id = rte_socket_id(),
111 struct rte_pipeline *p;
112 uint32_t port_in_id[APP_MAX_PORTS];
113 uint32_t port_out_id[APP_MAX_PORTS];
116 uint32_t special, ext, key_size;
118 translate_options(&special, &ext, &key_size);
120 RTE_LOG(INFO, USER1, "Core %u is doing work "
121 "(pipeline with hash table, %s, %s, %d-byte key)\n",
123 special ? "specialized" : "non-specialized",
124 ext ? "extendible bucket" : "LRU",
127 /* Pipeline configuration */
128 p = rte_pipeline_create(&pipeline_params);
130 rte_panic("Unable to configure the pipeline\n");
132 /* Input port configuration */
133 for (i = 0; i < app.n_ports; i++) {
134 struct rte_port_ring_reader_params port_ring_params = {
135 .ring = app.rings_rx[i],
138 struct rte_pipeline_port_in_params port_params = {
139 .ops = &rte_port_ring_reader_ops,
140 .arg_create = (void *) &port_ring_params,
143 .burst_size = app.burst_size_worker_read,
146 if (rte_pipeline_port_in_create(p, &port_params,
148 rte_panic("Unable to configure input port for "
152 /* Output port configuration */
153 for (i = 0; i < app.n_ports; i++) {
154 struct rte_port_ring_writer_params port_ring_params = {
155 .ring = app.rings_tx[i],
156 .tx_burst_sz = app.burst_size_worker_write,
159 struct rte_pipeline_port_out_params port_params = {
160 .ops = &rte_port_ring_writer_ops,
161 .arg_create = (void *) &port_ring_params,
166 if (rte_pipeline_port_out_create(p, &port_params,
168 rte_panic("Unable to configure output port for "
172 /* Table configuration */
173 switch (app.pipeline_type) {
174 case e_APP_PIPELINE_HASH_KEY8_EXT:
175 case e_APP_PIPELINE_HASH_KEY16_EXT:
176 case e_APP_PIPELINE_HASH_KEY32_EXT:
178 struct rte_table_hash_ext_params table_hash_params = {
179 .key_size = key_size,
181 .n_buckets = 1 << 22,
182 .n_buckets_ext = 1 << 21,
185 .signature_offset = APP_METADATA_OFFSET(0),
186 .key_offset = APP_METADATA_OFFSET(32),
189 struct rte_pipeline_table_params table_params = {
190 .ops = &rte_table_hash_ext_ops,
191 .arg_create = &table_hash_params,
192 .f_action_hit = NULL,
193 .f_action_miss = NULL,
195 .action_data_size = 0,
198 if (rte_pipeline_table_create(p, &table_params, &table_id))
199 rte_panic("Unable to configure the hash table\n");
203 case e_APP_PIPELINE_HASH_KEY8_LRU:
204 case e_APP_PIPELINE_HASH_KEY16_LRU:
205 case e_APP_PIPELINE_HASH_KEY32_LRU:
207 struct rte_table_hash_lru_params table_hash_params = {
208 .key_size = key_size,
210 .n_buckets = 1 << 22,
213 .signature_offset = APP_METADATA_OFFSET(0),
214 .key_offset = APP_METADATA_OFFSET(32),
217 struct rte_pipeline_table_params table_params = {
218 .ops = &rte_table_hash_lru_ops,
219 .arg_create = &table_hash_params,
220 .f_action_hit = NULL,
221 .f_action_miss = NULL,
223 .action_data_size = 0,
226 if (rte_pipeline_table_create(p, &table_params, &table_id))
227 rte_panic("Unable to configure the hash table\n");
231 case e_APP_PIPELINE_HASH_SPEC_KEY8_EXT:
233 struct rte_table_hash_key8_ext_params table_hash_params = {
234 .n_entries = 1 << 24,
235 .n_entries_ext = 1 << 23,
236 .signature_offset = APP_METADATA_OFFSET(0),
237 .key_offset = APP_METADATA_OFFSET(32),
243 struct rte_pipeline_table_params table_params = {
244 .ops = &rte_table_hash_key8_ext_ops,
245 .arg_create = &table_hash_params,
246 .f_action_hit = NULL,
247 .f_action_miss = NULL,
249 .action_data_size = 0,
252 if (rte_pipeline_table_create(p, &table_params, &table_id))
253 rte_panic("Unable to configure the hash table\n");
257 case e_APP_PIPELINE_HASH_SPEC_KEY8_LRU:
259 struct rte_table_hash_key8_lru_params table_hash_params = {
260 .n_entries = 1 << 24,
261 .signature_offset = APP_METADATA_OFFSET(0),
262 .key_offset = APP_METADATA_OFFSET(32),
268 struct rte_pipeline_table_params table_params = {
269 .ops = &rte_table_hash_key8_lru_ops,
270 .arg_create = &table_hash_params,
271 .f_action_hit = NULL,
272 .f_action_miss = NULL,
274 .action_data_size = 0,
277 if (rte_pipeline_table_create(p, &table_params, &table_id))
278 rte_panic("Unable to configure the hash table\n");
282 case e_APP_PIPELINE_HASH_SPEC_KEY16_EXT:
284 struct rte_table_hash_key16_ext_params table_hash_params = {
285 .n_entries = 1 << 24,
286 .n_entries_ext = 1 << 23,
287 .signature_offset = APP_METADATA_OFFSET(0),
288 .key_offset = APP_METADATA_OFFSET(32),
294 struct rte_pipeline_table_params table_params = {
295 .ops = &rte_table_hash_key16_ext_ops,
296 .arg_create = &table_hash_params,
297 .f_action_hit = NULL,
298 .f_action_miss = NULL,
300 .action_data_size = 0,
303 if (rte_pipeline_table_create(p, &table_params, &table_id))
304 rte_panic("Unable to configure the hash table)\n");
308 case e_APP_PIPELINE_HASH_SPEC_KEY16_LRU:
310 struct rte_table_hash_key16_lru_params table_hash_params = {
311 .n_entries = 1 << 24,
312 .signature_offset = APP_METADATA_OFFSET(0),
313 .key_offset = APP_METADATA_OFFSET(32),
319 struct rte_pipeline_table_params table_params = {
320 .ops = &rte_table_hash_key16_lru_ops,
321 .arg_create = &table_hash_params,
322 .f_action_hit = NULL,
323 .f_action_miss = NULL,
325 .action_data_size = 0,
328 if (rte_pipeline_table_create(p, &table_params, &table_id))
329 rte_panic("Unable to configure the hash table\n");
333 case e_APP_PIPELINE_HASH_SPEC_KEY32_EXT:
335 struct rte_table_hash_key32_ext_params table_hash_params = {
336 .n_entries = 1 << 24,
337 .n_entries_ext = 1 << 23,
338 .signature_offset = APP_METADATA_OFFSET(0),
339 .key_offset = APP_METADATA_OFFSET(32),
344 struct rte_pipeline_table_params table_params = {
345 .ops = &rte_table_hash_key32_ext_ops,
346 .arg_create = &table_hash_params,
347 .f_action_hit = NULL,
348 .f_action_miss = NULL,
350 .action_data_size = 0,
353 if (rte_pipeline_table_create(p, &table_params, &table_id))
354 rte_panic("Unable to configure the hash table\n");
359 case e_APP_PIPELINE_HASH_SPEC_KEY32_LRU:
361 struct rte_table_hash_key32_lru_params table_hash_params = {
362 .n_entries = 1 << 24,
363 .signature_offset = APP_METADATA_OFFSET(0),
364 .key_offset = APP_METADATA_OFFSET(32),
369 struct rte_pipeline_table_params table_params = {
370 .ops = &rte_table_hash_key32_lru_ops,
371 .arg_create = &table_hash_params,
372 .f_action_hit = NULL,
373 .f_action_miss = NULL,
375 .action_data_size = 0,
378 if (rte_pipeline_table_create(p, &table_params, &table_id))
379 rte_panic("Unable to configure the hash table\n");
383 case e_APP_PIPELINE_HASH_CUCKOO_KEY8:
384 case e_APP_PIPELINE_HASH_CUCKOO_KEY16:
385 case e_APP_PIPELINE_HASH_CUCKOO_KEY32:
386 case e_APP_PIPELINE_HASH_CUCKOO_KEY48:
387 case e_APP_PIPELINE_HASH_CUCKOO_KEY64:
388 case e_APP_PIPELINE_HASH_CUCKOO_KEY80:
389 case e_APP_PIPELINE_HASH_CUCKOO_KEY96:
390 case e_APP_PIPELINE_HASH_CUCKOO_KEY112:
391 case e_APP_PIPELINE_HASH_CUCKOO_KEY128:
393 char hash_name[RTE_HASH_NAMESIZE];
395 snprintf(hash_name, sizeof(hash_name), "RTE_TH_CUCKOO_%d",
398 struct rte_table_hash_cuckoo_params table_hash_params = {
399 .key_size = key_size,
400 .n_keys = (1 << 24) + 1,
403 .signature_offset = APP_METADATA_OFFSET(0),
404 .key_offset = APP_METADATA_OFFSET(32),
408 struct rte_pipeline_table_params table_params = {
409 .ops = &rte_table_hash_cuckoo_dosig_ops,
410 .arg_create = &table_hash_params,
411 .f_action_hit = NULL,
412 .f_action_miss = NULL,
414 .action_data_size = 0,
417 if (rte_pipeline_table_create(p, &table_params, &table_id))
418 rte_panic("Unable to configure the hash table\n");
423 rte_panic("Invalid hash table type or key size\n");
426 /* Interconnecting ports and tables */
427 for (i = 0; i < app.n_ports; i++)
428 if (rte_pipeline_port_in_connect_to_table(p, port_in_id[i],
430 rte_panic("Unable to connect input port %u to "
431 "table %u\n", port_in_id[i], table_id);
433 /* Add entries to tables */
434 for (i = 0; i < (1 << 24); i++) {
435 struct rte_pipeline_table_entry entry = {
436 .action = RTE_PIPELINE_ACTION_PORT,
437 {.port_id = port_out_id[i & (app.n_ports - 1)]},
439 struct rte_pipeline_table_entry *entry_ptr;
441 uint32_t *k32 = (uint32_t *) key;
442 int key_found, status;
444 memset(key, 0, sizeof(key));
445 k32[0] = rte_be_to_cpu_32(i);
447 status = rte_pipeline_table_entry_add(p, table_id, key, &entry,
448 &key_found, &entry_ptr);
450 rte_panic("Unable to add entry to table %u (%d)\n",
454 /* Enable input ports */
455 for (i = 0; i < app.n_ports; i++)
456 if (rte_pipeline_port_in_enable(p, port_in_id[i]))
457 rte_panic("Unable to enable input port %u\n",
460 /* Check pipeline consistency */
461 if (rte_pipeline_check(p) < 0)
462 rte_panic("Pipeline consistency check failed\n");
472 if ((i & APP_FLUSH) == 0)
473 rte_pipeline_flush(p);
480 __attribute__((unused)) uint32_t key_size,
481 __attribute__((unused)) uint64_t seed)
483 uint32_t *k32 = (uint32_t *) key;
484 uint32_t ip_dst = rte_be_to_cpu_32(k32[0]);
485 uint64_t signature = (ip_dst >> 2) | ((ip_dst & 0x3) << 30);
491 app_main_loop_rx_metadata(void) {
495 RTE_LOG(INFO, USER1, "Core %u is doing RX (with meta-data)\n",
498 for (i = 0; ; i = ((i + 1) & (app.n_ports - 1))) {
501 n_mbufs = rte_eth_rx_burst(
505 app.burst_size_rx_read);
510 for (j = 0; j < n_mbufs; j++) {
512 uint8_t *m_data, *key;
513 struct ipv4_hdr *ip_hdr;
514 struct ipv6_hdr *ipv6_hdr;
517 uint32_t *signature, *k32;
519 m = app.mbuf_rx.array[j];
520 m_data = rte_pktmbuf_mtod(m, uint8_t *);
521 signature = RTE_MBUF_METADATA_UINT32_PTR(m,
522 APP_METADATA_OFFSET(0));
523 key = RTE_MBUF_METADATA_UINT8_PTR(m,
524 APP_METADATA_OFFSET(32));
526 if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) {
527 ip_hdr = (struct ipv4_hdr *)
528 &m_data[sizeof(struct ether_hdr)];
529 ip_dst = ip_hdr->dst_addr;
531 k32 = (uint32_t *) key;
532 k32[0] = ip_dst & 0xFFFFFF00;
533 } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) {
534 ipv6_hdr = (struct ipv6_hdr *)
535 &m_data[sizeof(struct ether_hdr)];
536 ipv6_dst = ipv6_hdr->dst_addr;
538 memcpy(key, ipv6_dst, 16);
542 *signature = test_hash(key, 0, 0);
546 ret = rte_ring_sp_enqueue_bulk(
548 (void **) app.mbuf_rx.array,