4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include <rte_common.h>
37 #include <rte_malloc.h>
38 #include <rte_byteorder.h>
39 #include <rte_table_stub.h>
40 #include <rte_table_hash.h>
41 #include <rte_pipeline.h>
43 #include "pipeline_passthrough_be.h"
44 #include "pipeline_actions_common.h"
46 #include "hash_func.h"
48 struct pipeline_passthrough {
50 struct pipeline_passthrough_params params;
51 rte_table_hash_op_hash f_hash;
52 } __rte_cache_aligned;
54 static pipeline_msg_req_handler handlers[] = {
55 [PIPELINE_MSG_REQ_PING] =
56 pipeline_msg_req_ping_handler,
57 [PIPELINE_MSG_REQ_STATS_PORT_IN] =
58 pipeline_msg_req_stats_port_in_handler,
59 [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
60 pipeline_msg_req_stats_port_out_handler,
61 [PIPELINE_MSG_REQ_STATS_TABLE] =
62 pipeline_msg_req_stats_table_handler,
63 [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
64 pipeline_msg_req_port_in_enable_handler,
65 [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
66 pipeline_msg_req_port_in_disable_handler,
67 [PIPELINE_MSG_REQ_CUSTOM] =
68 pipeline_msg_req_invalid_handler,
71 static inline __attribute__((always_inline)) void
76 uint32_t hash_enabled)
78 struct pipeline_passthrough *p = arg;
80 uint64_t *dma_dst = RTE_MBUF_METADATA_UINT64_PTR(pkt,
81 p->params.dma_dst_offset);
82 uint64_t *dma_src = RTE_MBUF_METADATA_UINT64_PTR(pkt,
83 p->params.dma_src_offset);
84 uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
85 uint32_t *dma_hash = RTE_MBUF_METADATA_UINT32_PTR(pkt,
86 p->params.dma_hash_offset);
89 /* Read (dma_src), compute (dma_dst), write (dma_dst) */
90 for (i = 0; i < (dma_size / 8); i++)
91 dma_dst[i] = dma_src[i] & dma_mask[i];
93 /* Read (dma_dst), compute (hash), write (hash) */
95 *dma_hash = p->f_hash(dma_dst, dma_size, 0);
98 static inline __attribute__((always_inline)) void
100 struct rte_mbuf **pkts,
103 uint32_t hash_enabled)
105 struct pipeline_passthrough *p = arg;
107 uint64_t *dma_dst0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
108 p->params.dma_dst_offset);
109 uint64_t *dma_dst1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
110 p->params.dma_dst_offset);
111 uint64_t *dma_dst2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
112 p->params.dma_dst_offset);
113 uint64_t *dma_dst3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
114 p->params.dma_dst_offset);
116 uint64_t *dma_src0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
117 p->params.dma_src_offset);
118 uint64_t *dma_src1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
119 p->params.dma_src_offset);
120 uint64_t *dma_src2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
121 p->params.dma_src_offset);
122 uint64_t *dma_src3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
123 p->params.dma_src_offset);
125 uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
127 uint32_t *dma_hash0 = RTE_MBUF_METADATA_UINT32_PTR(pkts[0],
128 p->params.dma_hash_offset);
129 uint32_t *dma_hash1 = RTE_MBUF_METADATA_UINT32_PTR(pkts[1],
130 p->params.dma_hash_offset);
131 uint32_t *dma_hash2 = RTE_MBUF_METADATA_UINT32_PTR(pkts[2],
132 p->params.dma_hash_offset);
133 uint32_t *dma_hash3 = RTE_MBUF_METADATA_UINT32_PTR(pkts[3],
134 p->params.dma_hash_offset);
138 /* Read (dma_src), compute (dma_dst), write (dma_dst) */
139 for (i = 0; i < (dma_size / 8); i++) {
140 dma_dst0[i] = dma_src0[i] & dma_mask[i];
141 dma_dst1[i] = dma_src1[i] & dma_mask[i];
142 dma_dst2[i] = dma_src2[i] & dma_mask[i];
143 dma_dst3[i] = dma_src3[i] & dma_mask[i];
146 /* Read (dma_dst), compute (hash), write (hash) */
148 *dma_hash0 = p->f_hash(dma_dst0, dma_size, 0);
149 *dma_hash1 = p->f_hash(dma_dst1, dma_size, 0);
150 *dma_hash2 = p->f_hash(dma_dst2, dma_size, 0);
151 *dma_hash3 = p->f_hash(dma_dst3, dma_size, 0);
155 #define PKT_WORK(dma_size, hash_enabled) \
157 pkt_work_size##dma_size##_hash##hash_enabled( \
158 struct rte_mbuf *pkt, \
161 pkt_work(pkt, arg, dma_size, hash_enabled); \
164 #define PKT4_WORK(dma_size, hash_enabled) \
166 pkt4_work_size##dma_size##_hash##hash_enabled( \
167 struct rte_mbuf **pkts, \
170 pkt4_work(pkts, arg, dma_size, hash_enabled); \
173 #define port_in_ah(dma_size, hash_enabled) \
174 PKT_WORK(dma_size, hash_enabled) \
175 PKT4_WORK(dma_size, hash_enabled) \
176 PIPELINE_PORT_IN_AH(port_in_ah_size##dma_size##_hash##hash_enabled,\
177 pkt_work_size##dma_size##_hash##hash_enabled, \
178 pkt4_work_size##dma_size##_hash##hash_enabled)
198 static rte_pipeline_port_in_action_handler
199 get_port_in_ah(struct pipeline_passthrough *p)
201 if (p->params.dma_enabled == 0)
204 if (p->params.dma_hash_enabled)
205 switch (p->params.dma_size) {
207 case 8: return port_in_ah_size8_hash1;
208 case 16: return port_in_ah_size16_hash1;
209 case 24: return port_in_ah_size24_hash1;
210 case 32: return port_in_ah_size32_hash1;
211 case 40: return port_in_ah_size40_hash1;
212 case 48: return port_in_ah_size48_hash1;
213 case 56: return port_in_ah_size56_hash1;
214 case 64: return port_in_ah_size64_hash1;
215 default: return NULL;
218 switch (p->params.dma_size) {
220 case 8: return port_in_ah_size8_hash0;
221 case 16: return port_in_ah_size16_hash0;
222 case 24: return port_in_ah_size24_hash0;
223 case 32: return port_in_ah_size32_hash0;
224 case 40: return port_in_ah_size40_hash0;
225 case 48: return port_in_ah_size48_hash0;
226 case 56: return port_in_ah_size56_hash0;
227 case 64: return port_in_ah_size64_hash0;
228 default: return NULL;
233 pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p,
234 struct pipeline_params *params)
236 uint32_t dma_dst_offset_present = 0;
237 uint32_t dma_src_offset_present = 0;
238 uint32_t dma_src_mask_present = 0;
239 uint32_t dma_size_present = 0;
240 uint32_t dma_hash_offset_present = 0;
242 char dma_mask_str[PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2];
246 p->dma_hash_enabled = 0;
247 memset(p->dma_src_mask, 0xFF, sizeof(p->dma_src_mask));
249 for (i = 0; i < params->n_args; i++) {
250 char *arg_name = params->args_name[i];
251 char *arg_value = params->args_value[i];
254 if (strcmp(arg_name, "dma_dst_offset") == 0) {
257 PIPELINE_PARSE_ERR_DUPLICATE(
258 dma_dst_offset_present == 0, params->name,
260 dma_dst_offset_present = 1;
262 status = parser_read_uint32(&p->dma_dst_offset,
264 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
265 params->name, arg_name, arg_value);
266 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
267 params->name, arg_name, arg_value);
275 if (strcmp(arg_name, "dma_src_offset") == 0) {
278 PIPELINE_PARSE_ERR_DUPLICATE(
279 dma_src_offset_present == 0, params->name,
281 dma_src_offset_present = 1;
283 status = parser_read_uint32(&p->dma_src_offset,
285 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
286 params->name, arg_name, arg_value);
287 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
288 params->name, arg_name, arg_value);
296 if (strcmp(arg_name, "dma_size") == 0) {
299 PIPELINE_PARSE_ERR_DUPLICATE(
300 dma_size_present == 0, params->name,
302 dma_size_present = 1;
304 status = parser_read_uint32(&p->dma_size,
306 PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
307 (p->dma_size != 0) &&
308 ((p->dma_size % 8) == 0)),
309 params->name, arg_name, arg_value);
310 PIPELINE_PARSE_ERR_OUT_RNG(((status != -ERANGE) &&
312 PIPELINE_PASSTHROUGH_DMA_SIZE_MAX)),
313 params->name, arg_name, arg_value);
321 if (strcmp(arg_name, "dma_src_mask") == 0) {
322 int mask_str_len = strlen(arg_value);
324 PIPELINE_PARSE_ERR_DUPLICATE(
325 dma_src_mask_present == 0,
326 params->name, arg_name);
327 dma_src_mask_present = 1;
329 PIPELINE_ARG_CHECK((mask_str_len <
330 (PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2)),
331 "Parse error in section \"%s\": entry "
332 "\"%s\" too long", params->name,
335 snprintf(dma_mask_str, mask_str_len + 1,
343 /* dma_hash_offset */
344 if (strcmp(arg_name, "dma_hash_offset") == 0) {
347 PIPELINE_PARSE_ERR_DUPLICATE(
348 dma_hash_offset_present == 0,
349 params->name, arg_name);
350 dma_hash_offset_present = 1;
352 status = parser_read_uint32(&p->dma_hash_offset,
354 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
355 params->name, arg_name, arg_value);
356 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
357 params->name, arg_name, arg_value);
359 p->dma_hash_enabled = 1;
366 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
369 /* Check correlations between arguments */
370 PIPELINE_ARG_CHECK((dma_dst_offset_present == p->dma_enabled),
371 "Parse error in section \"%s\": missing entry "
372 "\"dma_dst_offset\"", params->name);
373 PIPELINE_ARG_CHECK((dma_src_offset_present == p->dma_enabled),
374 "Parse error in section \"%s\": missing entry "
375 "\"dma_src_offset\"", params->name);
376 PIPELINE_ARG_CHECK((dma_size_present == p->dma_enabled),
377 "Parse error in section \"%s\": missing entry "
378 "\"dma_size\"", params->name);
379 PIPELINE_ARG_CHECK((dma_hash_offset_present == p->dma_enabled),
380 "Parse error in section \"%s\": missing entry "
381 "\"dma_hash_offset\"", params->name);
383 if (dma_src_mask_present) {
384 uint32_t dma_size = p->dma_size;
387 PIPELINE_ARG_CHECK((strlen(dma_mask_str) ==
388 (dma_size * 2)), "Parse error in section "
389 "\"%s\": dma_src_mask should have exactly %u hex "
390 "digits", params->name, (dma_size * 2));
392 status = parse_hex_string(dma_mask_str, p->dma_src_mask,
395 PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
396 (dma_size == p->dma_size)), params->name,
397 "dma_src_mask", dma_mask_str);
404 static rte_table_hash_op_hash
405 get_hash_function(struct pipeline_passthrough *p)
407 switch (p->params.dma_size) {
409 case 8: return hash_default_key8;
410 case 16: return hash_default_key16;
411 case 24: return hash_default_key24;
412 case 32: return hash_default_key32;
413 case 40: return hash_default_key40;
414 case 48: return hash_default_key48;
415 case 56: return hash_default_key56;
416 case 64: return hash_default_key64;
417 default: return NULL;
422 pipeline_passthrough_init(struct pipeline_params *params,
423 __rte_unused void *arg)
426 struct pipeline_passthrough *p_pt;
429 /* Check input arguments */
430 if ((params == NULL) ||
431 (params->n_ports_in == 0) ||
432 (params->n_ports_out == 0) ||
433 (params->n_ports_in < params->n_ports_out) ||
434 (params->n_ports_in % params->n_ports_out))
437 /* Memory allocation */
438 size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_passthrough));
439 p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
440 p_pt = (struct pipeline_passthrough *) p;
444 strcpy(p->name, params->name);
445 p->log_level = params->log_level;
447 PLOG(p, HIGH, "Pass-through");
449 /* Parse arguments */
450 if (pipeline_passthrough_parse_args(&p_pt->params, params))
452 p_pt->f_hash = get_hash_function(p_pt);
456 struct rte_pipeline_params pipeline_params = {
457 .name = "PASS-THROUGH",
458 .socket_id = params->socket_id,
462 p->p = rte_pipeline_create(&pipeline_params);
470 p->n_ports_in = params->n_ports_in;
471 for (i = 0; i < p->n_ports_in; i++) {
472 struct rte_pipeline_port_in_params port_params = {
473 .ops = pipeline_port_in_params_get_ops(
474 ¶ms->port_in[i]),
475 .arg_create = pipeline_port_in_params_convert(
476 ¶ms->port_in[i]),
477 .f_action = get_port_in_ah(p_pt),
479 .burst_size = params->port_in[i].burst_size,
482 int status = rte_pipeline_port_in_create(p->p,
487 rte_pipeline_free(p->p);
494 p->n_ports_out = params->n_ports_out;
495 for (i = 0; i < p->n_ports_out; i++) {
496 struct rte_pipeline_port_out_params port_params = {
497 .ops = pipeline_port_out_params_get_ops(
498 ¶ms->port_out[i]),
499 .arg_create = pipeline_port_out_params_convert(
500 ¶ms->port_out[i]),
505 int status = rte_pipeline_port_out_create(p->p,
510 rte_pipeline_free(p->p);
517 p->n_tables = p->n_ports_in;
518 for (i = 0; i < p->n_ports_in; i++) {
519 struct rte_pipeline_table_params table_params = {
520 .ops = &rte_table_stub_ops,
522 .f_action_hit = NULL,
523 .f_action_miss = NULL,
525 .action_data_size = 0,
528 int status = rte_pipeline_table_create(p->p,
533 rte_pipeline_free(p->p);
539 /* Connecting input ports to tables */
540 for (i = 0; i < p->n_ports_in; i++) {
541 int status = rte_pipeline_port_in_connect_to_table(p->p,
546 rte_pipeline_free(p->p);
552 /* Add entries to tables */
553 for (i = 0; i < p->n_ports_in; i++) {
554 struct rte_pipeline_table_entry default_entry = {
555 .action = RTE_PIPELINE_ACTION_PORT,
556 {.port_id = p->port_out_id[
557 i / (p->n_ports_in / p->n_ports_out)]},
560 struct rte_pipeline_table_entry *default_entry_ptr;
562 int status = rte_pipeline_table_default_entry_add(p->p,
568 rte_pipeline_free(p->p);
574 /* Enable input ports */
575 for (i = 0; i < p->n_ports_in; i++) {
576 int status = rte_pipeline_port_in_enable(p->p,
580 rte_pipeline_free(p->p);
586 /* Check pipeline consistency */
587 if (rte_pipeline_check(p->p) < 0) {
588 rte_pipeline_free(p->p);
594 p->n_msgq = params->n_msgq;
595 for (i = 0; i < p->n_msgq; i++)
596 p->msgq_in[i] = params->msgq_in[i];
597 for (i = 0; i < p->n_msgq; i++)
598 p->msgq_out[i] = params->msgq_out[i];
600 /* Message handlers */
601 memcpy(p->handlers, handlers, sizeof(p->handlers));
607 pipeline_passthrough_free(void *pipeline)
609 struct pipeline *p = (struct pipeline *) pipeline;
611 /* Check input arguments */
616 rte_pipeline_free(p->p);
622 pipeline_passthrough_timer(void *pipeline)
624 struct pipeline *p = (struct pipeline *) pipeline;
626 pipeline_msg_req_handle(p);
627 rte_pipeline_flush(p->p);
633 pipeline_passthrough_track(void *pipeline, uint32_t port_in, uint32_t *port_out)
635 struct pipeline *p = (struct pipeline *) pipeline;
637 /* Check input arguments */
639 (port_in >= p->n_ports_in) ||
643 *port_out = port_in / p->n_ports_in;
647 struct pipeline_be_ops pipeline_passthrough_be_ops = {
648 .f_init = pipeline_passthrough_init,
649 .f_free = pipeline_passthrough_free,
651 .f_timer = pipeline_passthrough_timer,
652 .f_track = pipeline_passthrough_track,