1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2016 Intel Corporation
8 #include <rte_common.h>
9 #include <rte_malloc.h>
10 #include <rte_byteorder.h>
11 #include <rte_string_fns.h>
12 #include <rte_table_stub.h>
13 #include <rte_table_hash.h>
14 #include <rte_pipeline.h>
16 #include "pipeline_passthrough_be.h"
17 #include "pipeline_actions_common.h"
19 #include "hash_func.h"
21 #define SWAP_DIM (PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX * \
22 (PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX / sizeof(uint64_t)))
24 struct pipeline_passthrough {
26 struct pipeline_passthrough_params params;
27 rte_table_hash_op_hash f_hash;
28 uint32_t swap_field0_offset[SWAP_DIM];
29 uint32_t swap_field1_offset[SWAP_DIM];
30 uint64_t swap_field_mask[SWAP_DIM];
31 uint32_t swap_n_fields;
32 } __rte_cache_aligned;
34 static pipeline_msg_req_handler handlers[] = {
35 [PIPELINE_MSG_REQ_PING] =
36 pipeline_msg_req_ping_handler,
37 [PIPELINE_MSG_REQ_STATS_PORT_IN] =
38 pipeline_msg_req_stats_port_in_handler,
39 [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
40 pipeline_msg_req_stats_port_out_handler,
41 [PIPELINE_MSG_REQ_STATS_TABLE] =
42 pipeline_msg_req_stats_table_handler,
43 [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
44 pipeline_msg_req_port_in_enable_handler,
45 [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
46 pipeline_msg_req_port_in_disable_handler,
47 [PIPELINE_MSG_REQ_CUSTOM] =
48 pipeline_msg_req_invalid_handler,
51 static __rte_always_inline void
56 uint32_t hash_enabled,
58 uint32_t port_out_pow2)
60 struct pipeline_passthrough *p = arg;
62 uint64_t *dma_dst = RTE_MBUF_METADATA_UINT64_PTR(pkt,
63 p->params.dma_dst_offset);
64 uint64_t *dma_src = RTE_MBUF_METADATA_UINT64_PTR(pkt,
65 p->params.dma_src_offset);
66 uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
67 uint32_t *dma_hash = RTE_MBUF_METADATA_UINT32_PTR(pkt,
68 p->params.dma_hash_offset);
71 /* Read (dma_src), compute (dma_dst), write (dma_dst) */
72 for (i = 0; i < (dma_size / 8); i++)
73 dma_dst[i] = dma_src[i] & dma_mask[i];
75 /* Read (dma_dst), compute (hash), write (hash) */
77 uint32_t hash = p->f_hash(dma_src, dma_mask, dma_size, 0);
85 = hash & (p->p.n_ports_out - 1);
88 = hash % p->p.n_ports_out;
90 rte_pipeline_port_out_packet_insert(p->p.p,
96 static __rte_always_inline void
98 struct rte_mbuf **pkts,
101 uint32_t hash_enabled,
103 uint32_t port_out_pow2)
105 struct pipeline_passthrough *p = arg;
107 uint64_t *dma_dst0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
108 p->params.dma_dst_offset);
109 uint64_t *dma_dst1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
110 p->params.dma_dst_offset);
111 uint64_t *dma_dst2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
112 p->params.dma_dst_offset);
113 uint64_t *dma_dst3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
114 p->params.dma_dst_offset);
116 uint64_t *dma_src0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
117 p->params.dma_src_offset);
118 uint64_t *dma_src1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
119 p->params.dma_src_offset);
120 uint64_t *dma_src2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
121 p->params.dma_src_offset);
122 uint64_t *dma_src3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
123 p->params.dma_src_offset);
125 uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
127 uint32_t *dma_hash0 = RTE_MBUF_METADATA_UINT32_PTR(pkts[0],
128 p->params.dma_hash_offset);
129 uint32_t *dma_hash1 = RTE_MBUF_METADATA_UINT32_PTR(pkts[1],
130 p->params.dma_hash_offset);
131 uint32_t *dma_hash2 = RTE_MBUF_METADATA_UINT32_PTR(pkts[2],
132 p->params.dma_hash_offset);
133 uint32_t *dma_hash3 = RTE_MBUF_METADATA_UINT32_PTR(pkts[3],
134 p->params.dma_hash_offset);
138 /* Read (dma_src), compute (dma_dst), write (dma_dst) */
139 for (i = 0; i < (dma_size / 8); i++) {
140 dma_dst0[i] = dma_src0[i] & dma_mask[i];
141 dma_dst1[i] = dma_src1[i] & dma_mask[i];
142 dma_dst2[i] = dma_src2[i] & dma_mask[i];
143 dma_dst3[i] = dma_src3[i] & dma_mask[i];
146 /* Read (dma_dst), compute (hash), write (hash) */
148 uint32_t hash0 = p->f_hash(dma_src0, dma_mask, dma_size, 0);
149 uint32_t hash1 = p->f_hash(dma_src1, dma_mask, dma_size, 0);
150 uint32_t hash2 = p->f_hash(dma_src2, dma_mask, dma_size, 0);
151 uint32_t hash3 = p->f_hash(dma_src3, dma_mask, dma_size, 0);
159 uint32_t port_out0, port_out1, port_out2, port_out3;
163 = hash0 & (p->p.n_ports_out - 1);
165 = hash1 & (p->p.n_ports_out - 1);
167 = hash2 & (p->p.n_ports_out - 1);
169 = hash3 & (p->p.n_ports_out - 1);
172 = hash0 % p->p.n_ports_out;
174 = hash1 % p->p.n_ports_out;
176 = hash2 % p->p.n_ports_out;
178 = hash3 % p->p.n_ports_out;
180 rte_pipeline_port_out_packet_insert(p->p.p,
182 rte_pipeline_port_out_packet_insert(p->p.p,
184 rte_pipeline_port_out_packet_insert(p->p.p,
186 rte_pipeline_port_out_packet_insert(p->p.p,
192 static __rte_always_inline void
194 struct rte_mbuf *pkt,
197 struct pipeline_passthrough *p = arg;
200 /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
201 for (i = 0; i < p->swap_n_fields; i++) {
202 uint64_t *field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
203 p->swap_field0_offset[i]);
204 uint64_t *field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
205 p->swap_field1_offset[i]);
206 uint64_t mask = p->swap_field_mask[i];
208 uint64_t field0 = *field0_ptr;
209 uint64_t field1 = *field1_ptr;
211 *field0_ptr = (field0 & (~mask)) + (field1 & mask);
212 *field1_ptr = (field0 & mask) + (field1 & (~mask));
216 static __rte_always_inline void
218 struct rte_mbuf **pkts,
221 struct pipeline_passthrough *p = arg;
224 /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
225 for (i = 0; i < p->swap_n_fields; i++) {
226 uint64_t *pkt0_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
227 p->swap_field0_offset[i]);
228 uint64_t *pkt1_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
229 p->swap_field0_offset[i]);
230 uint64_t *pkt2_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
231 p->swap_field0_offset[i]);
232 uint64_t *pkt3_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
233 p->swap_field0_offset[i]);
235 uint64_t *pkt0_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
236 p->swap_field1_offset[i]);
237 uint64_t *pkt1_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
238 p->swap_field1_offset[i]);
239 uint64_t *pkt2_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
240 p->swap_field1_offset[i]);
241 uint64_t *pkt3_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
242 p->swap_field1_offset[i]);
244 uint64_t mask = p->swap_field_mask[i];
246 uint64_t pkt0_field0 = *pkt0_field0_ptr;
247 uint64_t pkt1_field0 = *pkt1_field0_ptr;
248 uint64_t pkt2_field0 = *pkt2_field0_ptr;
249 uint64_t pkt3_field0 = *pkt3_field0_ptr;
251 uint64_t pkt0_field1 = *pkt0_field1_ptr;
252 uint64_t pkt1_field1 = *pkt1_field1_ptr;
253 uint64_t pkt2_field1 = *pkt2_field1_ptr;
254 uint64_t pkt3_field1 = *pkt3_field1_ptr;
256 *pkt0_field0_ptr = (pkt0_field0 & (~mask)) + (pkt0_field1 & mask);
257 *pkt1_field0_ptr = (pkt1_field0 & (~mask)) + (pkt1_field1 & mask);
258 *pkt2_field0_ptr = (pkt2_field0 & (~mask)) + (pkt2_field1 & mask);
259 *pkt3_field0_ptr = (pkt3_field0 & (~mask)) + (pkt3_field1 & mask);
261 *pkt0_field1_ptr = (pkt0_field0 & mask) + (pkt0_field1 & (~mask));
262 *pkt1_field1_ptr = (pkt1_field0 & mask) + (pkt1_field1 & (~mask));
263 *pkt2_field1_ptr = (pkt2_field0 & mask) + (pkt2_field1 & (~mask));
264 *pkt3_field1_ptr = (pkt3_field0 & mask) + (pkt3_field1 & (~mask));
268 #define PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
270 pkt_work_dma_size##dma_size##_hash##hash_enabled \
271 ##_lb##lb_hash##_pw##port_pow2( \
272 struct rte_mbuf *pkt, \
275 pkt_work_dma(pkt, arg, dma_size, hash_enabled, lb_hash, port_pow2); \
278 #define PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
280 pkt4_work_dma_size##dma_size##_hash##hash_enabled \
281 ##_lb##lb_hash##_pw##port_pow2( \
282 struct rte_mbuf **pkts, \
285 pkt4_work_dma(pkts, arg, dma_size, hash_enabled, lb_hash, port_pow2); \
288 #define port_in_ah_dma(dma_size, hash_enabled, lb_hash, port_pow2) \
289 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
290 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
291 PIPELINE_PORT_IN_AH(port_in_ah_dma_size##dma_size##_hash \
292 ##hash_enabled##_lb##lb_hash##_pw##port_pow2, \
293 pkt_work_dma_size##dma_size##_hash##hash_enabled \
294 ##_lb##lb_hash##_pw##port_pow2, \
295 pkt4_work_dma_size##dma_size##_hash##hash_enabled \
296 ##_lb##lb_hash##_pw##port_pow2)
299 #define port_in_ah_lb(dma_size, hash_enabled, lb_hash, port_pow2) \
300 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
301 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2) \
302 PIPELINE_PORT_IN_AH_HIJACK_ALL( \
303 port_in_ah_lb_size##dma_size##_hash##hash_enabled \
304 ##_lb##lb_hash##_pw##port_pow2, \
305 pkt_work_dma_size##dma_size##_hash##hash_enabled \
306 ##_lb##lb_hash##_pw##port_pow2, \
307 pkt4_work_dma_size##dma_size##_hash##hash_enabled \
308 ##_lb##lb_hash##_pw##port_pow2)
310 PIPELINE_PORT_IN_AH(port_in_ah_swap, pkt_work_swap, pkt4_work_swap)
313 /* Port in AH DMA(dma_size, hash_enabled, lb_hash, port_pow2) */
315 port_in_ah_dma(8, 0, 0, 0)
316 port_in_ah_dma(8, 1, 0, 0)
317 port_in_ah_lb(8, 1, 1, 0)
318 port_in_ah_lb(8, 1, 1, 1)
320 port_in_ah_dma(16, 0, 0, 0)
321 port_in_ah_dma(16, 1, 0, 0)
322 port_in_ah_lb(16, 1, 1, 0)
323 port_in_ah_lb(16, 1, 1, 1)
325 port_in_ah_dma(24, 0, 0, 0)
326 port_in_ah_dma(24, 1, 0, 0)
327 port_in_ah_lb(24, 1, 1, 0)
328 port_in_ah_lb(24, 1, 1, 1)
330 port_in_ah_dma(32, 0, 0, 0)
331 port_in_ah_dma(32, 1, 0, 0)
332 port_in_ah_lb(32, 1, 1, 0)
333 port_in_ah_lb(32, 1, 1, 1)
335 port_in_ah_dma(40, 0, 0, 0)
336 port_in_ah_dma(40, 1, 0, 0)
337 port_in_ah_lb(40, 1, 1, 0)
338 port_in_ah_lb(40, 1, 1, 1)
340 port_in_ah_dma(48, 0, 0, 0)
341 port_in_ah_dma(48, 1, 0, 0)
342 port_in_ah_lb(48, 1, 1, 0)
343 port_in_ah_lb(48, 1, 1, 1)
345 port_in_ah_dma(56, 0, 0, 0)
346 port_in_ah_dma(56, 1, 0, 0)
347 port_in_ah_lb(56, 1, 1, 0)
348 port_in_ah_lb(56, 1, 1, 1)
350 port_in_ah_dma(64, 0, 0, 0)
351 port_in_ah_dma(64, 1, 0, 0)
352 port_in_ah_lb(64, 1, 1, 0)
353 port_in_ah_lb(64, 1, 1, 1)
355 static rte_pipeline_port_in_action_handler
356 get_port_in_ah(struct pipeline_passthrough *p)
358 if ((p->params.dma_enabled == 0) &&
359 (p->params.swap_enabled == 0))
362 if (p->params.swap_enabled)
363 return port_in_ah_swap;
365 if (p->params.dma_hash_enabled) {
366 if (p->params.dma_hash_lb_enabled) {
367 if (rte_is_power_of_2(p->p.n_ports_out))
368 switch (p->params.dma_size) {
370 case 8: return port_in_ah_lb_size8_hash1_lb1_pw1;
371 case 16: return port_in_ah_lb_size16_hash1_lb1_pw1;
372 case 24: return port_in_ah_lb_size24_hash1_lb1_pw1;
373 case 32: return port_in_ah_lb_size32_hash1_lb1_pw1;
374 case 40: return port_in_ah_lb_size40_hash1_lb1_pw1;
375 case 48: return port_in_ah_lb_size48_hash1_lb1_pw1;
376 case 56: return port_in_ah_lb_size56_hash1_lb1_pw1;
377 case 64: return port_in_ah_lb_size64_hash1_lb1_pw1;
378 default: return NULL;
381 switch (p->params.dma_size) {
383 case 8: return port_in_ah_lb_size8_hash1_lb1_pw0;
384 case 16: return port_in_ah_lb_size16_hash1_lb1_pw0;
385 case 24: return port_in_ah_lb_size24_hash1_lb1_pw0;
386 case 32: return port_in_ah_lb_size32_hash1_lb1_pw0;
387 case 40: return port_in_ah_lb_size40_hash1_lb1_pw0;
388 case 48: return port_in_ah_lb_size48_hash1_lb1_pw0;
389 case 56: return port_in_ah_lb_size56_hash1_lb1_pw0;
390 case 64: return port_in_ah_lb_size64_hash1_lb1_pw0;
391 default: return NULL;
394 switch (p->params.dma_size) {
396 case 8: return port_in_ah_dma_size8_hash1_lb0_pw0;
397 case 16: return port_in_ah_dma_size16_hash1_lb0_pw0;
398 case 24: return port_in_ah_dma_size24_hash1_lb0_pw0;
399 case 32: return port_in_ah_dma_size32_hash1_lb0_pw0;
400 case 40: return port_in_ah_dma_size40_hash1_lb0_pw0;
401 case 48: return port_in_ah_dma_size48_hash1_lb0_pw0;
402 case 56: return port_in_ah_dma_size56_hash1_lb0_pw0;
403 case 64: return port_in_ah_dma_size64_hash1_lb0_pw0;
404 default: return NULL;
407 switch (p->params.dma_size) {
409 case 8: return port_in_ah_dma_size8_hash0_lb0_pw0;
410 case 16: return port_in_ah_dma_size16_hash0_lb0_pw0;
411 case 24: return port_in_ah_dma_size24_hash0_lb0_pw0;
412 case 32: return port_in_ah_dma_size32_hash0_lb0_pw0;
413 case 40: return port_in_ah_dma_size40_hash0_lb0_pw0;
414 case 48: return port_in_ah_dma_size48_hash0_lb0_pw0;
415 case 56: return port_in_ah_dma_size56_hash0_lb0_pw0;
416 case 64: return port_in_ah_dma_size64_hash0_lb0_pw0;
417 default: return NULL;
422 pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p,
423 struct pipeline_params *params)
425 uint32_t dma_dst_offset_present = 0;
426 uint32_t dma_src_offset_present = 0;
427 uint32_t dma_src_mask_present = 0;
428 char dma_mask_str[PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2 + 1];
429 uint32_t dma_size_present = 0;
430 uint32_t dma_hash_offset_present = 0;
431 uint32_t dma_hash_lb_present = 0;
436 p->dma_hash_enabled = 0;
437 p->dma_hash_lb_enabled = 0;
438 memset(p->dma_src_mask, 0xFF, sizeof(p->dma_src_mask));
440 p->swap_n_fields = 0;
442 for (i = 0; i < params->n_args; i++) {
443 char *arg_name = params->args_name[i];
444 char *arg_value = params->args_value[i];
447 if (strcmp(arg_name, "dma_dst_offset") == 0) {
450 PIPELINE_PARSE_ERR_DUPLICATE(
451 dma_dst_offset_present == 0, params->name,
453 dma_dst_offset_present = 1;
455 status = parser_read_uint32(&p->dma_dst_offset,
457 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
458 params->name, arg_name, arg_value);
459 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
460 params->name, arg_name, arg_value);
468 if (strcmp(arg_name, "dma_src_offset") == 0) {
471 PIPELINE_PARSE_ERR_DUPLICATE(
472 dma_src_offset_present == 0, params->name,
474 dma_src_offset_present = 1;
476 status = parser_read_uint32(&p->dma_src_offset,
478 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
479 params->name, arg_name, arg_value);
480 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
481 params->name, arg_name, arg_value);
489 if (strcmp(arg_name, "dma_size") == 0) {
492 PIPELINE_PARSE_ERR_DUPLICATE(
493 dma_size_present == 0, params->name,
495 dma_size_present = 1;
497 status = parser_read_uint32(&p->dma_size,
499 PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
500 (p->dma_size != 0) &&
501 ((p->dma_size % 8) == 0)),
502 params->name, arg_name, arg_value);
503 PIPELINE_PARSE_ERR_OUT_RNG(((status != -ERANGE) &&
505 PIPELINE_PASSTHROUGH_DMA_SIZE_MAX)),
506 params->name, arg_name, arg_value);
514 if (strcmp(arg_name, "dma_src_mask") == 0) {
515 int mask_str_len = strlen(arg_value);
517 PIPELINE_PARSE_ERR_DUPLICATE(
518 dma_src_mask_present == 0,
519 params->name, arg_name);
520 dma_src_mask_present = 1;
522 PIPELINE_ARG_CHECK((mask_str_len <=
523 (PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2)),
524 "Parse error in section \"%s\": entry "
525 "\"%s\" too long", params->name,
528 strlcpy(dma_mask_str, arg_value, mask_str_len + 1);
535 /* dma_hash_offset */
536 if (strcmp(arg_name, "dma_hash_offset") == 0) {
539 PIPELINE_PARSE_ERR_DUPLICATE(
540 dma_hash_offset_present == 0,
541 params->name, arg_name);
542 dma_hash_offset_present = 1;
544 status = parser_read_uint32(&p->dma_hash_offset,
546 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
547 params->name, arg_name, arg_value);
548 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
549 params->name, arg_name, arg_value);
551 p->dma_hash_enabled = 1;
556 /* load_balance mode */
557 if (strcmp(arg_name, "lb") == 0) {
558 PIPELINE_PARSE_ERR_DUPLICATE(
559 dma_hash_lb_present == 0,
560 params->name, arg_name);
561 dma_hash_lb_present = 1;
563 if (strcmp(arg_value, "hash") &&
564 strcmp(arg_value, "HASH"))
566 PIPELINE_PARSE_ERR_INV_VAL(0,
571 p->dma_hash_lb_enabled = 1;
577 if (strcmp(arg_name, "swap") == 0) {
578 uint32_t a, b, n_args;
581 n_args = sscanf(arg_value, "%" SCNu32 " %" SCNu32 "%n",
583 PIPELINE_PARSE_ERR_INV_VAL(((n_args == 2) &&
584 ((size_t) len == strlen(arg_value))),
585 params->name, arg_name, arg_value);
587 p->swap_field0_offset[p->swap_n_fields] = a;
588 p->swap_field1_offset[p->swap_n_fields] = b;
596 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
599 /* Check correlations between arguments */
600 PIPELINE_ARG_CHECK((p->dma_enabled + p->swap_enabled < 2),
601 "Parse error in section \"%s\": DMA and SWAP actions are both enabled",
603 PIPELINE_ARG_CHECK((dma_dst_offset_present == p->dma_enabled),
604 "Parse error in section \"%s\": missing entry "
605 "\"dma_dst_offset\"", params->name);
606 PIPELINE_ARG_CHECK((dma_src_offset_present == p->dma_enabled),
607 "Parse error in section \"%s\": missing entry "
608 "\"dma_src_offset\"", params->name);
609 PIPELINE_ARG_CHECK((dma_size_present == p->dma_enabled),
610 "Parse error in section \"%s\": missing entry "
611 "\"dma_size\"", params->name);
612 PIPELINE_ARG_CHECK((p->dma_hash_enabled <= p->dma_enabled),
613 "Parse error in section \"%s\": missing all DMA entries",
615 PIPELINE_ARG_CHECK((p->dma_hash_lb_enabled <= p->dma_hash_enabled),
616 "Parse error in section \"%s\": missing all DMA hash entries ",
619 if (dma_src_mask_present) {
620 uint32_t dma_size = p->dma_size;
623 PIPELINE_ARG_CHECK((strlen(dma_mask_str) ==
624 (dma_size * 2)), "Parse error in section "
625 "\"%s\": dma_src_mask should have exactly %u hex "
626 "digits", params->name, (dma_size * 2));
628 status = parse_hex_string(dma_mask_str, p->dma_src_mask,
631 PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
632 (dma_size == p->dma_size)), params->name,
633 "dma_src_mask", dma_mask_str);
636 if (p->dma_hash_lb_enabled)
637 PIPELINE_ARG_CHECK((params->n_ports_out > 1),
638 "Parse error in section \"%s\": entry \"lb\" not "
639 "allowed for single output port pipeline",
642 PIPELINE_ARG_CHECK(((params->n_ports_in >= params->n_ports_out)
643 && ((params->n_ports_in % params->n_ports_out) == 0)),
644 "Parse error in section \"%s\": n_ports_in needs to be "
645 "a multiple of n_ports_out (lb mode disabled)",
651 static rte_table_hash_op_hash
652 get_hash_function(struct pipeline_passthrough *p)
654 switch (p->params.dma_size) {
656 case 8: return hash_default_key8;
657 case 16: return hash_default_key16;
658 case 24: return hash_default_key24;
659 case 32: return hash_default_key32;
660 case 40: return hash_default_key40;
661 case 48: return hash_default_key48;
662 case 56: return hash_default_key56;
663 case 64: return hash_default_key64;
664 default: return NULL;
669 pipeline_passthrough_swap_convert(struct pipeline_passthrough *p)
673 p->swap_n_fields = 0;
675 for (i = 0; i < p->params.swap_n_fields; i++) {
676 uint32_t offset0 = p->params.swap_field0_offset[i];
677 uint32_t offset1 = p->params.swap_field1_offset[i];
678 uint32_t size = offset1 - offset0;
682 if ((offset0 >= offset1) ||
683 (size > PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX) ||
684 (p->swap_n_fields >= SWAP_DIM))
687 for (j = 0; j < (size / sizeof(uint64_t)); j++) {
688 p->swap_field0_offset[p->swap_n_fields] = offset0;
689 p->swap_field1_offset[p->swap_n_fields] = offset1;
690 p->swap_field_mask[p->swap_n_fields] = UINT64_MAX;
692 offset0 += sizeof(uint64_t);
693 offset1 += sizeof(uint64_t);
695 if (size % sizeof(uint64_t)) {
696 uint32_t n_bits = (size % sizeof(uint64_t)) * 8;
698 p->swap_field0_offset[p->swap_n_fields] = offset0;
699 p->swap_field1_offset[p->swap_n_fields] = offset1;
700 p->swap_field_mask[p->swap_n_fields] =
701 RTE_LEN2MASK(n_bits, uint64_t);
710 pipeline_passthrough_init(struct pipeline_params *params,
711 __rte_unused void *arg)
714 struct pipeline_passthrough *p_pt;
717 /* Check input arguments */
718 if ((params == NULL) ||
719 (params->n_ports_in == 0) ||
720 (params->n_ports_out == 0))
723 /* Memory allocation */
724 size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_passthrough));
725 p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
726 p_pt = (struct pipeline_passthrough *) p;
730 strcpy(p->name, params->name);
731 p->log_level = params->log_level;
733 PLOG(p, HIGH, "Pass-through");
735 /* Parse arguments */
736 if (pipeline_passthrough_parse_args(&p_pt->params, params))
738 if (pipeline_passthrough_swap_convert(p_pt))
740 p_pt->f_hash = get_hash_function(p_pt);
744 struct rte_pipeline_params pipeline_params = {
745 .name = "PASS-THROUGH",
746 .socket_id = params->socket_id,
750 p->p = rte_pipeline_create(&pipeline_params);
757 p->n_ports_in = params->n_ports_in;
758 p->n_ports_out = params->n_ports_out;
759 p->n_tables = p->n_ports_in;
762 for (i = 0; i < p->n_ports_in; i++) {
763 struct rte_pipeline_port_in_params port_params = {
764 .ops = pipeline_port_in_params_get_ops(
765 ¶ms->port_in[i]),
766 .arg_create = pipeline_port_in_params_convert(
767 ¶ms->port_in[i]),
768 .f_action = get_port_in_ah(p_pt),
770 .burst_size = params->port_in[i].burst_size,
773 int status = rte_pipeline_port_in_create(p->p,
778 rte_pipeline_free(p->p);
785 for (i = 0; i < p->n_ports_out; i++) {
786 struct rte_pipeline_port_out_params port_params = {
787 .ops = pipeline_port_out_params_get_ops(
788 ¶ms->port_out[i]),
789 .arg_create = pipeline_port_out_params_convert(
790 ¶ms->port_out[i]),
795 int status = rte_pipeline_port_out_create(p->p,
800 rte_pipeline_free(p->p);
807 for (i = 0; i < p->n_ports_in; i++) {
808 struct rte_pipeline_table_params table_params = {
809 .ops = &rte_table_stub_ops,
811 .f_action_hit = NULL,
812 .f_action_miss = NULL,
814 .action_data_size = 0,
817 int status = rte_pipeline_table_create(p->p,
822 rte_pipeline_free(p->p);
828 /* Connecting input ports to tables */
829 for (i = 0; i < p->n_ports_in; i++) {
830 int status = rte_pipeline_port_in_connect_to_table(p->p,
835 rte_pipeline_free(p->p);
841 /* Add entries to tables */
842 for (i = 0; i < p->n_ports_in; i++) {
843 uint32_t port_out_id = (p_pt->params.dma_hash_lb_enabled == 0) ?
844 (i / (p->n_ports_in / p->n_ports_out)) :
847 struct rte_pipeline_table_entry default_entry = {
848 .action = RTE_PIPELINE_ACTION_PORT,
849 {.port_id = p->port_out_id[port_out_id]},
852 struct rte_pipeline_table_entry *default_entry_ptr;
854 int status = rte_pipeline_table_default_entry_add(p->p,
860 rte_pipeline_free(p->p);
866 /* Enable input ports */
867 for (i = 0; i < p->n_ports_in; i++) {
868 int status = rte_pipeline_port_in_enable(p->p,
872 rte_pipeline_free(p->p);
878 /* Check pipeline consistency */
879 if (rte_pipeline_check(p->p) < 0) {
880 rte_pipeline_free(p->p);
886 p->n_msgq = params->n_msgq;
887 for (i = 0; i < p->n_msgq; i++)
888 p->msgq_in[i] = params->msgq_in[i];
889 for (i = 0; i < p->n_msgq; i++)
890 p->msgq_out[i] = params->msgq_out[i];
892 /* Message handlers */
893 memcpy(p->handlers, handlers, sizeof(p->handlers));
899 pipeline_passthrough_free(void *pipeline)
901 struct pipeline *p = (struct pipeline *) pipeline;
903 /* Check input arguments */
908 rte_pipeline_free(p->p);
914 pipeline_passthrough_timer(void *pipeline)
916 struct pipeline *p = (struct pipeline *) pipeline;
918 pipeline_msg_req_handle(p);
919 rte_pipeline_flush(p->p);
924 struct pipeline_be_ops pipeline_passthrough_be_ops = {
925 .f_init = pipeline_passthrough_init,
926 .f_free = pipeline_passthrough_free,
928 .f_timer = pipeline_passthrough_timer,