4 * Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include <rte_common.h>
37 #include <rte_malloc.h>
38 #include <rte_table_hash.h>
39 #include <rte_byteorder.h>
42 #include "pipeline_flow_classification_be.h"
43 #include "pipeline_actions_common.h"
45 #include "hash_func.h"
47 struct pipeline_flow_classification {
49 pipeline_msg_req_handler custom_handlers[PIPELINE_FC_MSG_REQS];
57 uint8_t key_mask[PIPELINE_FC_FLOW_KEY_MAX_SIZE];
58 uint32_t flow_id_offset;
60 } __rte_cache_aligned;
63 pipeline_fc_msg_req_custom_handler(struct pipeline *p, void *msg);
65 static pipeline_msg_req_handler handlers[] = {
66 [PIPELINE_MSG_REQ_PING] =
67 pipeline_msg_req_ping_handler,
68 [PIPELINE_MSG_REQ_STATS_PORT_IN] =
69 pipeline_msg_req_stats_port_in_handler,
70 [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
71 pipeline_msg_req_stats_port_out_handler,
72 [PIPELINE_MSG_REQ_STATS_TABLE] =
73 pipeline_msg_req_stats_table_handler,
74 [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
75 pipeline_msg_req_port_in_enable_handler,
76 [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
77 pipeline_msg_req_port_in_disable_handler,
78 [PIPELINE_MSG_REQ_CUSTOM] =
79 pipeline_fc_msg_req_custom_handler,
83 pipeline_fc_msg_req_add_handler(struct pipeline *p, void *msg);
86 pipeline_fc_msg_req_add_bulk_handler(struct pipeline *p, void *msg);
89 pipeline_fc_msg_req_del_handler(struct pipeline *p, void *msg);
92 pipeline_fc_msg_req_add_default_handler(struct pipeline *p, void *msg);
95 pipeline_fc_msg_req_del_default_handler(struct pipeline *p, void *msg);
97 static pipeline_msg_req_handler custom_handlers[] = {
98 [PIPELINE_FC_MSG_REQ_FLOW_ADD] =
99 pipeline_fc_msg_req_add_handler,
100 [PIPELINE_FC_MSG_REQ_FLOW_ADD_BULK] =
101 pipeline_fc_msg_req_add_bulk_handler,
102 [PIPELINE_FC_MSG_REQ_FLOW_DEL] =
103 pipeline_fc_msg_req_del_handler,
104 [PIPELINE_FC_MSG_REQ_FLOW_ADD_DEFAULT] =
105 pipeline_fc_msg_req_add_default_handler,
106 [PIPELINE_FC_MSG_REQ_FLOW_DEL_DEFAULT] =
107 pipeline_fc_msg_req_del_default_handler,
113 struct flow_table_entry {
114 struct rte_pipeline_table_entry head;
120 rte_table_hash_op_hash hash_func[] = {
132 * Flow table AH - Write flow_id to packet meta-data
136 struct rte_mbuf *pkt,
137 struct rte_pipeline_table_entry *table_entry,
140 struct pipeline_flow_classification *p_fc = arg;
141 uint32_t *flow_id_ptr =
142 RTE_MBUF_METADATA_UINT32_PTR(pkt, p_fc->flow_id_offset);
143 struct flow_table_entry *entry =
144 (struct flow_table_entry *) table_entry;
147 uint32_t flow_id = entry->flow_id;
152 *flow_id_ptr = flow_id;
157 struct rte_mbuf **pkts,
158 struct rte_pipeline_table_entry **table_entries,
161 struct pipeline_flow_classification *p_fc = arg;
163 uint32_t *flow_id_ptr0 =
164 RTE_MBUF_METADATA_UINT32_PTR(pkts[0], p_fc->flow_id_offset);
165 uint32_t *flow_id_ptr1 =
166 RTE_MBUF_METADATA_UINT32_PTR(pkts[1], p_fc->flow_id_offset);
167 uint32_t *flow_id_ptr2 =
168 RTE_MBUF_METADATA_UINT32_PTR(pkts[2], p_fc->flow_id_offset);
169 uint32_t *flow_id_ptr3 =
170 RTE_MBUF_METADATA_UINT32_PTR(pkts[3], p_fc->flow_id_offset);
172 struct flow_table_entry *entry0 =
173 (struct flow_table_entry *) table_entries[0];
174 struct flow_table_entry *entry1 =
175 (struct flow_table_entry *) table_entries[1];
176 struct flow_table_entry *entry2 =
177 (struct flow_table_entry *) table_entries[2];
178 struct flow_table_entry *entry3 =
179 (struct flow_table_entry *) table_entries[3];
182 uint32_t flow_id0 = entry0->flow_id;
183 uint32_t flow_id1 = entry1->flow_id;
184 uint32_t flow_id2 = entry2->flow_id;
185 uint32_t flow_id3 = entry3->flow_id;
190 *flow_id_ptr0 = flow_id0;
191 *flow_id_ptr1 = flow_id1;
192 *flow_id_ptr2 = flow_id2;
193 *flow_id_ptr3 = flow_id3;
196 PIPELINE_TABLE_AH_HIT(fc_table_ah_hit,
197 pkt_work_flow_id, pkt4_work_flow_id);
199 static rte_pipeline_table_action_handler_hit
200 get_fc_table_ah_hit(struct pipeline_flow_classification *p)
203 return fc_table_ah_hit;
212 pipeline_fc_parse_args(struct pipeline_flow_classification *p,
213 struct pipeline_params *params)
215 uint32_t n_flows_present = 0;
216 uint32_t key_offset_present = 0;
217 uint32_t key_size_present = 0;
218 uint32_t hash_offset_present = 0;
219 uint32_t key_mask_present = 0;
220 uint32_t flow_id_offset_present = 0;
223 char key_mask_str[PIPELINE_FC_FLOW_KEY_MAX_SIZE * 2];
230 for (i = 0; i < params->n_args; i++) {
231 char *arg_name = params->args_name[i];
232 char *arg_value = params->args_value[i];
235 if (strcmp(arg_name, "n_flows") == 0) {
238 PIPELINE_PARSE_ERR_DUPLICATE(
239 n_flows_present == 0, params->name,
243 status = parser_read_uint32(&p->n_flows,
245 PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
246 (p->n_flows != 0)), params->name,
247 arg_name, arg_value);
248 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
249 params->name, arg_name, arg_value);
255 if (strcmp(arg_name, "key_offset") == 0) {
258 PIPELINE_PARSE_ERR_DUPLICATE(
259 key_offset_present == 0, params->name,
261 key_offset_present = 1;
263 status = parser_read_uint32(&p->key_offset,
265 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
266 params->name, arg_name, arg_value);
267 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
268 params->name, arg_name, arg_value);
274 if (strcmp(arg_name, "key_size") == 0) {
277 PIPELINE_PARSE_ERR_DUPLICATE(
278 key_size_present == 0, params->name,
280 key_size_present = 1;
282 status = parser_read_uint32(&p->key_size,
284 PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
285 (p->key_size != 0) &&
286 (p->key_size % 8 == 0)),
287 params->name, arg_name, arg_value);
288 PIPELINE_PARSE_ERR_OUT_RNG(((status != -ERANGE) &&
290 PIPELINE_FC_FLOW_KEY_MAX_SIZE)),
291 params->name, arg_name, arg_value);
297 if (strcmp(arg_name, "key_mask") == 0) {
298 int mask_str_len = strlen(arg_value);
300 PIPELINE_PARSE_ERR_DUPLICATE(
301 key_mask_present == 0,
302 params->name, arg_name);
303 key_mask_present = 1;
305 PIPELINE_ARG_CHECK((mask_str_len <
306 (PIPELINE_FC_FLOW_KEY_MAX_SIZE * 2)),
307 "Parse error in section \"%s\": entry "
308 "\"%s\" is too long", params->name,
311 snprintf(key_mask_str, mask_str_len, "%s",
318 if (strcmp(arg_name, "hash_offset") == 0) {
321 PIPELINE_PARSE_ERR_DUPLICATE(
322 hash_offset_present == 0, params->name,
324 hash_offset_present = 1;
326 status = parser_read_uint32(&p->hash_offset,
328 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
329 params->name, arg_name, arg_value);
330 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
331 params->name, arg_name, arg_value);
337 if (strcmp(arg_name, "flowid_offset") == 0) {
340 PIPELINE_PARSE_ERR_DUPLICATE(
341 flow_id_offset_present == 0, params->name,
343 flow_id_offset_present = 1;
345 status = parser_read_uint32(&p->flow_id_offset,
347 PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
348 params->name, arg_name, arg_value);
349 PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
350 params->name, arg_name, arg_value);
357 /* Unknown argument */
358 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
361 /* Check that mandatory arguments are present */
362 PIPELINE_PARSE_ERR_MANDATORY((n_flows_present), params->name,
364 PIPELINE_PARSE_ERR_MANDATORY((key_offset_present), params->name,
366 PIPELINE_PARSE_ERR_MANDATORY((key_size_present), params->name,
369 if (key_mask_present) {
370 uint32_t key_size = p->key_size;
373 PIPELINE_ARG_CHECK((strlen(key_mask_str) ==
374 (key_size * 2)), "Parse error in section "
375 "\"%s\": key_mask should have exactly %u hex "
376 "digits", params->name, (key_size * 2));
378 status = parse_hex_string(key_mask_str, p->key_mask,
381 PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
382 (key_size == p->key_size)), params->name,
383 "key_mask", key_mask_str);
389 static void *pipeline_fc_init(struct pipeline_params *params,
390 __rte_unused void *arg)
393 struct pipeline_flow_classification *p_fc;
396 /* Check input arguments */
400 /* Memory allocation */
401 size = RTE_CACHE_LINE_ROUNDUP(
402 sizeof(struct pipeline_flow_classification));
403 p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
406 p_fc = (struct pipeline_flow_classification *) p;
408 strcpy(p->name, params->name);
409 p->log_level = params->log_level;
411 PLOG(p, HIGH, "Flow classification");
413 /* Parse arguments */
414 if (pipeline_fc_parse_args(p_fc, params))
419 struct rte_pipeline_params pipeline_params = {
420 .name = params->name,
421 .socket_id = params->socket_id,
425 p->p = rte_pipeline_create(&pipeline_params);
433 p->n_ports_in = params->n_ports_in;
434 for (i = 0; i < p->n_ports_in; i++) {
435 struct rte_pipeline_port_in_params port_params = {
436 .ops = pipeline_port_in_params_get_ops(
437 ¶ms->port_in[i]),
438 .arg_create = pipeline_port_in_params_convert(
439 ¶ms->port_in[i]),
442 .burst_size = params->port_in[i].burst_size,
445 int status = rte_pipeline_port_in_create(p->p,
450 rte_pipeline_free(p->p);
457 p->n_ports_out = params->n_ports_out;
458 for (i = 0; i < p->n_ports_out; i++) {
459 struct rte_pipeline_port_out_params port_params = {
460 .ops = pipeline_port_out_params_get_ops(
461 ¶ms->port_out[i]),
462 .arg_create = pipeline_port_out_params_convert(
463 ¶ms->port_out[i]),
468 int status = rte_pipeline_port_out_create(p->p,
473 rte_pipeline_free(p->p);
482 struct rte_table_hash_key8_ext_params
483 table_hash_key8_params = {
484 .n_entries = p_fc->n_flows,
485 .n_entries_ext = p_fc->n_flows,
486 .signature_offset = p_fc->hash_offset,
487 .key_offset = p_fc->key_offset,
488 .f_hash = hash_func[(p_fc->key_size / 8) - 1],
489 .key_mask = p_fc->key_mask,
493 struct rte_table_hash_key16_ext_params
494 table_hash_key16_params = {
495 .n_entries = p_fc->n_flows,
496 .n_entries_ext = p_fc->n_flows,
497 .signature_offset = p_fc->hash_offset,
498 .key_offset = p_fc->key_offset,
499 .f_hash = hash_func[(p_fc->key_size / 8) - 1],
500 .key_mask = p_fc->key_mask,
504 struct rte_table_hash_ext_params
505 table_hash_params = {
506 .key_size = p_fc->key_size,
507 .n_keys = p_fc->n_flows,
508 .n_buckets = p_fc->n_flows / 4,
509 .n_buckets_ext = p_fc->n_flows / 4,
510 .f_hash = hash_func[(p_fc->key_size / 8) - 1],
512 .signature_offset = p_fc->hash_offset,
513 .key_offset = p_fc->key_offset,
516 struct rte_pipeline_table_params table_params = {
517 .ops = NULL, /* set below */
518 .arg_create = NULL, /* set below */
519 .f_action_hit = get_fc_table_ah_hit(p_fc),
520 .f_action_miss = NULL,
522 .action_data_size = sizeof(struct flow_table_entry) -
523 sizeof(struct rte_pipeline_table_entry),
528 switch (p_fc->key_size) {
530 if (p_fc->hash_offset != 0) {
532 &rte_table_hash_key8_ext_ops;
535 &rte_table_hash_key8_ext_dosig_ops;
537 table_params.arg_create = &table_hash_key8_params;
541 if (p_fc->hash_offset != 0) {
543 &rte_table_hash_key16_ext_ops;
546 &rte_table_hash_key16_ext_dosig_ops;
548 table_params.arg_create = &table_hash_key16_params;
552 table_params.ops = &rte_table_hash_ext_ops;
553 table_params.arg_create = &table_hash_params;
556 status = rte_pipeline_table_create(p->p,
561 rte_pipeline_free(p->p);
567 /* Connecting input ports to tables */
568 for (i = 0; i < p->n_ports_in; i++) {
569 int status = rte_pipeline_port_in_connect_to_table(p->p,
574 rte_pipeline_free(p->p);
580 /* Enable input ports */
581 for (i = 0; i < p->n_ports_in; i++) {
582 int status = rte_pipeline_port_in_enable(p->p,
586 rte_pipeline_free(p->p);
592 /* Check pipeline consistency */
593 if (rte_pipeline_check(p->p) < 0) {
594 rte_pipeline_free(p->p);
600 p->n_msgq = params->n_msgq;
601 for (i = 0; i < p->n_msgq; i++)
602 p->msgq_in[i] = params->msgq_in[i];
603 for (i = 0; i < p->n_msgq; i++)
604 p->msgq_out[i] = params->msgq_out[i];
606 /* Message handlers */
607 memcpy(p->handlers, handlers, sizeof(p->handlers));
608 memcpy(p_fc->custom_handlers,
610 sizeof(p_fc->custom_handlers));
616 pipeline_fc_free(void *pipeline)
618 struct pipeline *p = (struct pipeline *) pipeline;
620 /* Check input arguments */
625 rte_pipeline_free(p->p);
631 pipeline_fc_track(void *pipeline,
632 __rte_unused uint32_t port_in,
635 struct pipeline *p = (struct pipeline *) pipeline;
637 /* Check input arguments */
639 (port_in >= p->n_ports_in) ||
643 if (p->n_ports_in == 1) {
652 pipeline_fc_timer(void *pipeline)
654 struct pipeline *p = (struct pipeline *) pipeline;
656 pipeline_msg_req_handle(p);
657 rte_pipeline_flush(p->p);
663 pipeline_fc_msg_req_custom_handler(struct pipeline *p, void *msg)
665 struct pipeline_flow_classification *p_fc =
666 (struct pipeline_flow_classification *) p;
667 struct pipeline_custom_msg_req *req = msg;
668 pipeline_msg_req_handler f_handle;
670 f_handle = (req->subtype < PIPELINE_FC_MSG_REQS) ?
671 p_fc->custom_handlers[req->subtype] :
672 pipeline_msg_req_invalid_handler;
674 if (f_handle == NULL)
675 f_handle = pipeline_msg_req_invalid_handler;
677 return f_handle(p, req);
681 pipeline_fc_msg_req_add_handler(struct pipeline *p, void *msg)
683 struct pipeline_fc_add_msg_req *req = msg;
684 struct pipeline_fc_add_msg_rsp *rsp = msg;
686 struct flow_table_entry entry = {
688 .action = RTE_PIPELINE_ACTION_PORT,
689 {.port_id = p->port_out_id[req->port_id]},
691 .flow_id = req->flow_id,
694 rsp->status = rte_pipeline_table_entry_add(p->p,
697 (struct rte_pipeline_table_entry *) &entry,
699 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
705 pipeline_fc_msg_req_add_bulk_handler(struct pipeline *p, void *msg)
707 struct pipeline_fc_add_bulk_msg_req *req = msg;
708 struct pipeline_fc_add_bulk_msg_rsp *rsp = msg;
711 for (i = 0; i < req->n_keys; i++) {
712 struct pipeline_fc_add_bulk_flow_req *flow_req = &req->req[i];
713 struct pipeline_fc_add_bulk_flow_rsp *flow_rsp = &req->rsp[i];
715 struct flow_table_entry entry = {
717 .action = RTE_PIPELINE_ACTION_PORT,
718 {.port_id = p->port_out_id[flow_req->port_id]},
720 .flow_id = flow_req->flow_id,
723 int status = rte_pipeline_table_entry_add(p->p,
726 (struct rte_pipeline_table_entry *) &entry,
727 &flow_rsp->key_found,
728 (struct rte_pipeline_table_entry **)
729 &flow_rsp->entry_ptr);
741 pipeline_fc_msg_req_del_handler(struct pipeline *p, void *msg)
743 struct pipeline_fc_del_msg_req *req = msg;
744 struct pipeline_fc_del_msg_rsp *rsp = msg;
746 rsp->status = rte_pipeline_table_entry_delete(p->p,
756 pipeline_fc_msg_req_add_default_handler(struct pipeline *p, void *msg)
758 struct pipeline_fc_add_default_msg_req *req = msg;
759 struct pipeline_fc_add_default_msg_rsp *rsp = msg;
761 struct flow_table_entry default_entry = {
763 .action = RTE_PIPELINE_ACTION_PORT,
764 {.port_id = p->port_out_id[req->port_id]},
770 rsp->status = rte_pipeline_table_default_entry_add(p->p,
772 (struct rte_pipeline_table_entry *) &default_entry,
773 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
779 pipeline_fc_msg_req_del_default_handler(struct pipeline *p, void *msg)
781 struct pipeline_fc_del_default_msg_rsp *rsp = msg;
783 rsp->status = rte_pipeline_table_default_entry_delete(p->p,
790 struct pipeline_be_ops pipeline_flow_classification_be_ops = {
791 .f_init = pipeline_fc_init,
792 .f_free = pipeline_fc_free,
794 .f_timer = pipeline_fc_timer,
795 .f_track = pipeline_fc_track,