examples/ip_pipeline/pipeline/pipeline_passthrough_be.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <string.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_malloc.h>
  10 #include <rte_byteorder.h>
  11 #include <rte_string_fns.h>
  12 #include <rte_table_stub.h>
  13 #include <rte_table_hash.h>
  14 #include <rte_pipeline.h>
  15
  16 #include "pipeline_passthrough_be.h"
  17 #include "pipeline_actions_common.h"
  18 #include "parser.h"
  19 #include "hash_func.h"
  20
  21 #define SWAP_DIM (PIPELINE_PASSTHROUGH_SWAP_N_FIELDS_MAX * \
  22         (PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX / sizeof(uint64_t)))
  23
  24 struct pipeline_passthrough {
  25         struct pipeline p;
  26         struct pipeline_passthrough_params params;
  27         rte_table_hash_op_hash f_hash;
  28         uint32_t swap_field0_offset[SWAP_DIM];
  29         uint32_t swap_field1_offset[SWAP_DIM];
  30         uint64_t swap_field_mask[SWAP_DIM];
  31         uint32_t swap_n_fields;
  32 } __rte_cache_aligned;
  33
  34 static pipeline_msg_req_handler handlers[] = {
  35         [PIPELINE_MSG_REQ_PING] =
  36                 pipeline_msg_req_ping_handler,
  37         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  38                 pipeline_msg_req_stats_port_in_handler,
  39         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  40                 pipeline_msg_req_stats_port_out_handler,
  41         [PIPELINE_MSG_REQ_STATS_TABLE] =
  42                 pipeline_msg_req_stats_table_handler,
  43         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  44                 pipeline_msg_req_port_in_enable_handler,
  45         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
  46                 pipeline_msg_req_port_in_disable_handler,
  47         [PIPELINE_MSG_REQ_CUSTOM] =
  48                 pipeline_msg_req_invalid_handler,
  49 };
  50
  51 static __rte_always_inline void
  52 pkt_work_dma(
  53         struct rte_mbuf *pkt,
  54         void *arg,
  55         uint32_t dma_size,
  56         uint32_t hash_enabled,
  57         uint32_t lb_hash,
  58         uint32_t port_out_pow2)
  59 {
  60         struct pipeline_passthrough *p = arg;
  61
  62         uint64_t *dma_dst = RTE_MBUF_METADATA_UINT64_PTR(pkt,
  63                 p->params.dma_dst_offset);
  64         uint64_t *dma_src = RTE_MBUF_METADATA_UINT64_PTR(pkt,
  65                 p->params.dma_src_offset);
  66         uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
  67         uint32_t *dma_hash = RTE_MBUF_METADATA_UINT32_PTR(pkt,
  68                 p->params.dma_hash_offset);
  69         uint32_t i;
  70
  71         /* Read (dma_src), compute (dma_dst), write (dma_dst) */
  72         for (i = 0; i < (dma_size / 8); i++)
  73                 dma_dst[i] = dma_src[i] & dma_mask[i];
  74
  75         /* Read (dma_dst), compute (hash), write (hash) */
  76         if (hash_enabled) {
  77                 uint32_t hash = p->f_hash(dma_src, dma_mask, dma_size, 0);
  78                 *dma_hash = hash;
  79
  80                 if (lb_hash) {
  81                         uint32_t port_out;
  82
  83                         if (port_out_pow2)
  84                                 port_out
  85                                         = hash & (p->p.n_ports_out - 1);
  86                         else
  87                                 port_out
  88                                         = hash % p->p.n_ports_out;
  89
  90                         rte_pipeline_port_out_packet_insert(p->p.p,
  91                                 port_out, pkt);
  92                 }
  93         }
  94 }
  95
  96 static __rte_always_inline void
  97 pkt4_work_dma(
  98         struct rte_mbuf **pkts,
  99         void *arg,
 100         uint32_t dma_size,
 101         uint32_t hash_enabled,
 102         uint32_t lb_hash,
 103         uint32_t port_out_pow2)
 104 {
 105         struct pipeline_passthrough *p = arg;
 106
 107         uint64_t *dma_dst0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 108                 p->params.dma_dst_offset);
 109         uint64_t *dma_dst1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 110                 p->params.dma_dst_offset);
 111         uint64_t *dma_dst2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 112                 p->params.dma_dst_offset);
 113         uint64_t *dma_dst3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 114                 p->params.dma_dst_offset);
 115
 116         uint64_t *dma_src0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 117                 p->params.dma_src_offset);
 118         uint64_t *dma_src1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 119                 p->params.dma_src_offset);
 120         uint64_t *dma_src2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 121                 p->params.dma_src_offset);
 122         uint64_t *dma_src3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 123                 p->params.dma_src_offset);
 124
 125         uint64_t *dma_mask = (uint64_t *) p->params.dma_src_mask;
 126
 127         uint32_t *dma_hash0 = RTE_MBUF_METADATA_UINT32_PTR(pkts[0],
 128                 p->params.dma_hash_offset);
 129         uint32_t *dma_hash1 = RTE_MBUF_METADATA_UINT32_PTR(pkts[1],
 130                 p->params.dma_hash_offset);
 131         uint32_t *dma_hash2 = RTE_MBUF_METADATA_UINT32_PTR(pkts[2],
 132                 p->params.dma_hash_offset);
 133         uint32_t *dma_hash3 = RTE_MBUF_METADATA_UINT32_PTR(pkts[3],
 134                 p->params.dma_hash_offset);
 135
 136         uint32_t i;
 137
 138         /* Read (dma_src), compute (dma_dst), write (dma_dst) */
 139         for (i = 0; i < (dma_size / 8); i++) {
 140                 dma_dst0[i] = dma_src0[i] & dma_mask[i];
 141                 dma_dst1[i] = dma_src1[i] & dma_mask[i];
 142                 dma_dst2[i] = dma_src2[i] & dma_mask[i];
 143                 dma_dst3[i] = dma_src3[i] & dma_mask[i];
 144         }
 145
 146         /* Read (dma_dst), compute (hash), write (hash) */
 147         if (hash_enabled) {
 148                 uint32_t hash0 = p->f_hash(dma_src0, dma_mask, dma_size, 0);
 149                 uint32_t hash1 = p->f_hash(dma_src1, dma_mask, dma_size, 0);
 150                 uint32_t hash2 = p->f_hash(dma_src2, dma_mask, dma_size, 0);
 151                 uint32_t hash3 = p->f_hash(dma_src3, dma_mask, dma_size, 0);
 152
 153                 *dma_hash0 = hash0;
 154                 *dma_hash1 = hash1;
 155                 *dma_hash2 = hash2;
 156                 *dma_hash3 = hash3;
 157
 158                 if (lb_hash) {
 159                         uint32_t port_out0, port_out1, port_out2, port_out3;
 160
 161                         if (port_out_pow2) {
 162                                 port_out0
 163                                         = hash0 & (p->p.n_ports_out - 1);
 164                                 port_out1
 165                                         = hash1 & (p->p.n_ports_out - 1);
 166                                 port_out2
 167                                         = hash2 & (p->p.n_ports_out - 1);
 168                                 port_out3
 169                                         = hash3 & (p->p.n_ports_out - 1);
 170                         } else {
 171                                 port_out0
 172                                         = hash0 % p->p.n_ports_out;
 173                                 port_out1
 174                                         = hash1 % p->p.n_ports_out;
 175                                 port_out2
 176                                         = hash2 % p->p.n_ports_out;
 177                                 port_out3
 178                                         = hash3 % p->p.n_ports_out;
 179                         }
 180                         rte_pipeline_port_out_packet_insert(p->p.p,
 181                                 port_out0, pkts[0]);
 182                         rte_pipeline_port_out_packet_insert(p->p.p,
 183                                 port_out1, pkts[1]);
 184                         rte_pipeline_port_out_packet_insert(p->p.p,
 185                                 port_out2, pkts[2]);
 186                         rte_pipeline_port_out_packet_insert(p->p.p,
 187                                 port_out3, pkts[3]);
 188                 }
 189         }
 190 }
 191
 192 static __rte_always_inline void
 193 pkt_work_swap(
 194         struct rte_mbuf *pkt,
 195         void *arg)
 196 {
 197         struct pipeline_passthrough *p = arg;
 198         uint32_t i;
 199
 200         /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
 201         for (i = 0; i < p->swap_n_fields; i++) {
 202                 uint64_t *field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
 203                         p->swap_field0_offset[i]);
 204                 uint64_t *field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt,
 205                         p->swap_field1_offset[i]);
 206                 uint64_t mask = p->swap_field_mask[i];
 207
 208                 uint64_t field0 = *field0_ptr;
 209                 uint64_t field1 = *field1_ptr;
 210
 211                 *field0_ptr = (field0 & (~mask)) + (field1 & mask);
 212                 *field1_ptr = (field0 & mask) + (field1 & (~mask));
 213         }
 214 }
 215
 216 static __rte_always_inline void
 217 pkt4_work_swap(
 218         struct rte_mbuf **pkts,
 219         void *arg)
 220 {
 221         struct pipeline_passthrough *p = arg;
 222         uint32_t i;
 223
 224         /* Read(field0, field1), compute(field0, field1), write(field0, field1) */
 225         for (i = 0; i < p->swap_n_fields; i++) {
 226                 uint64_t *pkt0_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 227                         p->swap_field0_offset[i]);
 228                 uint64_t *pkt1_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 229                         p->swap_field0_offset[i]);
 230                 uint64_t *pkt2_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 231                         p->swap_field0_offset[i]);
 232                 uint64_t *pkt3_field0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 233                         p->swap_field0_offset[i]);
 234
 235                 uint64_t *pkt0_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 236                         p->swap_field1_offset[i]);
 237                 uint64_t *pkt1_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 238                         p->swap_field1_offset[i]);
 239                 uint64_t *pkt2_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 240                         p->swap_field1_offset[i]);
 241                 uint64_t *pkt3_field1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 242                         p->swap_field1_offset[i]);
 243
 244                 uint64_t mask = p->swap_field_mask[i];
 245
 246                 uint64_t pkt0_field0 = *pkt0_field0_ptr;
 247                 uint64_t pkt1_field0 = *pkt1_field0_ptr;
 248                 uint64_t pkt2_field0 = *pkt2_field0_ptr;
 249                 uint64_t pkt3_field0 = *pkt3_field0_ptr;
 250
 251                 uint64_t pkt0_field1 = *pkt0_field1_ptr;
 252                 uint64_t pkt1_field1 = *pkt1_field1_ptr;
 253                 uint64_t pkt2_field1 = *pkt2_field1_ptr;
 254                 uint64_t pkt3_field1 = *pkt3_field1_ptr;
 255
 256                 *pkt0_field0_ptr = (pkt0_field0 & (~mask)) + (pkt0_field1 & mask);
 257                 *pkt1_field0_ptr = (pkt1_field0 & (~mask)) + (pkt1_field1 & mask);
 258                 *pkt2_field0_ptr = (pkt2_field0 & (~mask)) + (pkt2_field1 & mask);
 259                 *pkt3_field0_ptr = (pkt3_field0 & (~mask)) + (pkt3_field1 & mask);
 260
 261                 *pkt0_field1_ptr = (pkt0_field0 & mask) + (pkt0_field1 & (~mask));
 262                 *pkt1_field1_ptr = (pkt1_field0 & mask) + (pkt1_field1 & (~mask));
 263                 *pkt2_field1_ptr = (pkt2_field0 & mask) + (pkt2_field1 & (~mask));
 264                 *pkt3_field1_ptr = (pkt3_field0 & mask) + (pkt3_field1 & (~mask));
 265         }
 266 }
 267
 268 #define PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)        \
 269 static inline void                                              \
 270 pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 271         ##_lb##lb_hash##_pw##port_pow2(                 \
 272         struct rte_mbuf *pkt,                                   \
 273         void *arg)                                              \
 274 {                                                               \
 275         pkt_work_dma(pkt, arg, dma_size, hash_enabled, lb_hash, port_pow2);     \
 276 }
 277
 278 #define PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)       \
 279 static inline void                                              \
 280 pkt4_work_dma_size##dma_size##_hash##hash_enabled                       \
 281         ##_lb##lb_hash##_pw##port_pow2(                 \
 282         struct rte_mbuf **pkts,                                 \
 283         void *arg)                                              \
 284 {                                                               \
 285         pkt4_work_dma(pkts, arg, dma_size, hash_enabled, lb_hash, port_pow2); \
 286 }
 287
 288 #define port_in_ah_dma(dma_size, hash_enabled, lb_hash, port_pow2)      \
 289 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                        \
 290 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                       \
 291 PIPELINE_PORT_IN_AH(port_in_ah_dma_size##dma_size##_hash        \
 292         ##hash_enabled##_lb##lb_hash##_pw##port_pow2,           \
 293         pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 294         ##_lb##lb_hash##_pw##port_pow2,                 \
 295         pkt4_work_dma_size##dma_size##_hash##hash_enabled               \
 296         ##_lb##lb_hash##_pw##port_pow2)
 297
 298
 299 #define port_in_ah_lb(dma_size, hash_enabled, lb_hash, port_pow2) \
 300 PKT_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)                \
 301 PKT4_WORK_DMA(dma_size, hash_enabled, lb_hash, port_pow2)       \
 302 PIPELINE_PORT_IN_AH_HIJACK_ALL(                                         \
 303         port_in_ah_lb_size##dma_size##_hash##hash_enabled               \
 304         ##_lb##lb_hash##_pw##port_pow2,                 \
 305         pkt_work_dma_size##dma_size##_hash##hash_enabled                \
 306         ##_lb##lb_hash##_pw##port_pow2, \
 307         pkt4_work_dma_size##dma_size##_hash##hash_enabled               \
 308         ##_lb##lb_hash##_pw##port_pow2)
 309
 310 PIPELINE_PORT_IN_AH(port_in_ah_swap, pkt_work_swap,     pkt4_work_swap)
 311
 312
 313 /* Port in AH DMA(dma_size, hash_enabled, lb_hash, port_pow2) */
 314
 315 port_in_ah_dma(8, 0, 0, 0)
 316 port_in_ah_dma(8, 1, 0, 0)
 317 port_in_ah_lb(8, 1, 1, 0)
 318 port_in_ah_lb(8, 1, 1, 1)
 319
 320 port_in_ah_dma(16, 0, 0, 0)
 321 port_in_ah_dma(16, 1, 0, 0)
 322 port_in_ah_lb(16, 1, 1, 0)
 323 port_in_ah_lb(16, 1, 1, 1)
 324
 325 port_in_ah_dma(24, 0, 0, 0)
 326 port_in_ah_dma(24, 1, 0, 0)
 327 port_in_ah_lb(24, 1, 1, 0)
 328 port_in_ah_lb(24, 1, 1, 1)
 329
 330 port_in_ah_dma(32, 0, 0, 0)
 331 port_in_ah_dma(32, 1, 0, 0)
 332 port_in_ah_lb(32, 1, 1, 0)
 333 port_in_ah_lb(32, 1, 1, 1)
 334
 335 port_in_ah_dma(40, 0, 0, 0)
 336 port_in_ah_dma(40, 1, 0, 0)
 337 port_in_ah_lb(40, 1, 1, 0)
 338 port_in_ah_lb(40, 1, 1, 1)
 339
 340 port_in_ah_dma(48, 0, 0, 0)
 341 port_in_ah_dma(48, 1, 0, 0)
 342 port_in_ah_lb(48, 1, 1, 0)
 343 port_in_ah_lb(48, 1, 1, 1)
 344
 345 port_in_ah_dma(56, 0, 0, 0)
 346 port_in_ah_dma(56, 1, 0, 0)
 347 port_in_ah_lb(56, 1, 1, 0)
 348 port_in_ah_lb(56, 1, 1, 1)
 349
 350 port_in_ah_dma(64, 0, 0, 0)
 351 port_in_ah_dma(64, 1, 0, 0)
 352 port_in_ah_lb(64, 1, 1, 0)
 353 port_in_ah_lb(64, 1, 1, 1)
 354
 355 static rte_pipeline_port_in_action_handler
 356 get_port_in_ah(struct pipeline_passthrough *p)
 357 {
 358         if ((p->params.dma_enabled == 0) &&
 359                 (p->params.swap_enabled == 0))
 360                 return NULL;
 361
 362         if (p->params.swap_enabled)
 363                 return port_in_ah_swap;
 364
 365         if (p->params.dma_hash_enabled) {
 366                 if (p->params.dma_hash_lb_enabled) {
 367                         if (rte_is_power_of_2(p->p.n_ports_out))
 368                                 switch (p->params.dma_size) {
 369
 370                                 case 8: return port_in_ah_lb_size8_hash1_lb1_pw1;
 371                                 case 16: return port_in_ah_lb_size16_hash1_lb1_pw1;
 372                                 case 24: return port_in_ah_lb_size24_hash1_lb1_pw1;
 373                                 case 32: return port_in_ah_lb_size32_hash1_lb1_pw1;
 374                                 case 40: return port_in_ah_lb_size40_hash1_lb1_pw1;
 375                                 case 48: return port_in_ah_lb_size48_hash1_lb1_pw1;
 376                                 case 56: return port_in_ah_lb_size56_hash1_lb1_pw1;
 377                                 case 64: return port_in_ah_lb_size64_hash1_lb1_pw1;
 378                                 default: return NULL;
 379                                 }
 380                         else
 381                                 switch (p->params.dma_size) {
 382
 383                                 case 8: return port_in_ah_lb_size8_hash1_lb1_pw0;
 384                                 case 16: return port_in_ah_lb_size16_hash1_lb1_pw0;
 385                                 case 24: return port_in_ah_lb_size24_hash1_lb1_pw0;
 386                                 case 32: return port_in_ah_lb_size32_hash1_lb1_pw0;
 387                                 case 40: return port_in_ah_lb_size40_hash1_lb1_pw0;
 388                                 case 48: return port_in_ah_lb_size48_hash1_lb1_pw0;
 389                                 case 56: return port_in_ah_lb_size56_hash1_lb1_pw0;
 390                                 case 64: return port_in_ah_lb_size64_hash1_lb1_pw0;
 391                                 default: return NULL;
 392                         }
 393                 } else
 394                         switch (p->params.dma_size) {
 395
 396                         case 8: return port_in_ah_dma_size8_hash1_lb0_pw0;
 397                         case 16: return port_in_ah_dma_size16_hash1_lb0_pw0;
 398                         case 24: return port_in_ah_dma_size24_hash1_lb0_pw0;
 399                         case 32: return port_in_ah_dma_size32_hash1_lb0_pw0;
 400                         case 40: return port_in_ah_dma_size40_hash1_lb0_pw0;
 401                         case 48: return port_in_ah_dma_size48_hash1_lb0_pw0;
 402                         case 56: return port_in_ah_dma_size56_hash1_lb0_pw0;
 403                         case 64: return port_in_ah_dma_size64_hash1_lb0_pw0;
 404                         default: return NULL;
 405                 }
 406         } else
 407                 switch (p->params.dma_size) {
 408
 409                 case 8: return port_in_ah_dma_size8_hash0_lb0_pw0;
 410                 case 16: return port_in_ah_dma_size16_hash0_lb0_pw0;
 411                 case 24: return port_in_ah_dma_size24_hash0_lb0_pw0;
 412                 case 32: return port_in_ah_dma_size32_hash0_lb0_pw0;
 413                 case 40: return port_in_ah_dma_size40_hash0_lb0_pw0;
 414                 case 48: return port_in_ah_dma_size48_hash0_lb0_pw0;
 415                 case 56: return port_in_ah_dma_size56_hash0_lb0_pw0;
 416                 case 64: return port_in_ah_dma_size64_hash0_lb0_pw0;
 417                 default: return NULL;
 418                 }
 419 }
 420
 421 int
 422 pipeline_passthrough_parse_args(struct pipeline_passthrough_params *p,
 423         struct pipeline_params *params)
 424 {
 425         uint32_t dma_dst_offset_present = 0;
 426         uint32_t dma_src_offset_present = 0;
 427         uint32_t dma_src_mask_present = 0;
 428         char dma_mask_str[PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2 + 1];
 429         uint32_t dma_size_present = 0;
 430         uint32_t dma_hash_offset_present = 0;
 431         uint32_t dma_hash_lb_present = 0;
 432         uint32_t i;
 433
 434         /* default values */
 435         p->dma_enabled = 0;
 436         p->dma_hash_enabled = 0;
 437         p->dma_hash_lb_enabled = 0;
 438         memset(p->dma_src_mask, 0xFF, sizeof(p->dma_src_mask));
 439         p->swap_enabled = 0;
 440         p->swap_n_fields = 0;
 441
 442         for (i = 0; i < params->n_args; i++) {
 443                 char *arg_name = params->args_name[i];
 444                 char *arg_value = params->args_value[i];
 445
 446                 /* dma_dst_offset */
 447                 if (strcmp(arg_name, "dma_dst_offset") == 0) {
 448                         int status;
 449
 450                         PIPELINE_PARSE_ERR_DUPLICATE(
 451                                 dma_dst_offset_present == 0, params->name,
 452                                 arg_name);
 453                         dma_dst_offset_present = 1;
 454
 455                         status = parser_read_uint32(&p->dma_dst_offset,
 456                                 arg_value);
 457                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 458                                 params->name, arg_name, arg_value);
 459                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 460                                 params->name, arg_name, arg_value);
 461
 462                         p->dma_enabled = 1;
 463
 464                         continue;
 465                 }
 466
 467                 /* dma_src_offset */
 468                 if (strcmp(arg_name, "dma_src_offset") == 0) {
 469                         int status;
 470
 471                         PIPELINE_PARSE_ERR_DUPLICATE(
 472                                 dma_src_offset_present == 0, params->name,
 473                                 arg_name);
 474                         dma_src_offset_present = 1;
 475
 476                         status = parser_read_uint32(&p->dma_src_offset,
 477                                 arg_value);
 478                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 479                                 params->name, arg_name, arg_value);
 480                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 481                                 params->name, arg_name, arg_value);
 482
 483                         p->dma_enabled = 1;
 484
 485                         continue;
 486                 }
 487
 488                 /* dma_size */
 489                 if (strcmp(arg_name, "dma_size") == 0) {
 490                         int status;
 491
 492                         PIPELINE_PARSE_ERR_DUPLICATE(
 493                                 dma_size_present == 0, params->name,
 494                                 arg_name);
 495                         dma_size_present = 1;
 496
 497                         status = parser_read_uint32(&p->dma_size,
 498                                 arg_value);
 499                         PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
 500                                 (p->dma_size != 0) &&
 501                                 ((p->dma_size % 8) == 0)),
 502                                 params->name, arg_name, arg_value);
 503                         PIPELINE_PARSE_ERR_OUT_RNG(((status != -ERANGE) &&
 504                                 (p->dma_size <=
 505                                 PIPELINE_PASSTHROUGH_DMA_SIZE_MAX)),
 506                                 params->name, arg_name, arg_value);
 507
 508                         p->dma_enabled = 1;
 509
 510                         continue;
 511                 }
 512
 513                 /* dma_src_mask */
 514                 if (strcmp(arg_name, "dma_src_mask") == 0) {
 515                         int mask_str_len = strlen(arg_value);
 516
 517                         PIPELINE_PARSE_ERR_DUPLICATE(
 518                                 dma_src_mask_present == 0,
 519                                 params->name, arg_name);
 520                         dma_src_mask_present = 1;
 521
 522                         PIPELINE_ARG_CHECK((mask_str_len <=
 523                                 (PIPELINE_PASSTHROUGH_DMA_SIZE_MAX * 2)),
 524                                 "Parse error in section \"%s\": entry "
 525                                 "\"%s\" too long", params->name,
 526                                 arg_name);
 527
 528                         strlcpy(dma_mask_str, arg_value, mask_str_len + 1);
 529
 530                         p->dma_enabled = 1;
 531
 532                         continue;
 533                 }
 534
 535                 /* dma_hash_offset */
 536                 if (strcmp(arg_name, "dma_hash_offset") == 0) {
 537                         int status;
 538
 539                         PIPELINE_PARSE_ERR_DUPLICATE(
 540                                 dma_hash_offset_present == 0,
 541                                 params->name, arg_name);
 542                         dma_hash_offset_present = 1;
 543
 544                         status = parser_read_uint32(&p->dma_hash_offset,
 545                                 arg_value);
 546                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
 547                                 params->name, arg_name, arg_value);
 548                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 549                                 params->name, arg_name, arg_value);
 550
 551                         p->dma_hash_enabled = 1;
 552
 553                         continue;
 554                 }
 555
 556                 /* load_balance mode */
 557                 if (strcmp(arg_name, "lb") == 0) {
 558                         PIPELINE_PARSE_ERR_DUPLICATE(
 559                                 dma_hash_lb_present == 0,
 560                                 params->name, arg_name);
 561                         dma_hash_lb_present = 1;
 562
 563                         if (strcmp(arg_value, "hash") &&
 564                                 strcmp(arg_value, "HASH"))
 565
 566                                 PIPELINE_PARSE_ERR_INV_VAL(0,
 567                                         params->name,
 568                                         arg_name,
 569                                         arg_value);
 570
 571                         p->dma_hash_lb_enabled = 1;
 572
 573                         continue;
 574                 }
 575
 576                 /* swap */
 577                 if (strcmp(arg_name, "swap") == 0) {
 578                         uint32_t a, b, n_args;
 579                         int len;
 580
 581                         n_args = sscanf(arg_value, "%" SCNu32 " %" SCNu32 "%n",
 582                                 &a, &b, &len);
 583                         PIPELINE_PARSE_ERR_INV_VAL(((n_args == 2) &&
 584                                 ((size_t) len == strlen(arg_value))),
 585                                 params->name, arg_name, arg_value);
 586
 587                         p->swap_field0_offset[p->swap_n_fields] = a;
 588                         p->swap_field1_offset[p->swap_n_fields] = b;
 589                         p->swap_n_fields++;
 590                         p->swap_enabled = 1;
 591
 592                         continue;
 593                 }
 594
 595                 /* any other */
 596                 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
 597         }
 598
 599         /* Check correlations between arguments */
 600         PIPELINE_ARG_CHECK((p->dma_enabled + p->swap_enabled < 2),
 601                 "Parse error in section \"%s\": DMA and SWAP actions are both enabled",
 602                 params->name);
 603         PIPELINE_ARG_CHECK((dma_dst_offset_present == p->dma_enabled),
 604                 "Parse error in section \"%s\": missing entry "
 605                 "\"dma_dst_offset\"", params->name);
 606         PIPELINE_ARG_CHECK((dma_src_offset_present == p->dma_enabled),
 607                 "Parse error in section \"%s\": missing entry "
 608                 "\"dma_src_offset\"", params->name);
 609         PIPELINE_ARG_CHECK((dma_size_present == p->dma_enabled),
 610                 "Parse error in section \"%s\": missing entry "
 611                 "\"dma_size\"", params->name);
 612         PIPELINE_ARG_CHECK((p->dma_hash_enabled <= p->dma_enabled),
 613                 "Parse error in section \"%s\": missing all DMA entries",
 614                 params->name);
 615         PIPELINE_ARG_CHECK((p->dma_hash_lb_enabled <= p->dma_hash_enabled),
 616                 "Parse error in section \"%s\": missing all DMA hash entries ",
 617                 params->name);
 618
 619         if (dma_src_mask_present) {
 620                 uint32_t dma_size = p->dma_size;
 621                 int status;
 622
 623                 PIPELINE_ARG_CHECK((strlen(dma_mask_str) ==
 624                         (dma_size * 2)), "Parse error in section "
 625                         "\"%s\": dma_src_mask should have exactly %u hex "
 626                         "digits", params->name, (dma_size * 2));
 627
 628                 status = parse_hex_string(dma_mask_str, p->dma_src_mask,
 629                         &p->dma_size);
 630
 631                 PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
 632                         (dma_size == p->dma_size)), params->name,
 633                         "dma_src_mask", dma_mask_str);
 634         }
 635
 636         if (p->dma_hash_lb_enabled)
 637                 PIPELINE_ARG_CHECK((params->n_ports_out > 1),
 638                         "Parse error in section \"%s\": entry \"lb\" not "
 639                         "allowed for single output port pipeline",
 640                         params->name);
 641         else
 642                 PIPELINE_ARG_CHECK(((params->n_ports_in >= params->n_ports_out)
 643                         && ((params->n_ports_in % params->n_ports_out) == 0)),
 644                         "Parse error in section \"%s\": n_ports_in needs to be "
 645                         "a multiple of n_ports_out (lb mode disabled)",
 646                         params->name);
 647
 648         return 0;
 649 }
 650
 651 static rte_table_hash_op_hash
 652 get_hash_function(struct pipeline_passthrough *p)
 653 {
 654         switch (p->params.dma_size) {
 655
 656         case 8: return hash_default_key8;
 657         case 16: return hash_default_key16;
 658         case 24: return hash_default_key24;
 659         case 32: return hash_default_key32;
 660         case 40: return hash_default_key40;
 661         case 48: return hash_default_key48;
 662         case 56: return hash_default_key56;
 663         case 64: return hash_default_key64;
 664         default: return NULL;
 665         }
 666 }
 667
 668 static int
 669 pipeline_passthrough_swap_convert(struct pipeline_passthrough *p)
 670 {
 671         uint32_t i;
 672
 673         p->swap_n_fields = 0;
 674
 675         for (i = 0; i < p->params.swap_n_fields; i++) {
 676                 uint32_t offset0 = p->params.swap_field0_offset[i];
 677                 uint32_t offset1 = p->params.swap_field1_offset[i];
 678                 uint32_t size = offset1 - offset0;
 679                 uint32_t j;
 680
 681                 /* Check */
 682                 if ((offset0 >= offset1) ||
 683                         (size > PIPELINE_PASSTHROUGH_SWAP_FIELD_SIZE_MAX) ||
 684                         (p->swap_n_fields >= SWAP_DIM))
 685                         return -1;
 686
 687                 for (j = 0; j < (size / sizeof(uint64_t)); j++) {
 688                         p->swap_field0_offset[p->swap_n_fields] = offset0;
 689                         p->swap_field1_offset[p->swap_n_fields] = offset1;
 690                         p->swap_field_mask[p->swap_n_fields] = UINT64_MAX;
 691                         p->swap_n_fields++;
 692                         offset0 += sizeof(uint64_t);
 693                         offset1 += sizeof(uint64_t);
 694                 }
 695                 if (size % sizeof(uint64_t)) {
 696                         uint32_t n_bits = (size % sizeof(uint64_t)) * 8;
 697
 698                         p->swap_field0_offset[p->swap_n_fields] = offset0;
 699                         p->swap_field1_offset[p->swap_n_fields] = offset1;
 700                         p->swap_field_mask[p->swap_n_fields] =
 701                                 RTE_LEN2MASK(n_bits, uint64_t);
 702                         p->swap_n_fields++;
 703                 }
 704         }
 705
 706         return 0;
 707 }
 708
 709 static void*
 710 pipeline_passthrough_init(struct pipeline_params *params,
 711         __rte_unused void *arg)
 712 {
 713         struct pipeline *p;
 714         struct pipeline_passthrough *p_pt;
 715         uint32_t size, i;
 716
 717         /* Check input arguments */
 718         if ((params == NULL) ||
 719                 (params->n_ports_in == 0) ||
 720                 (params->n_ports_out == 0))
 721                 return NULL;
 722
 723         /* Memory allocation */
 724         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_passthrough));
 725         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
 726         p_pt = (struct pipeline_passthrough *) p;
 727         if (p == NULL)
 728                 return NULL;
 729
 730         strcpy(p->name, params->name);
 731         p->log_level = params->log_level;
 732
 733         PLOG(p, HIGH, "Pass-through");
 734
 735         /* Parse arguments */
 736         if (pipeline_passthrough_parse_args(&p_pt->params, params))
 737                 return NULL;
 738         if (pipeline_passthrough_swap_convert(p_pt))
 739                 return NULL;
 740         p_pt->f_hash = get_hash_function(p_pt);
 741
 742         /* Pipeline */
 743         {
 744                 struct rte_pipeline_params pipeline_params = {
 745                         .name = "PASS-THROUGH",
 746                         .socket_id = params->socket_id,
 747                         .offset_port_id = 0,
 748                 };
 749
 750                 p->p = rte_pipeline_create(&pipeline_params);
 751                 if (p->p == NULL) {
 752                         rte_free(p);
 753                         return NULL;
 754                 }
 755         }
 756
 757         p->n_ports_in = params->n_ports_in;
 758         p->n_ports_out = params->n_ports_out;
 759         p->n_tables = p->n_ports_in;
 760
 761         /*Input ports*/
 762         for (i = 0; i < p->n_ports_in; i++) {
 763                 struct rte_pipeline_port_in_params port_params = {
 764                         .ops = pipeline_port_in_params_get_ops(
 765                                 &params->port_in[i]),
 766                         .arg_create = pipeline_port_in_params_convert(
 767                                 &params->port_in[i]),
 768                         .f_action = get_port_in_ah(p_pt),
 769                         .arg_ah = p_pt,
 770                         .burst_size = params->port_in[i].burst_size,
 771                 };
 772
 773                 int status = rte_pipeline_port_in_create(p->p,
 774                         &port_params,
 775                         &p->port_in_id[i]);
 776
 777                 if (status) {
 778                         rte_pipeline_free(p->p);
 779                         rte_free(p);
 780                         return NULL;
 781                 }
 782         }
 783
 784         /* Output ports */
 785         for (i = 0; i < p->n_ports_out; i++) {
 786                 struct rte_pipeline_port_out_params port_params = {
 787                         .ops = pipeline_port_out_params_get_ops(
 788                                 &params->port_out[i]),
 789                         .arg_create = pipeline_port_out_params_convert(
 790                                 &params->port_out[i]),
 791                         .f_action = NULL,
 792                         .arg_ah = NULL,
 793                 };
 794
 795                 int status = rte_pipeline_port_out_create(p->p,
 796                         &port_params,
 797                         &p->port_out_id[i]);
 798
 799                 if (status) {
 800                         rte_pipeline_free(p->p);
 801                         rte_free(p);
 802                         return NULL;
 803                 }
 804         }
 805
 806         /* Tables */
 807         for (i = 0; i < p->n_ports_in; i++) {
 808                 struct rte_pipeline_table_params table_params = {
 809                         .ops = &rte_table_stub_ops,
 810                         .arg_create = NULL,
 811                         .f_action_hit = NULL,
 812                         .f_action_miss = NULL,
 813                         .arg_ah = NULL,
 814                         .action_data_size = 0,
 815                 };
 816
 817                 int status = rte_pipeline_table_create(p->p,
 818                         &table_params,
 819                         &p->table_id[i]);
 820
 821                 if (status) {
 822                         rte_pipeline_free(p->p);
 823                         rte_free(p);
 824                         return NULL;
 825                 }
 826         }
 827
 828         /* Connecting input ports to tables */
 829         for (i = 0; i < p->n_ports_in; i++) {
 830                 int status = rte_pipeline_port_in_connect_to_table(p->p,
 831                         p->port_in_id[i],
 832                         p->table_id[i]);
 833
 834                 if (status) {
 835                         rte_pipeline_free(p->p);
 836                         rte_free(p);
 837                         return NULL;
 838                 }
 839         }
 840
 841         /* Add entries to tables */
 842         for (i = 0; i < p->n_ports_in; i++) {
 843                 uint32_t port_out_id = (p_pt->params.dma_hash_lb_enabled == 0) ?
 844                         (i / (p->n_ports_in / p->n_ports_out)) :
 845                         0;
 846
 847                 struct rte_pipeline_table_entry default_entry = {
 848                         .action = RTE_PIPELINE_ACTION_PORT,
 849                         {.port_id = p->port_out_id[port_out_id]},
 850                 };
 851
 852                 struct rte_pipeline_table_entry *default_entry_ptr;
 853
 854                 int status = rte_pipeline_table_default_entry_add(p->p,
 855                         p->table_id[i],
 856                         &default_entry,
 857                         &default_entry_ptr);
 858
 859                 if (status) {
 860                         rte_pipeline_free(p->p);
 861                         rte_free(p);
 862                         return NULL;
 863                 }
 864         }
 865
 866         /* Enable input ports */
 867         for (i = 0; i < p->n_ports_in; i++) {
 868                 int status = rte_pipeline_port_in_enable(p->p,
 869                         p->port_in_id[i]);
 870
 871                 if (status) {
 872                         rte_pipeline_free(p->p);
 873                         rte_free(p);
 874                         return NULL;
 875                 }
 876         }
 877
 878         /* Check pipeline consistency */
 879         if (rte_pipeline_check(p->p) < 0) {
 880                 rte_pipeline_free(p->p);
 881                 rte_free(p);
 882                 return NULL;
 883         }
 884
 885         /* Message queues */
 886         p->n_msgq = params->n_msgq;
 887         for (i = 0; i < p->n_msgq; i++)
 888                 p->msgq_in[i] = params->msgq_in[i];
 889         for (i = 0; i < p->n_msgq; i++)
 890                 p->msgq_out[i] = params->msgq_out[i];
 891
 892         /* Message handlers */
 893         memcpy(p->handlers, handlers, sizeof(p->handlers));
 894
 895         return p;
 896 }
 897
 898 static int
 899 pipeline_passthrough_free(void *pipeline)
 900 {
 901         struct pipeline *p = (struct pipeline *) pipeline;
 902
 903         /* Check input arguments */
 904         if (p == NULL)
 905                 return -1;
 906
 907         /* Free resources */
 908         rte_pipeline_free(p->p);
 909         rte_free(p);
 910         return 0;
 911 }
 912
 913 static int
 914 pipeline_passthrough_timer(void *pipeline)
 915 {
 916         struct pipeline *p = (struct pipeline *) pipeline;
 917
 918         pipeline_msg_req_handle(p);
 919         rte_pipeline_flush(p->p);
 920
 921         return 0;
 922 }
 923
 924 struct pipeline_be_ops pipeline_passthrough_be_ops = {
 925         .f_init = pipeline_passthrough_init,
 926         .f_free = pipeline_passthrough_free,
 927         .f_run = NULL,
 928         .f_timer = pipeline_passthrough_timer,
 929 };