lib/librte_pipeline/rte_table_action.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdlib.h>
   6 #include <string.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_byteorder.h>
  10 #include <rte_cycles.h>
  11 #include <rte_malloc.h>
  12 #include <rte_memcpy.h>
  13 #include <rte_ether.h>
  14 #include <rte_ip.h>
  15 #include <rte_esp.h>
  16 #include <rte_tcp.h>
  17 #include <rte_udp.h>
  18
  19 #include "rte_table_action.h"
  20
  21 #define rte_htons rte_cpu_to_be_16
  22 #define rte_htonl rte_cpu_to_be_32
  23
  24 #define rte_ntohs rte_be_to_cpu_16
  25 #define rte_ntohl rte_be_to_cpu_32
  26
  27 /**
  28  * RTE_TABLE_ACTION_FWD
  29  */
  30 #define fwd_data rte_pipeline_table_entry
  31
  32 static int
  33 fwd_apply(struct fwd_data *data,
  34         struct rte_table_action_fwd_params *p)
  35 {
  36         data->action = p->action;
  37
  38         if (p->action == RTE_PIPELINE_ACTION_PORT)
  39                 data->port_id = p->id;
  40
  41         if (p->action == RTE_PIPELINE_ACTION_TABLE)
  42                 data->table_id = p->id;
  43
  44         return 0;
  45 }
  46
  47 /**
  48  * RTE_TABLE_ACTION_MTR
  49  */
  50 static int
  51 mtr_cfg_check(struct rte_table_action_mtr_config *mtr)
  52 {
  53         if ((mtr->alg == RTE_TABLE_ACTION_METER_SRTCM) ||
  54                 ((mtr->n_tc != 1) && (mtr->n_tc != 4)) ||
  55                 (mtr->n_bytes_enabled != 0))
  56                 return -ENOTSUP;
  57         return 0;
  58 }
  59
  60 #define MBUF_SCHED_QUEUE_TC_COLOR(queue, tc, color)        \
  61         ((uint16_t)((((uint64_t)(queue)) & 0x3) |          \
  62         ((((uint64_t)(tc)) & 0x3) << 2) |                  \
  63         ((((uint64_t)(color)) & 0x3) << 4)))
  64
  65 #define MBUF_SCHED_COLOR(sched, color)                     \
  66         (((sched) & (~0x30LLU)) | ((color) << 4))
  67
  68 struct mtr_trtcm_data {
  69         struct rte_meter_trtcm trtcm;
  70         uint64_t stats[e_RTE_METER_COLORS];
  71 } __attribute__((__packed__));
  72
  73 #define MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data)          \
  74         (((data)->stats[e_RTE_METER_GREEN] & 0xF8LLU) >> 3)
  75
  76 static void
  77 mtr_trtcm_data_meter_profile_id_set(struct mtr_trtcm_data *data,
  78         uint32_t profile_id)
  79 {
  80         data->stats[e_RTE_METER_GREEN] &= ~0xF8LLU;
  81         data->stats[e_RTE_METER_GREEN] |= (profile_id % 32) << 3;
  82 }
  83
  84 #define MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color)\
  85         (((data)->stats[(color)] & 4LLU) >> 2)
  86
  87 #define MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color)\
  88         ((enum rte_meter_color)((data)->stats[(color)] & 3LLU))
  89
  90 static void
  91 mtr_trtcm_data_policer_action_set(struct mtr_trtcm_data *data,
  92         enum rte_meter_color color,
  93         enum rte_table_action_policer action)
  94 {
  95         if (action == RTE_TABLE_ACTION_POLICER_DROP) {
  96                 data->stats[color] |= 4LLU;
  97         } else {
  98                 data->stats[color] &= ~7LLU;
  99                 data->stats[color] |= color & 3LLU;
 100         }
 101 }
 102
 103 static uint64_t
 104 mtr_trtcm_data_stats_get(struct mtr_trtcm_data *data,
 105         enum rte_meter_color color)
 106 {
 107         return data->stats[color] >> 8;
 108 }
 109
 110 static void
 111 mtr_trtcm_data_stats_reset(struct mtr_trtcm_data *data,
 112         enum rte_meter_color color)
 113 {
 114         data->stats[color] &= 0xFFLU;
 115 }
 116
 117 #define MTR_TRTCM_DATA_STATS_INC(data, color)              \
 118         ((data)->stats[(color)] += (1LLU << 8))
 119
 120 static size_t
 121 mtr_data_size(struct rte_table_action_mtr_config *mtr)
 122 {
 123         return mtr->n_tc * sizeof(struct mtr_trtcm_data);
 124 }
 125
 126 struct dscp_table_entry_data {
 127         enum rte_meter_color color;
 128         uint16_t tc;
 129         uint16_t queue_tc_color;
 130 };
 131
 132 struct dscp_table_data {
 133         struct dscp_table_entry_data entry[64];
 134 };
 135
 136 struct meter_profile_data {
 137         struct rte_meter_trtcm_profile profile;
 138         uint32_t profile_id;
 139         int valid;
 140 };
 141
 142 static struct meter_profile_data *
 143 meter_profile_data_find(struct meter_profile_data *mp,
 144         uint32_t mp_size,
 145         uint32_t profile_id)
 146 {
 147         uint32_t i;
 148
 149         for (i = 0; i < mp_size; i++) {
 150                 struct meter_profile_data *mp_data = &mp[i];
 151
 152                 if (mp_data->valid && (mp_data->profile_id == profile_id))
 153                         return mp_data;
 154         }
 155
 156         return NULL;
 157 }
 158
 159 static struct meter_profile_data *
 160 meter_profile_data_find_unused(struct meter_profile_data *mp,
 161         uint32_t mp_size)
 162 {
 163         uint32_t i;
 164
 165         for (i = 0; i < mp_size; i++) {
 166                 struct meter_profile_data *mp_data = &mp[i];
 167
 168                 if (!mp_data->valid)
 169                         return mp_data;
 170         }
 171
 172         return NULL;
 173 }
 174
 175 static int
 176 mtr_apply_check(struct rte_table_action_mtr_params *p,
 177         struct rte_table_action_mtr_config *cfg,
 178         struct meter_profile_data *mp,
 179         uint32_t mp_size)
 180 {
 181         uint32_t i;
 182
 183         if (p->tc_mask > RTE_LEN2MASK(cfg->n_tc, uint32_t))
 184                 return -EINVAL;
 185
 186         for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) {
 187                 struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i];
 188                 struct meter_profile_data *mp_data;
 189
 190                 if ((p->tc_mask & (1LLU << i)) == 0)
 191                         continue;
 192
 193                 mp_data = meter_profile_data_find(mp,
 194                         mp_size,
 195                         p_tc->meter_profile_id);
 196                 if (!mp_data)
 197                         return -EINVAL;
 198         }
 199
 200         return 0;
 201 }
 202
 203 static int
 204 mtr_apply(struct mtr_trtcm_data *data,
 205         struct rte_table_action_mtr_params *p,
 206         struct rte_table_action_mtr_config *cfg,
 207         struct meter_profile_data *mp,
 208         uint32_t mp_size)
 209 {
 210         uint32_t i;
 211         int status;
 212
 213         /* Check input arguments */
 214         status = mtr_apply_check(p, cfg, mp, mp_size);
 215         if (status)
 216                 return status;
 217
 218         /* Apply */
 219         for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) {
 220                 struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i];
 221                 struct mtr_trtcm_data *data_tc = &data[i];
 222                 struct meter_profile_data *mp_data;
 223
 224                 if ((p->tc_mask & (1LLU << i)) == 0)
 225                         continue;
 226
 227                 /* Find profile */
 228                 mp_data = meter_profile_data_find(mp,
 229                         mp_size,
 230                         p_tc->meter_profile_id);
 231                 if (!mp_data)
 232                         return -EINVAL;
 233
 234                 memset(data_tc, 0, sizeof(*data_tc));
 235
 236                 /* Meter object */
 237                 status = rte_meter_trtcm_config(&data_tc->trtcm,
 238                         &mp_data->profile);
 239                 if (status)
 240                         return status;
 241
 242                 /* Meter profile */
 243                 mtr_trtcm_data_meter_profile_id_set(data_tc,
 244                         mp_data - mp);
 245
 246                 /* Policer actions */
 247                 mtr_trtcm_data_policer_action_set(data_tc,
 248                         e_RTE_METER_GREEN,
 249                         p_tc->policer[e_RTE_METER_GREEN]);
 250
 251                 mtr_trtcm_data_policer_action_set(data_tc,
 252                         e_RTE_METER_YELLOW,
 253                         p_tc->policer[e_RTE_METER_YELLOW]);
 254
 255                 mtr_trtcm_data_policer_action_set(data_tc,
 256                         e_RTE_METER_RED,
 257                         p_tc->policer[e_RTE_METER_RED]);
 258         }
 259
 260         return 0;
 261 }
 262
 263 static __rte_always_inline uint64_t
 264 pkt_work_mtr(struct rte_mbuf *mbuf,
 265         struct mtr_trtcm_data *data,
 266         struct dscp_table_data *dscp_table,
 267         struct meter_profile_data *mp,
 268         uint64_t time,
 269         uint32_t dscp,
 270         uint16_t total_length)
 271 {
 272         uint64_t drop_mask, sched;
 273         uint64_t *sched_ptr = (uint64_t *) &mbuf->hash.sched;
 274         struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp];
 275         enum rte_meter_color color_in, color_meter, color_policer;
 276         uint32_t tc, mp_id;
 277
 278         tc = dscp_entry->tc;
 279         color_in = dscp_entry->color;
 280         data += tc;
 281         mp_id = MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data);
 282         sched = *sched_ptr;
 283
 284         /* Meter */
 285         color_meter = rte_meter_trtcm_color_aware_check(
 286                 &data->trtcm,
 287                 &mp[mp_id].profile,
 288                 time,
 289                 total_length,
 290                 color_in);
 291
 292         /* Stats */
 293         MTR_TRTCM_DATA_STATS_INC(data, color_meter);
 294
 295         /* Police */
 296         drop_mask = MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color_meter);
 297         color_policer =
 298                 MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color_meter);
 299         *sched_ptr = MBUF_SCHED_COLOR(sched, color_policer);
 300
 301         return drop_mask;
 302 }
 303
 304 /**
 305  * RTE_TABLE_ACTION_TM
 306  */
 307 static int
 308 tm_cfg_check(struct rte_table_action_tm_config *tm)
 309 {
 310         if ((tm->n_subports_per_port == 0) ||
 311                 (rte_is_power_of_2(tm->n_subports_per_port) == 0) ||
 312                 (tm->n_subports_per_port > UINT16_MAX) ||
 313                 (tm->n_pipes_per_subport == 0) ||
 314                 (rte_is_power_of_2(tm->n_pipes_per_subport) == 0))
 315                 return -ENOTSUP;
 316
 317         return 0;
 318 }
 319
 320 struct tm_data {
 321         uint16_t queue_tc_color;
 322         uint16_t subport;
 323         uint32_t pipe;
 324 } __attribute__((__packed__));
 325
 326 static int
 327 tm_apply_check(struct rte_table_action_tm_params *p,
 328         struct rte_table_action_tm_config *cfg)
 329 {
 330         if ((p->subport_id >= cfg->n_subports_per_port) ||
 331                 (p->pipe_id >= cfg->n_pipes_per_subport))
 332                 return -EINVAL;
 333
 334         return 0;
 335 }
 336
 337 static int
 338 tm_apply(struct tm_data *data,
 339         struct rte_table_action_tm_params *p,
 340         struct rte_table_action_tm_config *cfg)
 341 {
 342         int status;
 343
 344         /* Check input arguments */
 345         status = tm_apply_check(p, cfg);
 346         if (status)
 347                 return status;
 348
 349         /* Apply */
 350         data->queue_tc_color = 0;
 351         data->subport = (uint16_t) p->subport_id;
 352         data->pipe = p->pipe_id;
 353
 354         return 0;
 355 }
 356
 357 static __rte_always_inline void
 358 pkt_work_tm(struct rte_mbuf *mbuf,
 359         struct tm_data *data,
 360         struct dscp_table_data *dscp_table,
 361         uint32_t dscp)
 362 {
 363         struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp];
 364         struct tm_data *sched_ptr = (struct tm_data *) &mbuf->hash.sched;
 365         struct tm_data sched;
 366
 367         sched = *data;
 368         sched.queue_tc_color = dscp_entry->queue_tc_color;
 369         *sched_ptr = sched;
 370 }
 371
 372 /**
 373  * RTE_TABLE_ACTION_ENCAP
 374  */
 375 static int
 376 encap_valid(enum rte_table_action_encap_type encap)
 377 {
 378         switch (encap) {
 379         case RTE_TABLE_ACTION_ENCAP_ETHER:
 380         case RTE_TABLE_ACTION_ENCAP_VLAN:
 381         case RTE_TABLE_ACTION_ENCAP_QINQ:
 382         case RTE_TABLE_ACTION_ENCAP_MPLS:
 383         case RTE_TABLE_ACTION_ENCAP_PPPOE:
 384                 return 1;
 385         default:
 386                 return 0;
 387         }
 388 }
 389
 390 static int
 391 encap_cfg_check(struct rte_table_action_encap_config *encap)
 392 {
 393         if ((encap->encap_mask == 0) ||
 394                 (__builtin_popcountll(encap->encap_mask) != 1))
 395                 return -ENOTSUP;
 396
 397         return 0;
 398 }
 399
 400 struct encap_ether_data {
 401         struct ether_hdr ether;
 402 } __attribute__((__packed__));
 403
 404 #define VLAN(pcp, dei, vid)                                \
 405         ((uint16_t)((((uint64_t)(pcp)) & 0x7LLU) << 13) |  \
 406         ((((uint64_t)(dei)) & 0x1LLU) << 12) |             \
 407         (((uint64_t)(vid)) & 0xFFFLLU))                    \
 408
 409 struct encap_vlan_data {
 410         struct ether_hdr ether;
 411         struct vlan_hdr vlan;
 412 } __attribute__((__packed__));
 413
 414 struct encap_qinq_data {
 415         struct ether_hdr ether;
 416         struct vlan_hdr svlan;
 417         struct vlan_hdr cvlan;
 418 } __attribute__((__packed__));
 419
 420 #define ETHER_TYPE_MPLS_UNICAST                            0x8847
 421
 422 #define ETHER_TYPE_MPLS_MULTICAST                          0x8848
 423
 424 #define MPLS(label, tc, s, ttl)                            \
 425         ((uint32_t)(((((uint64_t)(label)) & 0xFFFFFLLU) << 12) |\
 426         ((((uint64_t)(tc)) & 0x7LLU) << 9) |               \
 427         ((((uint64_t)(s)) & 0x1LLU) << 8) |                \
 428         (((uint64_t)(ttl)) & 0xFFLLU)))
 429
 430 struct encap_mpls_data {
 431         struct ether_hdr ether;
 432         uint32_t mpls[RTE_TABLE_ACTION_MPLS_LABELS_MAX];
 433         uint32_t mpls_count;
 434 } __attribute__((__packed__));
 435
 436 #define ETHER_TYPE_PPPOE_SESSION                           0x8864
 437
 438 #define PPP_PROTOCOL_IP                                    0x0021
 439
 440 struct pppoe_ppp_hdr {
 441         uint16_t ver_type_code;
 442         uint16_t session_id;
 443         uint16_t length;
 444         uint16_t protocol;
 445 } __attribute__((__packed__));
 446
 447 struct encap_pppoe_data {
 448         struct ether_hdr ether;
 449         struct pppoe_ppp_hdr pppoe_ppp;
 450 } __attribute__((__packed__));
 451
 452 static size_t
 453 encap_data_size(struct rte_table_action_encap_config *encap)
 454 {
 455         switch (encap->encap_mask) {
 456         case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER:
 457                 return sizeof(struct encap_ether_data);
 458
 459         case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN:
 460                 return sizeof(struct encap_vlan_data);
 461
 462         case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ:
 463                 return sizeof(struct encap_qinq_data);
 464
 465         case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS:
 466                 return sizeof(struct encap_mpls_data);
 467
 468         case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE:
 469                 return sizeof(struct encap_pppoe_data);
 470
 471         default:
 472                 return 0;
 473         }
 474 }
 475
 476 static int
 477 encap_apply_check(struct rte_table_action_encap_params *p,
 478         struct rte_table_action_encap_config *cfg)
 479 {
 480         if ((encap_valid(p->type) == 0) ||
 481                 ((cfg->encap_mask & (1LLU << p->type)) == 0))
 482                 return -EINVAL;
 483
 484         switch (p->type) {
 485         case RTE_TABLE_ACTION_ENCAP_ETHER:
 486                 return 0;
 487
 488         case RTE_TABLE_ACTION_ENCAP_VLAN:
 489                 return 0;
 490
 491         case RTE_TABLE_ACTION_ENCAP_QINQ:
 492                 return 0;
 493
 494         case RTE_TABLE_ACTION_ENCAP_MPLS:
 495                 if ((p->mpls.mpls_count == 0) ||
 496                         (p->mpls.mpls_count > RTE_TABLE_ACTION_MPLS_LABELS_MAX))
 497                         return -EINVAL;
 498
 499                 return 0;
 500
 501         case RTE_TABLE_ACTION_ENCAP_PPPOE:
 502                 return 0;
 503
 504         default:
 505                 return -EINVAL;
 506         }
 507 }
 508
 509 static int
 510 encap_ether_apply(void *data,
 511         struct rte_table_action_encap_params *p,
 512         struct rte_table_action_common_config *common_cfg)
 513 {
 514         struct encap_ether_data *d = data;
 515         uint16_t ethertype = (common_cfg->ip_version) ?
 516                 ETHER_TYPE_IPv4 :
 517                 ETHER_TYPE_IPv6;
 518
 519         /* Ethernet */
 520         ether_addr_copy(&p->ether.ether.da, &d->ether.d_addr);
 521         ether_addr_copy(&p->ether.ether.sa, &d->ether.s_addr);
 522         d->ether.ether_type = rte_htons(ethertype);
 523
 524         return 0;
 525 }
 526
 527 static int
 528 encap_vlan_apply(void *data,
 529         struct rte_table_action_encap_params *p,
 530         struct rte_table_action_common_config *common_cfg)
 531 {
 532         struct encap_vlan_data *d = data;
 533         uint16_t ethertype = (common_cfg->ip_version) ?
 534                 ETHER_TYPE_IPv4 :
 535                 ETHER_TYPE_IPv6;
 536
 537         /* Ethernet */
 538         ether_addr_copy(&p->vlan.ether.da, &d->ether.d_addr);
 539         ether_addr_copy(&p->vlan.ether.sa, &d->ether.s_addr);
 540         d->ether.ether_type = rte_htons(ETHER_TYPE_VLAN);
 541
 542         /* VLAN */
 543         d->vlan.vlan_tci = rte_htons(VLAN(p->vlan.vlan.pcp,
 544                 p->vlan.vlan.dei,
 545                 p->vlan.vlan.vid));
 546         d->vlan.eth_proto = rte_htons(ethertype);
 547
 548         return 0;
 549 }
 550
 551 static int
 552 encap_qinq_apply(void *data,
 553         struct rte_table_action_encap_params *p,
 554         struct rte_table_action_common_config *common_cfg)
 555 {
 556         struct encap_qinq_data *d = data;
 557         uint16_t ethertype = (common_cfg->ip_version) ?
 558                 ETHER_TYPE_IPv4 :
 559                 ETHER_TYPE_IPv6;
 560
 561         /* Ethernet */
 562         ether_addr_copy(&p->qinq.ether.da, &d->ether.d_addr);
 563         ether_addr_copy(&p->qinq.ether.sa, &d->ether.s_addr);
 564         d->ether.ether_type = rte_htons(ETHER_TYPE_QINQ);
 565
 566         /* SVLAN */
 567         d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp,
 568                 p->qinq.svlan.dei,
 569                 p->qinq.svlan.vid));
 570         d->svlan.eth_proto = rte_htons(ETHER_TYPE_VLAN);
 571
 572         /* CVLAN */
 573         d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp,
 574                 p->qinq.cvlan.dei,
 575                 p->qinq.cvlan.vid));
 576         d->cvlan.eth_proto = rte_htons(ethertype);
 577
 578         return 0;
 579 }
 580
 581 static int
 582 encap_mpls_apply(void *data,
 583         struct rte_table_action_encap_params *p)
 584 {
 585         struct encap_mpls_data *d = data;
 586         uint16_t ethertype = (p->mpls.unicast) ?
 587                 ETHER_TYPE_MPLS_UNICAST :
 588                 ETHER_TYPE_MPLS_MULTICAST;
 589         uint32_t i;
 590
 591         /* Ethernet */
 592         ether_addr_copy(&p->mpls.ether.da, &d->ether.d_addr);
 593         ether_addr_copy(&p->mpls.ether.sa, &d->ether.s_addr);
 594         d->ether.ether_type = rte_htons(ethertype);
 595
 596         /* MPLS */
 597         for (i = 0; i < p->mpls.mpls_count - 1; i++)
 598                 d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label,
 599                         p->mpls.mpls[i].tc,
 600                         0,
 601                         p->mpls.mpls[i].ttl));
 602
 603         d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label,
 604                 p->mpls.mpls[i].tc,
 605                 1,
 606                 p->mpls.mpls[i].ttl));
 607
 608         d->mpls_count = p->mpls.mpls_count;
 609         return 0;
 610 }
 611
 612 static int
 613 encap_pppoe_apply(void *data,
 614         struct rte_table_action_encap_params *p)
 615 {
 616         struct encap_pppoe_data *d = data;
 617
 618         /* Ethernet */
 619         ether_addr_copy(&p->pppoe.ether.da, &d->ether.d_addr);
 620         ether_addr_copy(&p->pppoe.ether.sa, &d->ether.s_addr);
 621         d->ether.ether_type = rte_htons(ETHER_TYPE_PPPOE_SESSION);
 622
 623         /* PPPoE and PPP*/
 624         d->pppoe_ppp.ver_type_code = rte_htons(0x1100);
 625         d->pppoe_ppp.session_id = rte_htons(p->pppoe.pppoe.session_id);
 626         d->pppoe_ppp.length = 0; /* not pre-computed */
 627         d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP);
 628
 629         return 0;
 630 }
 631
 632 static int
 633 encap_apply(void *data,
 634         struct rte_table_action_encap_params *p,
 635         struct rte_table_action_encap_config *cfg,
 636         struct rte_table_action_common_config *common_cfg)
 637 {
 638         int status;
 639
 640         /* Check input arguments */
 641         status = encap_apply_check(p, cfg);
 642         if (status)
 643                 return status;
 644
 645         switch (p->type) {
 646         case RTE_TABLE_ACTION_ENCAP_ETHER:
 647                 return encap_ether_apply(data, p, common_cfg);
 648
 649         case RTE_TABLE_ACTION_ENCAP_VLAN:
 650                 return encap_vlan_apply(data, p, common_cfg);
 651
 652         case RTE_TABLE_ACTION_ENCAP_QINQ:
 653                 return encap_qinq_apply(data, p, common_cfg);
 654
 655         case RTE_TABLE_ACTION_ENCAP_MPLS:
 656                 return encap_mpls_apply(data, p);
 657
 658         case RTE_TABLE_ACTION_ENCAP_PPPOE:
 659                 return encap_pppoe_apply(data, p);
 660
 661         default:
 662                 return -EINVAL;
 663         }
 664 }
 665
 666 static __rte_always_inline void *
 667 encap(void *dst, const void *src, size_t n)
 668 {
 669         dst = ((uint8_t *) dst) - n;
 670         return rte_memcpy(dst, src, n);
 671 }
 672
 673 static __rte_always_inline void
 674 pkt_work_encap(struct rte_mbuf *mbuf,
 675         void *data,
 676         struct rte_table_action_encap_config *cfg,
 677         void *ip,
 678         uint16_t total_length,
 679         uint32_t ip_offset)
 680 {
 681         switch (cfg->encap_mask) {
 682         case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER:
 683                 encap(ip, data, sizeof(struct encap_ether_data));
 684                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) +
 685                         sizeof(struct encap_ether_data));
 686                 mbuf->pkt_len = mbuf->data_len = total_length +
 687                         sizeof(struct encap_ether_data);
 688                 break;
 689
 690         case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN:
 691                 encap(ip, data, sizeof(struct encap_vlan_data));
 692                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) +
 693                         sizeof(struct encap_vlan_data));
 694                 mbuf->pkt_len = mbuf->data_len = total_length +
 695                         sizeof(struct encap_vlan_data);
 696                 break;
 697
 698         case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ:
 699                 encap(ip, data, sizeof(struct encap_qinq_data));
 700                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) +
 701                         sizeof(struct encap_qinq_data));
 702                 mbuf->pkt_len = mbuf->data_len = total_length +
 703                         sizeof(struct encap_qinq_data);
 704                 break;
 705
 706         case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS:
 707         {
 708                 struct encap_mpls_data *mpls = data;
 709                 size_t size = sizeof(struct ether_hdr) +
 710                         mpls->mpls_count * 4;
 711
 712                 encap(ip, data, size);
 713                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + size);
 714                 mbuf->pkt_len = mbuf->data_len = total_length + size;
 715                 break;
 716         }
 717
 718         case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE:
 719         {
 720                 struct encap_pppoe_data *pppoe =
 721                         encap(ip, data, sizeof(struct encap_pppoe_data));
 722                 pppoe->pppoe_ppp.length = rte_htons(total_length + 2);
 723                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) +
 724                         sizeof(struct encap_pppoe_data));
 725                 mbuf->pkt_len = mbuf->data_len = total_length +
 726                         sizeof(struct encap_pppoe_data);
 727                 break;
 728         }
 729
 730         default:
 731                 break;
 732         }
 733 }
 734
 735 /**
 736  * RTE_TABLE_ACTION_NAT
 737  */
 738 static int
 739 nat_cfg_check(struct rte_table_action_nat_config *nat)
 740 {
 741         if ((nat->proto != 0x06) &&
 742                 (nat->proto != 0x11))
 743                 return -ENOTSUP;
 744
 745         return 0;
 746 }
 747
 748 struct nat_ipv4_data {
 749         uint32_t addr;
 750         uint16_t port;
 751 } __attribute__((__packed__));
 752
 753 struct nat_ipv6_data {
 754         uint8_t addr[16];
 755         uint16_t port;
 756 } __attribute__((__packed__));
 757
 758 static size_t
 759 nat_data_size(struct rte_table_action_nat_config *nat __rte_unused,
 760         struct rte_table_action_common_config *common)
 761 {
 762         int ip_version = common->ip_version;
 763
 764         return (ip_version) ?
 765                 sizeof(struct nat_ipv4_data) :
 766                 sizeof(struct nat_ipv6_data);
 767 }
 768
 769 static int
 770 nat_apply_check(struct rte_table_action_nat_params *p,
 771         struct rte_table_action_common_config *cfg)
 772 {
 773         if ((p->ip_version && (cfg->ip_version == 0)) ||
 774                 ((p->ip_version == 0) && cfg->ip_version))
 775                 return -EINVAL;
 776
 777         return 0;
 778 }
 779
 780 static int
 781 nat_apply(void *data,
 782         struct rte_table_action_nat_params *p,
 783         struct rte_table_action_common_config *cfg)
 784 {
 785         int status;
 786
 787         /* Check input arguments */
 788         status = nat_apply_check(p, cfg);
 789         if (status)
 790                 return status;
 791
 792         /* Apply */
 793         if (p->ip_version) {
 794                 struct nat_ipv4_data *d = data;
 795
 796                 d->addr = rte_htonl(p->addr.ipv4);
 797                 d->port = rte_htons(p->port);
 798         } else {
 799                 struct nat_ipv6_data *d = data;
 800
 801                 memcpy(d->addr, p->addr.ipv6, sizeof(d->addr));
 802                 d->port = rte_htons(p->port);
 803         }
 804
 805         return 0;
 806 }
 807
 808 static __rte_always_inline uint16_t
 809 nat_ipv4_checksum_update(uint16_t cksum0,
 810         uint32_t ip0,
 811         uint32_t ip1)
 812 {
 813         int32_t cksum1;
 814
 815         cksum1 = cksum0;
 816         cksum1 = ~cksum1 & 0xFFFF;
 817
 818         /* Subtract ip0 (one's complement logic) */
 819         cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF);
 820         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 821         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 822
 823         /* Add ip1 (one's complement logic) */
 824         cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF);
 825         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 826         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 827
 828         return (uint16_t)(~cksum1);
 829 }
 830
 831 static __rte_always_inline uint16_t
 832 nat_ipv4_tcp_udp_checksum_update(uint16_t cksum0,
 833         uint32_t ip0,
 834         uint32_t ip1,
 835         uint16_t port0,
 836         uint16_t port1)
 837 {
 838         int32_t cksum1;
 839
 840         cksum1 = cksum0;
 841         cksum1 = ~cksum1 & 0xFFFF;
 842
 843         /* Subtract ip0 and port 0 (one's complement logic) */
 844         cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF) + port0;
 845         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 846         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 847
 848         /* Add ip1 and port1 (one's complement logic) */
 849         cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF) + port1;
 850         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 851         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 852
 853         return (uint16_t)(~cksum1);
 854 }
 855
 856 static __rte_always_inline uint16_t
 857 nat_ipv6_tcp_udp_checksum_update(uint16_t cksum0,
 858         uint16_t *ip0,
 859         uint16_t *ip1,
 860         uint16_t port0,
 861         uint16_t port1)
 862 {
 863         int32_t cksum1;
 864
 865         cksum1 = cksum0;
 866         cksum1 = ~cksum1 & 0xFFFF;
 867
 868         /* Subtract ip0 and port 0 (one's complement logic) */
 869         cksum1 -= ip0[0] + ip0[1] + ip0[2] + ip0[3] +
 870                 ip0[4] + ip0[5] + ip0[6] + ip0[7] + port0;
 871         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 872         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 873
 874         /* Add ip1 and port1 (one's complement logic) */
 875         cksum1 += ip1[0] + ip1[1] + ip1[2] + ip1[3] +
 876                 ip1[4] + ip1[5] + ip1[6] + ip1[7] + port1;
 877         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 878         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 879
 880         return (uint16_t)(~cksum1);
 881 }
 882
 883 static __rte_always_inline void
 884 pkt_ipv4_work_nat(struct ipv4_hdr *ip,
 885         struct nat_ipv4_data *data,
 886         struct rte_table_action_nat_config *cfg)
 887 {
 888         if (cfg->source_nat) {
 889                 if (cfg->proto == 0x6) {
 890                         struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1];
 891                         uint16_t ip_cksum, tcp_cksum;
 892
 893                         ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum,
 894                                 ip->src_addr,
 895                                 data->addr);
 896
 897                         tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum,
 898                                 ip->src_addr,
 899                                 data->addr,
 900                                 tcp->src_port,
 901                                 data->port);
 902
 903                         ip->src_addr = data->addr;
 904                         ip->hdr_checksum = ip_cksum;
 905                         tcp->src_port = data->port;
 906                         tcp->cksum = tcp_cksum;
 907                 } else {
 908                         struct udp_hdr *udp = (struct udp_hdr *) &ip[1];
 909                         uint16_t ip_cksum, udp_cksum;
 910
 911                         ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum,
 912                                 ip->src_addr,
 913                                 data->addr);
 914
 915                         udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum,
 916                                 ip->src_addr,
 917                                 data->addr,
 918                                 udp->src_port,
 919                                 data->port);
 920
 921                         ip->src_addr = data->addr;
 922                         ip->hdr_checksum = ip_cksum;
 923                         udp->src_port = data->port;
 924                         if (udp->dgram_cksum)
 925                                 udp->dgram_cksum = udp_cksum;
 926                 }
 927         } else {
 928                 if (cfg->proto == 0x6) {
 929                         struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1];
 930                         uint16_t ip_cksum, tcp_cksum;
 931
 932                         ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum,
 933                                 ip->dst_addr,
 934                                 data->addr);
 935
 936                         tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum,
 937                                 ip->dst_addr,
 938                                 data->addr,
 939                                 tcp->dst_port,
 940                                 data->port);
 941
 942                         ip->dst_addr = data->addr;
 943                         ip->hdr_checksum = ip_cksum;
 944                         tcp->dst_port = data->port;
 945                         tcp->cksum = tcp_cksum;
 946                 } else {
 947                         struct udp_hdr *udp = (struct udp_hdr *) &ip[1];
 948                         uint16_t ip_cksum, udp_cksum;
 949
 950                         ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum,
 951                                 ip->dst_addr,
 952                                 data->addr);
 953
 954                         udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum,
 955                                 ip->dst_addr,
 956                                 data->addr,
 957                                 udp->dst_port,
 958                                 data->port);
 959
 960                         ip->dst_addr = data->addr;
 961                         ip->hdr_checksum = ip_cksum;
 962                         udp->dst_port = data->port;
 963                         if (udp->dgram_cksum)
 964                                 udp->dgram_cksum = udp_cksum;
 965                 }
 966         }
 967 }
 968
 969 static __rte_always_inline void
 970 pkt_ipv6_work_nat(struct ipv6_hdr *ip,
 971         struct nat_ipv6_data *data,
 972         struct rte_table_action_nat_config *cfg)
 973 {
 974         if (cfg->source_nat) {
 975                 if (cfg->proto == 0x6) {
 976                         struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1];
 977                         uint16_t tcp_cksum;
 978
 979                         tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum,
 980                                 (uint16_t *)ip->src_addr,
 981                                 (uint16_t *)data->addr,
 982                                 tcp->src_port,
 983                                 data->port);
 984
 985                         rte_memcpy(ip->src_addr, data->addr, 16);
 986                         tcp->src_port = data->port;
 987                         tcp->cksum = tcp_cksum;
 988                 } else {
 989                         struct udp_hdr *udp = (struct udp_hdr *) &ip[1];
 990                         uint16_t udp_cksum;
 991
 992                         udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum,
 993                                 (uint16_t *)ip->src_addr,
 994                                 (uint16_t *)data->addr,
 995                                 udp->src_port,
 996                                 data->port);
 997
 998                         rte_memcpy(ip->src_addr, data->addr, 16);
 999                         udp->src_port = data->port;
1000                         udp->dgram_cksum = udp_cksum;
1001                 }
1002         } else {
1003                 if (cfg->proto == 0x6) {
1004                         struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1];
1005                         uint16_t tcp_cksum;
1006
1007                         tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum,
1008                                 (uint16_t *)ip->dst_addr,
1009                                 (uint16_t *)data->addr,
1010                                 tcp->dst_port,
1011                                 data->port);
1012
1013                         rte_memcpy(ip->dst_addr, data->addr, 16);
1014                         tcp->dst_port = data->port;
1015                         tcp->cksum = tcp_cksum;
1016                 } else {
1017                         struct udp_hdr *udp = (struct udp_hdr *) &ip[1];
1018                         uint16_t udp_cksum;
1019
1020                         udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum,
1021                                 (uint16_t *)ip->dst_addr,
1022                                 (uint16_t *)data->addr,
1023                                 udp->dst_port,
1024                                 data->port);
1025
1026                         rte_memcpy(ip->dst_addr, data->addr, 16);
1027                         udp->dst_port = data->port;
1028                         udp->dgram_cksum = udp_cksum;
1029                 }
1030         }
1031 }
1032
1033 /**
1034  * RTE_TABLE_ACTION_TTL
1035  */
1036 static int
1037 ttl_cfg_check(struct rte_table_action_ttl_config *ttl)
1038 {
1039         if (ttl->drop == 0)
1040                 return -ENOTSUP;
1041
1042         return 0;
1043 }
1044
1045 struct ttl_data {
1046         uint32_t n_packets;
1047 } __attribute__((__packed__));
1048
1049 #define TTL_INIT(data, decrement)                         \
1050         ((data)->n_packets = (decrement) ? 1 : 0)
1051
1052 #define TTL_DEC_GET(data)                                  \
1053         ((uint8_t)((data)->n_packets & 1))
1054
1055 #define TTL_STATS_RESET(data)                             \
1056         ((data)->n_packets = ((data)->n_packets & 1))
1057
1058 #define TTL_STATS_READ(data)                               \
1059         ((data)->n_packets >> 1)
1060
1061 #define TTL_STATS_ADD(data, value)                        \
1062         ((data)->n_packets =                                  \
1063                 (((((data)->n_packets >> 1) + (value)) << 1) |    \
1064                 ((data)->n_packets & 1)))
1065
1066 static int
1067 ttl_apply(void *data,
1068         struct rte_table_action_ttl_params *p)
1069 {
1070         struct ttl_data *d = data;
1071
1072         TTL_INIT(d, p->decrement);
1073
1074         return 0;
1075 }
1076
1077 static __rte_always_inline uint64_t
1078 pkt_ipv4_work_ttl(struct ipv4_hdr *ip,
1079         struct ttl_data *data)
1080 {
1081         uint32_t drop;
1082         uint16_t cksum = ip->hdr_checksum;
1083         uint8_t ttl = ip->time_to_live;
1084         uint8_t ttl_diff = TTL_DEC_GET(data);
1085
1086         cksum += ttl_diff;
1087         ttl -= ttl_diff;
1088
1089         ip->hdr_checksum = cksum;
1090         ip->time_to_live = ttl;
1091
1092         drop = (ttl == 0) ? 1 : 0;
1093         TTL_STATS_ADD(data, drop);
1094
1095         return drop;
1096 }
1097
1098 static __rte_always_inline uint64_t
1099 pkt_ipv6_work_ttl(struct ipv6_hdr *ip,
1100         struct ttl_data *data)
1101 {
1102         uint32_t drop;
1103         uint8_t ttl = ip->hop_limits;
1104         uint8_t ttl_diff = TTL_DEC_GET(data);
1105
1106         ttl -= ttl_diff;
1107
1108         ip->hop_limits = ttl;
1109
1110         drop = (ttl == 0) ? 1 : 0;
1111         TTL_STATS_ADD(data, drop);
1112
1113         return drop;
1114 }
1115
1116 /**
1117  * Action profile
1118  */
1119 static int
1120 action_valid(enum rte_table_action_type action)
1121 {
1122         switch (action) {
1123         case RTE_TABLE_ACTION_FWD:
1124         case RTE_TABLE_ACTION_MTR:
1125         case RTE_TABLE_ACTION_TM:
1126         case RTE_TABLE_ACTION_ENCAP:
1127         case RTE_TABLE_ACTION_NAT:
1128         case RTE_TABLE_ACTION_TTL:
1129                 return 1;
1130         default:
1131                 return 0;
1132         }
1133 }
1134
1135
1136 #define RTE_TABLE_ACTION_MAX                      64
1137
1138 struct ap_config {
1139         uint64_t action_mask;
1140         struct rte_table_action_common_config common;
1141         struct rte_table_action_mtr_config mtr;
1142         struct rte_table_action_tm_config tm;
1143         struct rte_table_action_encap_config encap;
1144         struct rte_table_action_nat_config nat;
1145         struct rte_table_action_ttl_config ttl;
1146 };
1147
1148 static size_t
1149 action_cfg_size(enum rte_table_action_type action)
1150 {
1151         switch (action) {
1152         case RTE_TABLE_ACTION_MTR:
1153                 return sizeof(struct rte_table_action_mtr_config);
1154         case RTE_TABLE_ACTION_TM:
1155                 return sizeof(struct rte_table_action_tm_config);
1156         case RTE_TABLE_ACTION_ENCAP:
1157                 return sizeof(struct rte_table_action_encap_config);
1158         case RTE_TABLE_ACTION_NAT:
1159                 return sizeof(struct rte_table_action_nat_config);
1160         case RTE_TABLE_ACTION_TTL:
1161                 return sizeof(struct rte_table_action_ttl_config);
1162         default:
1163                 return 0;
1164         }
1165 }
1166
1167 static void*
1168 action_cfg_get(struct ap_config *ap_config,
1169         enum rte_table_action_type type)
1170 {
1171         switch (type) {
1172         case RTE_TABLE_ACTION_MTR:
1173                 return &ap_config->mtr;
1174
1175         case RTE_TABLE_ACTION_TM:
1176                 return &ap_config->tm;
1177
1178         case RTE_TABLE_ACTION_ENCAP:
1179                 return &ap_config->encap;
1180
1181         case RTE_TABLE_ACTION_NAT:
1182                 return &ap_config->nat;
1183
1184         case RTE_TABLE_ACTION_TTL:
1185                 return &ap_config->ttl;
1186
1187         default:
1188                 return NULL;
1189         }
1190 }
1191
1192 static void
1193 action_cfg_set(struct ap_config *ap_config,
1194         enum rte_table_action_type type,
1195         void *action_cfg)
1196 {
1197         void *dst = action_cfg_get(ap_config, type);
1198
1199         if (dst)
1200                 memcpy(dst, action_cfg, action_cfg_size(type));
1201
1202         ap_config->action_mask |= 1LLU << type;
1203 }
1204
1205 struct ap_data {
1206         size_t offset[RTE_TABLE_ACTION_MAX];
1207         size_t total_size;
1208 };
1209
1210 static size_t
1211 action_data_size(enum rte_table_action_type action,
1212         struct ap_config *ap_config)
1213 {
1214         switch (action) {
1215         case RTE_TABLE_ACTION_FWD:
1216                 return sizeof(struct fwd_data);
1217
1218         case RTE_TABLE_ACTION_MTR:
1219                 return mtr_data_size(&ap_config->mtr);
1220
1221         case RTE_TABLE_ACTION_TM:
1222                 return sizeof(struct tm_data);
1223
1224         case RTE_TABLE_ACTION_ENCAP:
1225                 return encap_data_size(&ap_config->encap);
1226
1227         case RTE_TABLE_ACTION_NAT:
1228                 return nat_data_size(&ap_config->nat,
1229                         &ap_config->common);
1230
1231         case RTE_TABLE_ACTION_TTL:
1232                 return sizeof(struct ttl_data);
1233
1234         default:
1235                 return 0;
1236         }
1237 }
1238
1239
1240 static void
1241 action_data_offset_set(struct ap_data *ap_data,
1242         struct ap_config *ap_config)
1243 {
1244         uint64_t action_mask = ap_config->action_mask;
1245         size_t offset;
1246         uint32_t action;
1247
1248         memset(ap_data->offset, 0, sizeof(ap_data->offset));
1249
1250         offset = 0;
1251         for (action = 0; action < RTE_TABLE_ACTION_MAX; action++)
1252                 if (action_mask & (1LLU << action)) {
1253                         ap_data->offset[action] = offset;
1254                         offset += action_data_size((enum rte_table_action_type)action,
1255                                 ap_config);
1256                 }
1257
1258         ap_data->total_size = offset;
1259 }
1260
1261 struct rte_table_action_profile {
1262         struct ap_config cfg;
1263         struct ap_data data;
1264         int frozen;
1265 };
1266
1267 struct rte_table_action_profile *
1268 rte_table_action_profile_create(struct rte_table_action_common_config *common)
1269 {
1270         struct rte_table_action_profile *ap;
1271
1272         /* Check input arguments */
1273         if (common == NULL)
1274                 return NULL;
1275
1276         /* Memory allocation */
1277         ap = calloc(1, sizeof(struct rte_table_action_profile));
1278         if (ap == NULL)
1279                 return NULL;
1280
1281         /* Initialization */
1282         memcpy(&ap->cfg.common, common, sizeof(*common));
1283
1284         return ap;
1285 }
1286
1287
1288 int
1289 rte_table_action_profile_action_register(struct rte_table_action_profile *profile,
1290         enum rte_table_action_type type,
1291         void *action_config)
1292 {
1293         int status;
1294
1295         /* Check input arguments */
1296         if ((profile == NULL) ||
1297                 profile->frozen ||
1298                 (action_valid(type) == 0) ||
1299                 (profile->cfg.action_mask & (1LLU << type)) ||
1300                 ((action_cfg_size(type) == 0) && action_config) ||
1301                 (action_cfg_size(type) && (action_config == NULL)))
1302                 return -EINVAL;
1303
1304         switch (type) {
1305         case RTE_TABLE_ACTION_MTR:
1306                 status = mtr_cfg_check(action_config);
1307                 break;
1308
1309         case RTE_TABLE_ACTION_TM:
1310                 status = tm_cfg_check(action_config);
1311                 break;
1312
1313         case RTE_TABLE_ACTION_ENCAP:
1314                 status = encap_cfg_check(action_config);
1315                 break;
1316
1317         case RTE_TABLE_ACTION_NAT:
1318                 status = nat_cfg_check(action_config);
1319                 break;
1320
1321         case RTE_TABLE_ACTION_TTL:
1322                 status = ttl_cfg_check(action_config);
1323                 break;
1324
1325         default:
1326                 status = 0;
1327                 break;
1328         }
1329
1330         if (status)
1331                 return status;
1332
1333         /* Action enable */
1334         action_cfg_set(&profile->cfg, type, action_config);
1335
1336         return 0;
1337 }
1338
1339 int
1340 rte_table_action_profile_freeze(struct rte_table_action_profile *profile)
1341 {
1342         if (profile->frozen)
1343                 return -EBUSY;
1344
1345         profile->cfg.action_mask |= 1LLU << RTE_TABLE_ACTION_FWD;
1346         action_data_offset_set(&profile->data, &profile->cfg);
1347         profile->frozen = 1;
1348
1349         return 0;
1350 }
1351
1352 int
1353 rte_table_action_profile_free(struct rte_table_action_profile *profile)
1354 {
1355         if (profile == NULL)
1356                 return 0;
1357
1358         free(profile);
1359         return 0;
1360 }
1361
1362 /**
1363  * Action
1364  */
1365 #define METER_PROFILES_MAX                                 32
1366
1367 struct rte_table_action {
1368         struct ap_config cfg;
1369         struct ap_data data;
1370         struct dscp_table_data dscp_table;
1371         struct meter_profile_data mp[METER_PROFILES_MAX];
1372 };
1373
1374 struct rte_table_action *
1375 rte_table_action_create(struct rte_table_action_profile *profile,
1376         uint32_t socket_id)
1377 {
1378         struct rte_table_action *action;
1379
1380         /* Check input arguments */
1381         if ((profile == NULL) ||
1382                 (profile->frozen == 0))
1383                 return NULL;
1384
1385         /* Memory allocation */
1386         action = rte_zmalloc_socket(NULL,
1387                 sizeof(struct rte_table_action),
1388                 RTE_CACHE_LINE_SIZE,
1389                 socket_id);
1390         if (action == NULL)
1391                 return NULL;
1392
1393         /* Initialization */
1394         memcpy(&action->cfg, &profile->cfg, sizeof(profile->cfg));
1395         memcpy(&action->data, &profile->data, sizeof(profile->data));
1396
1397         return action;
1398 }
1399
1400 static __rte_always_inline void *
1401 action_data_get(void *data,
1402         struct rte_table_action *action,
1403         enum rte_table_action_type type)
1404 {
1405         size_t offset = action->data.offset[type];
1406         uint8_t *data_bytes = data;
1407
1408         return &data_bytes[offset];
1409 }
1410
1411 int
1412 rte_table_action_apply(struct rte_table_action *action,
1413         void *data,
1414         enum rte_table_action_type type,
1415         void *action_params)
1416 {
1417         void *action_data;
1418
1419         /* Check input arguments */
1420         if ((action == NULL) ||
1421                 (data == NULL) ||
1422                 (action_valid(type) == 0) ||
1423                 ((action->cfg.action_mask & (1LLU << type)) == 0) ||
1424                 (action_params == NULL))
1425                 return -EINVAL;
1426
1427         /* Data update */
1428         action_data = action_data_get(data, action, type);
1429
1430         switch (type) {
1431         case RTE_TABLE_ACTION_FWD:
1432                 return fwd_apply(action_data,
1433                         action_params);
1434
1435         case RTE_TABLE_ACTION_MTR:
1436                 return mtr_apply(action_data,
1437                         action_params,
1438                         &action->cfg.mtr,
1439                         action->mp,
1440                         RTE_DIM(action->mp));
1441
1442         case RTE_TABLE_ACTION_TM:
1443                 return tm_apply(action_data,
1444                         action_params,
1445                         &action->cfg.tm);
1446
1447         case RTE_TABLE_ACTION_ENCAP:
1448                 return encap_apply(action_data,
1449                         action_params,
1450                         &action->cfg.encap,
1451                         &action->cfg.common);
1452
1453         case RTE_TABLE_ACTION_NAT:
1454                 return nat_apply(action_data,
1455                         action_params,
1456                         &action->cfg.common);
1457
1458         case RTE_TABLE_ACTION_TTL:
1459                 return ttl_apply(action_data,
1460                         action_params);
1461
1462         default:
1463                 return -EINVAL;
1464         }
1465 }
1466
1467 int
1468 rte_table_action_dscp_table_update(struct rte_table_action *action,
1469         uint64_t dscp_mask,
1470         struct rte_table_action_dscp_table *table)
1471 {
1472         uint32_t i;
1473
1474         /* Check input arguments */
1475         if ((action == NULL) ||
1476                 ((action->cfg.action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) |
1477                 (1LLU << RTE_TABLE_ACTION_TM))) == 0) ||
1478                 (dscp_mask == 0) ||
1479                 (table == NULL))
1480                 return -EINVAL;
1481
1482         for (i = 0; i < RTE_DIM(table->entry); i++) {
1483                 struct dscp_table_entry_data *data =
1484                         &action->dscp_table.entry[i];
1485                 struct rte_table_action_dscp_table_entry *entry =
1486                         &table->entry[i];
1487                 uint16_t queue_tc_color =
1488                         MBUF_SCHED_QUEUE_TC_COLOR(entry->tc_queue_id,
1489                                 entry->tc_id,
1490                                 entry->color);
1491
1492                 if ((dscp_mask & (1LLU << i)) == 0)
1493                         continue;
1494
1495                 data->color = entry->color;
1496                 data->tc = entry->tc_id;
1497                 data->queue_tc_color = queue_tc_color;
1498         }
1499
1500         return 0;
1501 }
1502
1503 int
1504 rte_table_action_meter_profile_add(struct rte_table_action *action,
1505         uint32_t meter_profile_id,
1506         struct rte_table_action_meter_profile *profile)
1507 {
1508         struct meter_profile_data *mp_data;
1509         uint32_t status;
1510
1511         /* Check input arguments */
1512         if ((action == NULL) ||
1513                 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) ||
1514                 (profile == NULL))
1515                 return -EINVAL;
1516
1517         if (profile->alg != RTE_TABLE_ACTION_METER_TRTCM)
1518                 return -ENOTSUP;
1519
1520         mp_data = meter_profile_data_find(action->mp,
1521                 RTE_DIM(action->mp),
1522                 meter_profile_id);
1523         if (mp_data)
1524                 return -EEXIST;
1525
1526         mp_data = meter_profile_data_find_unused(action->mp,
1527                 RTE_DIM(action->mp));
1528         if (!mp_data)
1529                 return -ENOSPC;
1530
1531         /* Install new profile */
1532         status = rte_meter_trtcm_profile_config(&mp_data->profile,
1533                 &profile->trtcm);
1534         if (status)
1535                 return status;
1536
1537         mp_data->profile_id = meter_profile_id;
1538         mp_data->valid = 1;
1539
1540         return 0;
1541 }
1542
1543 int
1544 rte_table_action_meter_profile_delete(struct rte_table_action *action,
1545         uint32_t meter_profile_id)
1546 {
1547         struct meter_profile_data *mp_data;
1548
1549         /* Check input arguments */
1550         if ((action == NULL) ||
1551                 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0))
1552                 return -EINVAL;
1553
1554         mp_data = meter_profile_data_find(action->mp,
1555                 RTE_DIM(action->mp),
1556                 meter_profile_id);
1557         if (!mp_data)
1558                 return 0;
1559
1560         /* Uninstall profile */
1561         mp_data->valid = 0;
1562
1563         return 0;
1564 }
1565
1566 int
1567 rte_table_action_meter_read(struct rte_table_action *action,
1568         void *data,
1569         uint32_t tc_mask,
1570         struct rte_table_action_mtr_counters *stats,
1571         int clear)
1572 {
1573         struct mtr_trtcm_data *mtr_data;
1574         uint32_t i;
1575
1576         /* Check input arguments */
1577         if ((action == NULL) ||
1578                 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) ||
1579                 (data == NULL) ||
1580                 (tc_mask > RTE_LEN2MASK(action->cfg.mtr.n_tc, uint32_t)))
1581                 return -EINVAL;
1582
1583         mtr_data = action_data_get(data, action, RTE_TABLE_ACTION_MTR);
1584
1585         /* Read */
1586         if (stats) {
1587                 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) {
1588                         struct rte_table_action_mtr_counters_tc *dst =
1589                                 &stats->stats[i];
1590                         struct mtr_trtcm_data *src = &mtr_data[i];
1591
1592                         if ((tc_mask & (1 << i)) == 0)
1593                                 continue;
1594
1595                         dst->n_packets[e_RTE_METER_GREEN] =
1596                                 mtr_trtcm_data_stats_get(src, e_RTE_METER_GREEN);
1597
1598                         dst->n_packets[e_RTE_METER_YELLOW] =
1599                                 mtr_trtcm_data_stats_get(src, e_RTE_METER_YELLOW);
1600
1601                         dst->n_packets[e_RTE_METER_RED] =
1602                                 mtr_trtcm_data_stats_get(src, e_RTE_METER_RED);
1603
1604                         dst->n_packets_valid = 1;
1605                         dst->n_bytes_valid = 0;
1606                 }
1607
1608                 stats->tc_mask = tc_mask;
1609         }
1610
1611         /* Clear */
1612         if (clear)
1613                 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) {
1614                         struct mtr_trtcm_data *src = &mtr_data[i];
1615
1616                         if ((tc_mask & (1 << i)) == 0)
1617                                 continue;
1618
1619                         mtr_trtcm_data_stats_reset(src, e_RTE_METER_GREEN);
1620                         mtr_trtcm_data_stats_reset(src, e_RTE_METER_YELLOW);
1621                         mtr_trtcm_data_stats_reset(src, e_RTE_METER_RED);
1622                 }
1623
1624
1625         return 0;
1626 }
1627
1628 int
1629 rte_table_action_ttl_read(struct rte_table_action *action,
1630         void *data,
1631         struct rte_table_action_ttl_counters *stats,
1632         int clear)
1633 {
1634         struct ttl_data *ttl_data;
1635
1636         /* Check input arguments */
1637         if ((action == NULL) ||
1638                 ((action->cfg.action_mask &
1639                 (1LLU << RTE_TABLE_ACTION_TTL)) == 0) ||
1640                 (data == NULL))
1641                 return -EINVAL;
1642
1643         ttl_data = action_data_get(data, action, RTE_TABLE_ACTION_TTL);
1644
1645         /* Read */
1646         if (stats)
1647                 stats->n_packets = TTL_STATS_READ(ttl_data);
1648
1649         /* Clear */
1650         if (clear)
1651                 TTL_STATS_RESET(ttl_data);
1652
1653         return 0;
1654 }
1655
1656 static __rte_always_inline uint64_t
1657 pkt_work(struct rte_mbuf *mbuf,
1658         struct rte_pipeline_table_entry *table_entry,
1659         uint64_t time,
1660         struct rte_table_action *action,
1661         struct ap_config *cfg)
1662 {
1663         uint64_t drop_mask = 0;
1664
1665         uint32_t ip_offset = action->cfg.common.ip_offset;
1666         void *ip = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ip_offset);
1667
1668         uint32_t dscp;
1669         uint16_t total_length;
1670
1671         if (cfg->common.ip_version) {
1672                 struct ipv4_hdr *hdr = ip;
1673
1674                 dscp = hdr->type_of_service >> 2;
1675                 total_length = rte_ntohs(hdr->total_length);
1676         } else {
1677                 struct ipv6_hdr *hdr = ip;
1678
1679                 dscp = (rte_ntohl(hdr->vtc_flow) & 0x0F600000) >> 18;
1680                 total_length =
1681                         rte_ntohs(hdr->payload_len) + sizeof(struct ipv6_hdr);
1682         }
1683
1684         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1685                 void *data =
1686                         action_data_get(table_entry, action, RTE_TABLE_ACTION_MTR);
1687
1688                 drop_mask |= pkt_work_mtr(mbuf,
1689                         data,
1690                         &action->dscp_table,
1691                         action->mp,
1692                         time,
1693                         dscp,
1694                         total_length);
1695         }
1696
1697         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1698                 void *data =
1699                         action_data_get(table_entry, action, RTE_TABLE_ACTION_TM);
1700
1701                 pkt_work_tm(mbuf,
1702                         data,
1703                         &action->dscp_table,
1704                         dscp);
1705         }
1706
1707         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1708                 void *data =
1709                         action_data_get(table_entry, action, RTE_TABLE_ACTION_ENCAP);
1710
1711                 pkt_work_encap(mbuf,
1712                         data,
1713                         &cfg->encap,
1714                         ip,
1715                         total_length,
1716                         ip_offset);
1717         }
1718
1719         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1720                 void *data =
1721                         action_data_get(table_entry, action, RTE_TABLE_ACTION_NAT);
1722
1723                 if (cfg->common.ip_version)
1724                         pkt_ipv4_work_nat(ip, data, &cfg->nat);
1725                 else
1726                         pkt_ipv6_work_nat(ip, data, &cfg->nat);
1727         }
1728
1729         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
1730                 void *data =
1731                         action_data_get(table_entry, action, RTE_TABLE_ACTION_TTL);
1732
1733                 if (cfg->common.ip_version)
1734                         drop_mask |= pkt_ipv4_work_ttl(ip, data);
1735                 else
1736                         drop_mask |= pkt_ipv6_work_ttl(ip, data);
1737         }
1738
1739         return drop_mask;
1740 }
1741
1742 static __rte_always_inline uint64_t
1743 pkt4_work(struct rte_mbuf **mbufs,
1744         struct rte_pipeline_table_entry **table_entries,
1745         uint64_t time,
1746         struct rte_table_action *action,
1747         struct ap_config *cfg)
1748 {
1749         uint64_t drop_mask0 = 0;
1750         uint64_t drop_mask1 = 0;
1751         uint64_t drop_mask2 = 0;
1752         uint64_t drop_mask3 = 0;
1753
1754         struct rte_mbuf *mbuf0 = mbufs[0];
1755         struct rte_mbuf *mbuf1 = mbufs[1];
1756         struct rte_mbuf *mbuf2 = mbufs[2];
1757         struct rte_mbuf *mbuf3 = mbufs[3];
1758
1759         struct rte_pipeline_table_entry *table_entry0 = table_entries[0];
1760         struct rte_pipeline_table_entry *table_entry1 = table_entries[1];
1761         struct rte_pipeline_table_entry *table_entry2 = table_entries[2];
1762         struct rte_pipeline_table_entry *table_entry3 = table_entries[3];
1763
1764         uint32_t ip_offset = action->cfg.common.ip_offset;
1765         void *ip0 = RTE_MBUF_METADATA_UINT32_PTR(mbuf0, ip_offset);
1766         void *ip1 = RTE_MBUF_METADATA_UINT32_PTR(mbuf1, ip_offset);
1767         void *ip2 = RTE_MBUF_METADATA_UINT32_PTR(mbuf2, ip_offset);
1768         void *ip3 = RTE_MBUF_METADATA_UINT32_PTR(mbuf3, ip_offset);
1769
1770         uint32_t dscp0, dscp1, dscp2, dscp3;
1771         uint16_t total_length0, total_length1, total_length2, total_length3;
1772
1773         if (cfg->common.ip_version) {
1774                 struct ipv4_hdr *hdr0 = ip0;
1775                 struct ipv4_hdr *hdr1 = ip1;
1776                 struct ipv4_hdr *hdr2 = ip2;
1777                 struct ipv4_hdr *hdr3 = ip3;
1778
1779                 dscp0 = hdr0->type_of_service >> 2;
1780                 dscp1 = hdr1->type_of_service >> 2;
1781                 dscp2 = hdr2->type_of_service >> 2;
1782                 dscp3 = hdr3->type_of_service >> 2;
1783
1784                 total_length0 = rte_ntohs(hdr0->total_length);
1785                 total_length1 = rte_ntohs(hdr1->total_length);
1786                 total_length2 = rte_ntohs(hdr2->total_length);
1787                 total_length3 = rte_ntohs(hdr3->total_length);
1788         } else {
1789                 struct ipv6_hdr *hdr0 = ip0;
1790                 struct ipv6_hdr *hdr1 = ip1;
1791                 struct ipv6_hdr *hdr2 = ip2;
1792                 struct ipv6_hdr *hdr3 = ip3;
1793
1794                 dscp0 = (rte_ntohl(hdr0->vtc_flow) & 0x0F600000) >> 18;
1795                 dscp1 = (rte_ntohl(hdr1->vtc_flow) & 0x0F600000) >> 18;
1796                 dscp2 = (rte_ntohl(hdr2->vtc_flow) & 0x0F600000) >> 18;
1797                 dscp3 = (rte_ntohl(hdr3->vtc_flow) & 0x0F600000) >> 18;
1798
1799                 total_length0 =
1800                         rte_ntohs(hdr0->payload_len) + sizeof(struct ipv6_hdr);
1801                 total_length1 =
1802                         rte_ntohs(hdr1->payload_len) + sizeof(struct ipv6_hdr);
1803                 total_length2 =
1804                         rte_ntohs(hdr2->payload_len) + sizeof(struct ipv6_hdr);
1805                 total_length3 =
1806                         rte_ntohs(hdr3->payload_len) + sizeof(struct ipv6_hdr);
1807         }
1808
1809         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1810                 void *data0 =
1811                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_MTR);
1812                 void *data1 =
1813                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_MTR);
1814                 void *data2 =
1815                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_MTR);
1816                 void *data3 =
1817                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_MTR);
1818
1819                 drop_mask0 |= pkt_work_mtr(mbuf0,
1820                         data0,
1821                         &action->dscp_table,
1822                         action->mp,
1823                         time,
1824                         dscp0,
1825                         total_length0);
1826
1827                 drop_mask1 |= pkt_work_mtr(mbuf1,
1828                         data1,
1829                         &action->dscp_table,
1830                         action->mp,
1831                         time,
1832                         dscp1,
1833                         total_length1);
1834
1835                 drop_mask2 |= pkt_work_mtr(mbuf2,
1836                         data2,
1837                         &action->dscp_table,
1838                         action->mp,
1839                         time,
1840                         dscp2,
1841                         total_length2);
1842
1843                 drop_mask3 |= pkt_work_mtr(mbuf3,
1844                         data3,
1845                         &action->dscp_table,
1846                         action->mp,
1847                         time,
1848                         dscp3,
1849                         total_length3);
1850         }
1851
1852         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1853                 void *data0 =
1854                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_TM);
1855                 void *data1 =
1856                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_TM);
1857                 void *data2 =
1858                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_TM);
1859                 void *data3 =
1860                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_TM);
1861
1862                 pkt_work_tm(mbuf0,
1863                         data0,
1864                         &action->dscp_table,
1865                         dscp0);
1866
1867                 pkt_work_tm(mbuf1,
1868                         data1,
1869                         &action->dscp_table,
1870                         dscp1);
1871
1872                 pkt_work_tm(mbuf2,
1873                         data2,
1874                         &action->dscp_table,
1875                         dscp2);
1876
1877                 pkt_work_tm(mbuf3,
1878                         data3,
1879                         &action->dscp_table,
1880                         dscp3);
1881         }
1882
1883         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1884                 void *data0 =
1885                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_ENCAP);
1886                 void *data1 =
1887                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_ENCAP);
1888                 void *data2 =
1889                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_ENCAP);
1890                 void *data3 =
1891                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_ENCAP);
1892
1893                 pkt_work_encap(mbuf0,
1894                         data0,
1895                         &cfg->encap,
1896                         ip0,
1897                         total_length0,
1898                         ip_offset);
1899
1900                 pkt_work_encap(mbuf1,
1901                         data1,
1902                         &cfg->encap,
1903                         ip1,
1904                         total_length1,
1905                         ip_offset);
1906
1907                 pkt_work_encap(mbuf2,
1908                         data2,
1909                         &cfg->encap,
1910                         ip2,
1911                         total_length2,
1912                         ip_offset);
1913
1914                 pkt_work_encap(mbuf3,
1915                         data3,
1916                         &cfg->encap,
1917                         ip3,
1918                         total_length3,
1919                         ip_offset);
1920         }
1921
1922         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1923                 void *data0 =
1924                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_NAT);
1925                 void *data1 =
1926                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_NAT);
1927                 void *data2 =
1928                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_NAT);
1929                 void *data3 =
1930                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_NAT);
1931
1932                 if (cfg->common.ip_version) {
1933                         pkt_ipv4_work_nat(ip0, data0, &cfg->nat);
1934                         pkt_ipv4_work_nat(ip1, data1, &cfg->nat);
1935                         pkt_ipv4_work_nat(ip2, data2, &cfg->nat);
1936                         pkt_ipv4_work_nat(ip3, data3, &cfg->nat);
1937                 } else {
1938                         pkt_ipv6_work_nat(ip0, data0, &cfg->nat);
1939                         pkt_ipv6_work_nat(ip1, data1, &cfg->nat);
1940                         pkt_ipv6_work_nat(ip2, data2, &cfg->nat);
1941                         pkt_ipv6_work_nat(ip3, data3, &cfg->nat);
1942                 }
1943         }
1944
1945         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TTL)) {
1946                 void *data0 =
1947                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_TTL);
1948                 void *data1 =
1949                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_TTL);
1950                 void *data2 =
1951                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_TTL);
1952                 void *data3 =
1953                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_TTL);
1954
1955                 if (cfg->common.ip_version) {
1956                         drop_mask0 |= pkt_ipv4_work_ttl(ip0, data0);
1957                         drop_mask1 |= pkt_ipv4_work_ttl(ip1, data1);
1958                         drop_mask2 |= pkt_ipv4_work_ttl(ip2, data2);
1959                         drop_mask3 |= pkt_ipv4_work_ttl(ip3, data3);
1960                 } else {
1961                         drop_mask0 |= pkt_ipv6_work_ttl(ip0, data0);
1962                         drop_mask1 |= pkt_ipv6_work_ttl(ip1, data1);
1963                         drop_mask2 |= pkt_ipv6_work_ttl(ip2, data2);
1964                         drop_mask3 |= pkt_ipv6_work_ttl(ip3, data3);
1965                 }
1966         }
1967
1968         return drop_mask0 |
1969                 (drop_mask1 << 1) |
1970                 (drop_mask2 << 2) |
1971                 (drop_mask3 << 3);
1972 }
1973
1974 static __rte_always_inline int
1975 ah(struct rte_pipeline *p,
1976         struct rte_mbuf **pkts,
1977         uint64_t pkts_mask,
1978         struct rte_pipeline_table_entry **entries,
1979         struct rte_table_action *action,
1980         struct ap_config *cfg)
1981 {
1982         uint64_t pkts_drop_mask = 0;
1983         uint64_t time = 0;
1984
1985         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR))
1986                 time = rte_rdtsc();
1987
1988         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1989                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1990                 uint32_t i;
1991
1992                 for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4) {
1993                         uint64_t drop_mask;
1994
1995                         drop_mask = pkt4_work(&pkts[i],
1996                                 &entries[i],
1997                                 time,
1998                                 action,
1999                                 cfg);
2000
2001                         pkts_drop_mask |= drop_mask << i;
2002                 }
2003
2004                 for ( ; i < n_pkts; i++) {
2005                         uint64_t drop_mask;
2006
2007                         drop_mask = pkt_work(pkts[i],
2008                                 entries[i],
2009                                 time,
2010                                 action,
2011                                 cfg);
2012
2013                         pkts_drop_mask |= drop_mask << i;
2014                 }
2015         } else
2016                 for ( ; pkts_mask; ) {
2017                         uint32_t pos = __builtin_ctzll(pkts_mask);
2018                         uint64_t pkt_mask = 1LLU << pos;
2019                         uint64_t drop_mask;
2020
2021                         drop_mask = pkt_work(pkts[pos],
2022                                 entries[pos],
2023                                 time,
2024                                 action,
2025                                 cfg);
2026
2027                         pkts_mask &= ~pkt_mask;
2028                         pkts_drop_mask |= drop_mask << pos;
2029                 }
2030
2031         rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
2032
2033         return 0;
2034 }
2035
2036 static int
2037 ah_default(struct rte_pipeline *p,
2038         struct rte_mbuf **pkts,
2039         uint64_t pkts_mask,
2040         struct rte_pipeline_table_entry **entries,
2041         void *arg)
2042 {
2043         struct rte_table_action *action = arg;
2044
2045         return ah(p,
2046                 pkts,
2047                 pkts_mask,
2048                 entries,
2049                 action,
2050                 &action->cfg);
2051 }
2052
2053 static rte_pipeline_table_action_handler_hit
2054 ah_selector(struct rte_table_action *action)
2055 {
2056         if (action->cfg.action_mask == (1LLU << RTE_TABLE_ACTION_FWD))
2057                 return NULL;
2058
2059         return ah_default;
2060 }
2061
2062 int
2063 rte_table_action_table_params_get(struct rte_table_action *action,
2064         struct rte_pipeline_table_params *params)
2065 {
2066         rte_pipeline_table_action_handler_hit f_action_hit;
2067         uint32_t total_size;
2068
2069         /* Check input arguments */
2070         if ((action == NULL) ||
2071                 (params == NULL))
2072                 return -EINVAL;
2073
2074         f_action_hit = ah_selector(action);
2075         total_size = rte_align32pow2(action->data.total_size);
2076
2077         /* Fill in params */
2078         params->f_action_hit = f_action_hit;
2079         params->f_action_miss = NULL;
2080         params->arg_ah = (f_action_hit) ? action : NULL;
2081         params->action_data_size = total_size -
2082                 sizeof(struct rte_pipeline_table_entry);
2083
2084         return 0;
2085 }
2086
2087 int
2088 rte_table_action_free(struct rte_table_action *action)
2089 {
2090         if (action == NULL)
2091                 return 0;
2092
2093         rte_free(action);
2094
2095         return 0;
2096 }