lib/librte_pipeline/rte_table_action.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdlib.h>
   6 #include <string.h>
   7
   8 #include <rte_common.h>
   9 #include <rte_byteorder.h>
  10 #include <rte_cycles.h>
  11 #include <rte_malloc.h>
  12 #include <rte_memcpy.h>
  13 #include <rte_ether.h>
  14 #include <rte_ip.h>
  15 #include <rte_esp.h>
  16 #include <rte_tcp.h>
  17 #include <rte_udp.h>
  18
  19 #include "rte_table_action.h"
  20
  21 #define rte_htons rte_cpu_to_be_16
  22 #define rte_htonl rte_cpu_to_be_32
  23
  24 #define rte_ntohs rte_be_to_cpu_16
  25 #define rte_ntohl rte_be_to_cpu_32
  26
  27 /**
  28  * RTE_TABLE_ACTION_FWD
  29  */
  30 #define fwd_data rte_pipeline_table_entry
  31
  32 static int
  33 fwd_apply(struct fwd_data *data,
  34         struct rte_table_action_fwd_params *p)
  35 {
  36         data->action = p->action;
  37
  38         if (p->action == RTE_PIPELINE_ACTION_PORT)
  39                 data->port_id = p->id;
  40
  41         if (p->action == RTE_PIPELINE_ACTION_TABLE)
  42                 data->table_id = p->id;
  43
  44         return 0;
  45 }
  46
  47 /**
  48  * RTE_TABLE_ACTION_MTR
  49  */
  50 static int
  51 mtr_cfg_check(struct rte_table_action_mtr_config *mtr)
  52 {
  53         if ((mtr->alg == RTE_TABLE_ACTION_METER_SRTCM) ||
  54                 ((mtr->n_tc != 1) && (mtr->n_tc != 4)) ||
  55                 (mtr->n_bytes_enabled != 0))
  56                 return -ENOTSUP;
  57         return 0;
  58 }
  59
  60 #define MBUF_SCHED_QUEUE_TC_COLOR(queue, tc, color)        \
  61         ((uint16_t)((((uint64_t)(queue)) & 0x3) |          \
  62         ((((uint64_t)(tc)) & 0x3) << 2) |                  \
  63         ((((uint64_t)(color)) & 0x3) << 4)))
  64
  65 #define MBUF_SCHED_COLOR(sched, color)                     \
  66         (((sched) & (~0x30LLU)) | ((color) << 4))
  67
  68 struct mtr_trtcm_data {
  69         struct rte_meter_trtcm trtcm;
  70         uint64_t stats[e_RTE_METER_COLORS];
  71 } __attribute__((__packed__));
  72
  73 #define MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data)          \
  74         (((data)->stats[e_RTE_METER_GREEN] & 0xF8LLU) >> 3)
  75
  76 static void
  77 mtr_trtcm_data_meter_profile_id_set(struct mtr_trtcm_data *data,
  78         uint32_t profile_id)
  79 {
  80         data->stats[e_RTE_METER_GREEN] &= ~0xF8LLU;
  81         data->stats[e_RTE_METER_GREEN] |= (profile_id % 32) << 3;
  82 }
  83
  84 #define MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color)\
  85         (((data)->stats[(color)] & 4LLU) >> 2)
  86
  87 #define MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color)\
  88         ((enum rte_meter_color)((data)->stats[(color)] & 3LLU))
  89
  90 static void
  91 mtr_trtcm_data_policer_action_set(struct mtr_trtcm_data *data,
  92         enum rte_meter_color color,
  93         enum rte_table_action_policer action)
  94 {
  95         if (action == RTE_TABLE_ACTION_POLICER_DROP) {
  96                 data->stats[color] |= 4LLU;
  97         } else {
  98                 data->stats[color] &= ~7LLU;
  99                 data->stats[color] |= color & 3LLU;
 100         }
 101 }
 102
 103 static uint64_t
 104 mtr_trtcm_data_stats_get(struct mtr_trtcm_data *data,
 105         enum rte_meter_color color)
 106 {
 107         return data->stats[color] >> 8;
 108 }
 109
 110 static void
 111 mtr_trtcm_data_stats_reset(struct mtr_trtcm_data *data,
 112         enum rte_meter_color color)
 113 {
 114         data->stats[color] &= 0xFFLU;
 115 }
 116
 117 #define MTR_TRTCM_DATA_STATS_INC(data, color)              \
 118         ((data)->stats[(color)] += (1LLU << 8))
 119
 120 static size_t
 121 mtr_data_size(struct rte_table_action_mtr_config *mtr)
 122 {
 123         return mtr->n_tc * sizeof(struct mtr_trtcm_data);
 124 }
 125
 126 struct dscp_table_entry_data {
 127         enum rte_meter_color color;
 128         uint16_t tc;
 129         uint16_t queue_tc_color;
 130 };
 131
 132 struct dscp_table_data {
 133         struct dscp_table_entry_data entry[64];
 134 };
 135
 136 struct meter_profile_data {
 137         struct rte_meter_trtcm_profile profile;
 138         uint32_t profile_id;
 139         int valid;
 140 };
 141
 142 static struct meter_profile_data *
 143 meter_profile_data_find(struct meter_profile_data *mp,
 144         uint32_t mp_size,
 145         uint32_t profile_id)
 146 {
 147         uint32_t i;
 148
 149         for (i = 0; i < mp_size; i++) {
 150                 struct meter_profile_data *mp_data = &mp[i];
 151
 152                 if (mp_data->valid && (mp_data->profile_id == profile_id))
 153                         return mp_data;
 154         }
 155
 156         return NULL;
 157 }
 158
 159 static struct meter_profile_data *
 160 meter_profile_data_find_unused(struct meter_profile_data *mp,
 161         uint32_t mp_size)
 162 {
 163         uint32_t i;
 164
 165         for (i = 0; i < mp_size; i++) {
 166                 struct meter_profile_data *mp_data = &mp[i];
 167
 168                 if (!mp_data->valid)
 169                         return mp_data;
 170         }
 171
 172         return NULL;
 173 }
 174
 175 static int
 176 mtr_apply_check(struct rte_table_action_mtr_params *p,
 177         struct rte_table_action_mtr_config *cfg,
 178         struct meter_profile_data *mp,
 179         uint32_t mp_size)
 180 {
 181         uint32_t i;
 182
 183         if (p->tc_mask > RTE_LEN2MASK(cfg->n_tc, uint32_t))
 184                 return -EINVAL;
 185
 186         for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) {
 187                 struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i];
 188                 struct meter_profile_data *mp_data;
 189
 190                 if ((p->tc_mask & (1LLU << i)) == 0)
 191                         continue;
 192
 193                 mp_data = meter_profile_data_find(mp,
 194                         mp_size,
 195                         p_tc->meter_profile_id);
 196                 if (!mp_data)
 197                         return -EINVAL;
 198         }
 199
 200         return 0;
 201 }
 202
 203 static int
 204 mtr_apply(struct mtr_trtcm_data *data,
 205         struct rte_table_action_mtr_params *p,
 206         struct rte_table_action_mtr_config *cfg,
 207         struct meter_profile_data *mp,
 208         uint32_t mp_size)
 209 {
 210         uint32_t i;
 211         int status;
 212
 213         /* Check input arguments */
 214         status = mtr_apply_check(p, cfg, mp, mp_size);
 215         if (status)
 216                 return status;
 217
 218         /* Apply */
 219         for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) {
 220                 struct rte_table_action_mtr_tc_params *p_tc = &p->mtr[i];
 221                 struct mtr_trtcm_data *data_tc = &data[i];
 222                 struct meter_profile_data *mp_data;
 223
 224                 if ((p->tc_mask & (1LLU << i)) == 0)
 225                         continue;
 226
 227                 /* Find profile */
 228                 mp_data = meter_profile_data_find(mp,
 229                         mp_size,
 230                         p_tc->meter_profile_id);
 231                 if (!mp_data)
 232                         return -EINVAL;
 233
 234                 memset(data_tc, 0, sizeof(*data_tc));
 235
 236                 /* Meter object */
 237                 status = rte_meter_trtcm_config(&data_tc->trtcm,
 238                         &mp_data->profile);
 239                 if (status)
 240                         return status;
 241
 242                 /* Meter profile */
 243                 mtr_trtcm_data_meter_profile_id_set(data_tc,
 244                         mp_data - mp);
 245
 246                 /* Policer actions */
 247                 mtr_trtcm_data_policer_action_set(data_tc,
 248                         e_RTE_METER_GREEN,
 249                         p_tc->policer[e_RTE_METER_GREEN]);
 250
 251                 mtr_trtcm_data_policer_action_set(data_tc,
 252                         e_RTE_METER_YELLOW,
 253                         p_tc->policer[e_RTE_METER_YELLOW]);
 254
 255                 mtr_trtcm_data_policer_action_set(data_tc,
 256                         e_RTE_METER_RED,
 257                         p_tc->policer[e_RTE_METER_RED]);
 258         }
 259
 260         return 0;
 261 }
 262
 263 static __rte_always_inline uint64_t
 264 pkt_work_mtr(struct rte_mbuf *mbuf,
 265         struct mtr_trtcm_data *data,
 266         struct dscp_table_data *dscp_table,
 267         struct meter_profile_data *mp,
 268         uint64_t time,
 269         uint32_t dscp,
 270         uint16_t total_length)
 271 {
 272         uint64_t drop_mask, sched;
 273         uint64_t *sched_ptr = (uint64_t *) &mbuf->hash.sched;
 274         struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp];
 275         enum rte_meter_color color_in, color_meter, color_policer;
 276         uint32_t tc, mp_id;
 277
 278         tc = dscp_entry->tc;
 279         color_in = dscp_entry->color;
 280         data += tc;
 281         mp_id = MTR_TRTCM_DATA_METER_PROFILE_ID_GET(data);
 282         sched = *sched_ptr;
 283
 284         /* Meter */
 285         color_meter = rte_meter_trtcm_color_aware_check(
 286                 &data->trtcm,
 287                 &mp[mp_id].profile,
 288                 time,
 289                 total_length,
 290                 color_in);
 291
 292         /* Stats */
 293         MTR_TRTCM_DATA_STATS_INC(data, color_meter);
 294
 295         /* Police */
 296         drop_mask = MTR_TRTCM_DATA_POLICER_ACTION_DROP_GET(data, color_meter);
 297         color_policer =
 298                 MTR_TRTCM_DATA_POLICER_ACTION_COLOR_GET(data, color_meter);
 299         *sched_ptr = MBUF_SCHED_COLOR(sched, color_policer);
 300
 301         return drop_mask;
 302 }
 303
 304 /**
 305  * RTE_TABLE_ACTION_TM
 306  */
 307 static int
 308 tm_cfg_check(struct rte_table_action_tm_config *tm)
 309 {
 310         if ((tm->n_subports_per_port == 0) ||
 311                 (rte_is_power_of_2(tm->n_subports_per_port) == 0) ||
 312                 (tm->n_subports_per_port > UINT16_MAX) ||
 313                 (tm->n_pipes_per_subport == 0) ||
 314                 (rte_is_power_of_2(tm->n_pipes_per_subport) == 0))
 315                 return -ENOTSUP;
 316
 317         return 0;
 318 }
 319
 320 struct tm_data {
 321         uint16_t queue_tc_color;
 322         uint16_t subport;
 323         uint32_t pipe;
 324 } __attribute__((__packed__));
 325
 326 static int
 327 tm_apply_check(struct rte_table_action_tm_params *p,
 328         struct rte_table_action_tm_config *cfg)
 329 {
 330         if ((p->subport_id >= cfg->n_subports_per_port) ||
 331                 (p->pipe_id >= cfg->n_pipes_per_subport))
 332                 return -EINVAL;
 333
 334         return 0;
 335 }
 336
 337 static int
 338 tm_apply(struct tm_data *data,
 339         struct rte_table_action_tm_params *p,
 340         struct rte_table_action_tm_config *cfg)
 341 {
 342         int status;
 343
 344         /* Check input arguments */
 345         status = tm_apply_check(p, cfg);
 346         if (status)
 347                 return status;
 348
 349         /* Apply */
 350         data->queue_tc_color = 0;
 351         data->subport = (uint16_t) p->subport_id;
 352         data->pipe = p->pipe_id;
 353
 354         return 0;
 355 }
 356
 357 static __rte_always_inline void
 358 pkt_work_tm(struct rte_mbuf *mbuf,
 359         struct tm_data *data,
 360         struct dscp_table_data *dscp_table,
 361         uint32_t dscp)
 362 {
 363         struct dscp_table_entry_data *dscp_entry = &dscp_table->entry[dscp];
 364         struct tm_data *sched_ptr = (struct tm_data *) &mbuf->hash.sched;
 365         struct tm_data sched;
 366
 367         sched = *data;
 368         sched.queue_tc_color = dscp_entry->queue_tc_color;
 369         *sched_ptr = sched;
 370 }
 371
 372 /**
 373  * RTE_TABLE_ACTION_ENCAP
 374  */
 375 static int
 376 encap_valid(enum rte_table_action_encap_type encap)
 377 {
 378         switch (encap) {
 379         case RTE_TABLE_ACTION_ENCAP_ETHER:
 380         case RTE_TABLE_ACTION_ENCAP_VLAN:
 381         case RTE_TABLE_ACTION_ENCAP_QINQ:
 382         case RTE_TABLE_ACTION_ENCAP_MPLS:
 383         case RTE_TABLE_ACTION_ENCAP_PPPOE:
 384                 return 1;
 385         default:
 386                 return 0;
 387         }
 388 }
 389
 390 static int
 391 encap_cfg_check(struct rte_table_action_encap_config *encap)
 392 {
 393         if ((encap->encap_mask == 0) ||
 394                 (__builtin_popcountll(encap->encap_mask) != 1))
 395                 return -ENOTSUP;
 396
 397         return 0;
 398 }
 399
 400 struct encap_ether_data {
 401         struct ether_hdr ether;
 402 } __attribute__((__packed__));
 403
 404 #define VLAN(pcp, dei, vid)                                \
 405         ((uint16_t)((((uint64_t)(pcp)) & 0x7LLU) << 13) |  \
 406         ((((uint64_t)(dei)) & 0x1LLU) << 12) |             \
 407         (((uint64_t)(vid)) & 0xFFFLLU))                    \
 408
 409 struct encap_vlan_data {
 410         struct ether_hdr ether;
 411         struct vlan_hdr vlan;
 412 } __attribute__((__packed__));
 413
 414 struct encap_qinq_data {
 415         struct ether_hdr ether;
 416         struct vlan_hdr svlan;
 417         struct vlan_hdr cvlan;
 418 } __attribute__((__packed__));
 419
 420 #define ETHER_TYPE_MPLS_UNICAST                            0x8847
 421
 422 #define ETHER_TYPE_MPLS_MULTICAST                          0x8848
 423
 424 #define MPLS(label, tc, s, ttl)                            \
 425         ((uint32_t)(((((uint64_t)(label)) & 0xFFFFFLLU) << 12) |\
 426         ((((uint64_t)(tc)) & 0x7LLU) << 9) |               \
 427         ((((uint64_t)(s)) & 0x1LLU) << 8) |                \
 428         (((uint64_t)(ttl)) & 0xFFLLU)))
 429
 430 struct encap_mpls_data {
 431         struct ether_hdr ether;
 432         uint32_t mpls[RTE_TABLE_ACTION_MPLS_LABELS_MAX];
 433         uint32_t mpls_count;
 434 } __attribute__((__packed__));
 435
 436 #define ETHER_TYPE_PPPOE_SESSION                           0x8864
 437
 438 #define PPP_PROTOCOL_IP                                    0x0021
 439
 440 struct pppoe_ppp_hdr {
 441         uint16_t ver_type_code;
 442         uint16_t session_id;
 443         uint16_t length;
 444         uint16_t protocol;
 445 } __attribute__((__packed__));
 446
 447 struct encap_pppoe_data {
 448         struct ether_hdr ether;
 449         struct pppoe_ppp_hdr pppoe_ppp;
 450 } __attribute__((__packed__));
 451
 452 static size_t
 453 encap_data_size(struct rte_table_action_encap_config *encap)
 454 {
 455         switch (encap->encap_mask) {
 456         case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER:
 457                 return sizeof(struct encap_ether_data);
 458
 459         case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN:
 460                 return sizeof(struct encap_vlan_data);
 461
 462         case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ:
 463                 return sizeof(struct encap_qinq_data);
 464
 465         case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS:
 466                 return sizeof(struct encap_mpls_data);
 467
 468         case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE:
 469                 return sizeof(struct encap_pppoe_data);
 470
 471         default:
 472                 return 0;
 473         }
 474 }
 475
 476 static int
 477 encap_apply_check(struct rte_table_action_encap_params *p,
 478         struct rte_table_action_encap_config *cfg)
 479 {
 480         if ((encap_valid(p->type) == 0) ||
 481                 ((cfg->encap_mask & (1LLU << p->type)) == 0))
 482                 return -EINVAL;
 483
 484         switch (p->type) {
 485         case RTE_TABLE_ACTION_ENCAP_ETHER:
 486                 return 0;
 487
 488         case RTE_TABLE_ACTION_ENCAP_VLAN:
 489                 return 0;
 490
 491         case RTE_TABLE_ACTION_ENCAP_QINQ:
 492                 return 0;
 493
 494         case RTE_TABLE_ACTION_ENCAP_MPLS:
 495                 if ((p->mpls.mpls_count == 0) ||
 496                         (p->mpls.mpls_count > RTE_TABLE_ACTION_MPLS_LABELS_MAX))
 497                         return -EINVAL;
 498
 499                 return 0;
 500
 501         case RTE_TABLE_ACTION_ENCAP_PPPOE:
 502                 return 0;
 503
 504         default:
 505                 return -EINVAL;
 506         }
 507 }
 508
 509 static int
 510 encap_ether_apply(void *data,
 511         struct rte_table_action_encap_params *p,
 512         struct rte_table_action_common_config *common_cfg)
 513 {
 514         struct encap_ether_data *d = data;
 515         uint16_t ethertype = (common_cfg->ip_version) ?
 516                 ETHER_TYPE_IPv4 :
 517                 ETHER_TYPE_IPv6;
 518
 519         /* Ethernet */
 520         ether_addr_copy(&p->ether.ether.da, &d->ether.d_addr);
 521         ether_addr_copy(&p->ether.ether.sa, &d->ether.s_addr);
 522         d->ether.ether_type = rte_htons(ethertype);
 523
 524         return 0;
 525 }
 526
 527 static int
 528 encap_vlan_apply(void *data,
 529         struct rte_table_action_encap_params *p,
 530         struct rte_table_action_common_config *common_cfg)
 531 {
 532         struct encap_vlan_data *d = data;
 533         uint16_t ethertype = (common_cfg->ip_version) ?
 534                 ETHER_TYPE_IPv4 :
 535                 ETHER_TYPE_IPv6;
 536
 537         /* Ethernet */
 538         ether_addr_copy(&p->vlan.ether.da, &d->ether.d_addr);
 539         ether_addr_copy(&p->vlan.ether.sa, &d->ether.s_addr);
 540         d->ether.ether_type = rte_htons(ETHER_TYPE_VLAN);
 541
 542         /* VLAN */
 543         d->vlan.vlan_tci = rte_htons(VLAN(p->vlan.vlan.pcp,
 544                 p->vlan.vlan.dei,
 545                 p->vlan.vlan.vid));
 546         d->vlan.eth_proto = rte_htons(ethertype);
 547
 548         return 0;
 549 }
 550
 551 static int
 552 encap_qinq_apply(void *data,
 553         struct rte_table_action_encap_params *p,
 554         struct rte_table_action_common_config *common_cfg)
 555 {
 556         struct encap_qinq_data *d = data;
 557         uint16_t ethertype = (common_cfg->ip_version) ?
 558                 ETHER_TYPE_IPv4 :
 559                 ETHER_TYPE_IPv6;
 560
 561         /* Ethernet */
 562         ether_addr_copy(&p->qinq.ether.da, &d->ether.d_addr);
 563         ether_addr_copy(&p->qinq.ether.sa, &d->ether.s_addr);
 564         d->ether.ether_type = rte_htons(ETHER_TYPE_QINQ);
 565
 566         /* SVLAN */
 567         d->svlan.vlan_tci = rte_htons(VLAN(p->qinq.svlan.pcp,
 568                 p->qinq.svlan.dei,
 569                 p->qinq.svlan.vid));
 570         d->svlan.eth_proto = rte_htons(ETHER_TYPE_VLAN);
 571
 572         /* CVLAN */
 573         d->cvlan.vlan_tci = rte_htons(VLAN(p->qinq.cvlan.pcp,
 574                 p->qinq.cvlan.dei,
 575                 p->qinq.cvlan.vid));
 576         d->cvlan.eth_proto = rte_htons(ethertype);
 577
 578         return 0;
 579 }
 580
 581 static int
 582 encap_mpls_apply(void *data,
 583         struct rte_table_action_encap_params *p)
 584 {
 585         struct encap_mpls_data *d = data;
 586         uint16_t ethertype = (p->mpls.unicast) ?
 587                 ETHER_TYPE_MPLS_UNICAST :
 588                 ETHER_TYPE_MPLS_MULTICAST;
 589         uint32_t i;
 590
 591         /* Ethernet */
 592         ether_addr_copy(&p->mpls.ether.da, &d->ether.d_addr);
 593         ether_addr_copy(&p->mpls.ether.sa, &d->ether.s_addr);
 594         d->ether.ether_type = rte_htons(ethertype);
 595
 596         /* MPLS */
 597         for (i = 0; i < p->mpls.mpls_count - 1; i++)
 598                 d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label,
 599                         p->mpls.mpls[i].tc,
 600                         0,
 601                         p->mpls.mpls[i].ttl));
 602
 603         d->mpls[i] = rte_htonl(MPLS(p->mpls.mpls[i].label,
 604                 p->mpls.mpls[i].tc,
 605                 1,
 606                 p->mpls.mpls[i].ttl));
 607
 608         d->mpls_count = p->mpls.mpls_count;
 609         return 0;
 610 }
 611
 612 static int
 613 encap_pppoe_apply(void *data,
 614         struct rte_table_action_encap_params *p)
 615 {
 616         struct encap_pppoe_data *d = data;
 617
 618         /* Ethernet */
 619         ether_addr_copy(&p->pppoe.ether.da, &d->ether.d_addr);
 620         ether_addr_copy(&p->pppoe.ether.sa, &d->ether.s_addr);
 621         d->ether.ether_type = rte_htons(ETHER_TYPE_PPPOE_SESSION);
 622
 623         /* PPPoE and PPP*/
 624         d->pppoe_ppp.ver_type_code = rte_htons(0x1100);
 625         d->pppoe_ppp.session_id = rte_htons(p->pppoe.pppoe.session_id);
 626         d->pppoe_ppp.length = 0; /* not pre-computed */
 627         d->pppoe_ppp.protocol = rte_htons(PPP_PROTOCOL_IP);
 628
 629         return 0;
 630 }
 631
 632 static int
 633 encap_apply(void *data,
 634         struct rte_table_action_encap_params *p,
 635         struct rte_table_action_encap_config *cfg,
 636         struct rte_table_action_common_config *common_cfg)
 637 {
 638         int status;
 639
 640         /* Check input arguments */
 641         status = encap_apply_check(p, cfg);
 642         if (status)
 643                 return status;
 644
 645         switch (p->type) {
 646         case RTE_TABLE_ACTION_ENCAP_ETHER:
 647                 return encap_ether_apply(data, p, common_cfg);
 648
 649         case RTE_TABLE_ACTION_ENCAP_VLAN:
 650                 return encap_vlan_apply(data, p, common_cfg);
 651
 652         case RTE_TABLE_ACTION_ENCAP_QINQ:
 653                 return encap_qinq_apply(data, p, common_cfg);
 654
 655         case RTE_TABLE_ACTION_ENCAP_MPLS:
 656                 return encap_mpls_apply(data, p);
 657
 658         case RTE_TABLE_ACTION_ENCAP_PPPOE:
 659                 return encap_pppoe_apply(data, p);
 660
 661         default:
 662                 return -EINVAL;
 663         }
 664 }
 665
 666 static __rte_always_inline void *
 667 encap(void *dst, const void *src, size_t n)
 668 {
 669         dst = ((uint8_t *) dst) - n;
 670         return rte_memcpy(dst, src, n);
 671 }
 672
 673 static __rte_always_inline void
 674 pkt_work_encap(struct rte_mbuf *mbuf,
 675         void *data,
 676         struct rte_table_action_encap_config *cfg,
 677         void *ip,
 678         uint16_t total_length,
 679         uint32_t ip_offset)
 680 {
 681         switch (cfg->encap_mask) {
 682         case 1LLU << RTE_TABLE_ACTION_ENCAP_ETHER:
 683                 encap(ip, data, sizeof(struct encap_ether_data));
 684                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) +
 685                         sizeof(struct encap_ether_data));
 686                 mbuf->pkt_len = mbuf->data_len = total_length +
 687                         sizeof(struct encap_ether_data);
 688                 break;
 689
 690         case 1LLU << RTE_TABLE_ACTION_ENCAP_VLAN:
 691                 encap(ip, data, sizeof(struct encap_vlan_data));
 692                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) +
 693                         sizeof(struct encap_vlan_data));
 694                 mbuf->pkt_len = mbuf->data_len = total_length +
 695                         sizeof(struct encap_vlan_data);
 696                 break;
 697
 698         case 1LLU << RTE_TABLE_ACTION_ENCAP_QINQ:
 699                 encap(ip, data, sizeof(struct encap_qinq_data));
 700                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) +
 701                         sizeof(struct encap_qinq_data));
 702                 mbuf->pkt_len = mbuf->data_len = total_length +
 703                         sizeof(struct encap_qinq_data);
 704                 break;
 705
 706         case 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS:
 707         {
 708                 struct encap_mpls_data *mpls = data;
 709                 size_t size = sizeof(struct ether_hdr) +
 710                         mpls->mpls_count * 4;
 711
 712                 encap(ip, data, size);
 713                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) + size);
 714                 mbuf->pkt_len = mbuf->data_len = total_length + size;
 715                 break;
 716         }
 717
 718         case 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE:
 719         {
 720                 struct encap_pppoe_data *pppoe =
 721                         encap(ip, data, sizeof(struct encap_pppoe_data));
 722                 pppoe->pppoe_ppp.length = rte_htons(total_length + 2);
 723                 mbuf->data_off = ip_offset - (sizeof(struct rte_mbuf) +
 724                         sizeof(struct encap_pppoe_data));
 725                 mbuf->pkt_len = mbuf->data_len = total_length +
 726                         sizeof(struct encap_pppoe_data);
 727                 break;
 728         }
 729
 730         default:
 731                 break;
 732         }
 733 }
 734
 735 /**
 736  * RTE_TABLE_ACTION_NAT
 737  */
 738 static int
 739 nat_cfg_check(struct rte_table_action_nat_config *nat)
 740 {
 741         if ((nat->proto != 0x06) &&
 742                 (nat->proto != 0x11))
 743                 return -ENOTSUP;
 744
 745         return 0;
 746 }
 747
 748 struct nat_ipv4_data {
 749         uint32_t addr;
 750         uint16_t port;
 751 } __attribute__((__packed__));
 752
 753 struct nat_ipv6_data {
 754         uint8_t addr[16];
 755         uint16_t port;
 756 } __attribute__((__packed__));
 757
 758 static size_t
 759 nat_data_size(struct rte_table_action_nat_config *nat __rte_unused,
 760         struct rte_table_action_common_config *common)
 761 {
 762         int ip_version = common->ip_version;
 763
 764         return (ip_version) ?
 765                 sizeof(struct nat_ipv4_data) :
 766                 sizeof(struct nat_ipv6_data);
 767 }
 768
 769 static int
 770 nat_apply_check(struct rte_table_action_nat_params *p,
 771         struct rte_table_action_common_config *cfg)
 772 {
 773         if ((p->ip_version && (cfg->ip_version == 0)) ||
 774                 ((p->ip_version == 0) && cfg->ip_version))
 775                 return -EINVAL;
 776
 777         return 0;
 778 }
 779
 780 static int
 781 nat_apply(void *data,
 782         struct rte_table_action_nat_params *p,
 783         struct rte_table_action_common_config *cfg)
 784 {
 785         int status;
 786
 787         /* Check input arguments */
 788         status = nat_apply_check(p, cfg);
 789         if (status)
 790                 return status;
 791
 792         /* Apply */
 793         if (p->ip_version) {
 794                 struct nat_ipv4_data *d = data;
 795
 796                 d->addr = rte_htonl(p->addr.ipv4);
 797                 d->port = rte_htons(p->port);
 798         } else {
 799                 struct nat_ipv6_data *d = data;
 800
 801                 memcpy(d->addr, p->addr.ipv6, sizeof(d->addr));
 802                 d->port = rte_htons(p->port);
 803         }
 804
 805         return 0;
 806 }
 807
 808 static __rte_always_inline uint16_t
 809 nat_ipv4_checksum_update(uint16_t cksum0,
 810         uint32_t ip0,
 811         uint32_t ip1)
 812 {
 813         int32_t cksum1;
 814
 815         cksum1 = cksum0;
 816         cksum1 = ~cksum1 & 0xFFFF;
 817
 818         /* Subtract ip0 (one's complement logic) */
 819         cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF);
 820         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 821         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 822
 823         /* Add ip1 (one's complement logic) */
 824         cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF);
 825         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 826         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 827
 828         return (uint16_t)(~cksum1);
 829 }
 830
 831 static __rte_always_inline uint16_t
 832 nat_ipv4_tcp_udp_checksum_update(uint16_t cksum0,
 833         uint32_t ip0,
 834         uint32_t ip1,
 835         uint16_t port0,
 836         uint16_t port1)
 837 {
 838         int32_t cksum1;
 839
 840         cksum1 = cksum0;
 841         cksum1 = ~cksum1 & 0xFFFF;
 842
 843         /* Subtract ip0 and port 0 (one's complement logic) */
 844         cksum1 -= (ip0 >> 16) + (ip0 & 0xFFFF) + port0;
 845         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 846         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 847
 848         /* Add ip1 and port1 (one's complement logic) */
 849         cksum1 += (ip1 >> 16) + (ip1 & 0xFFFF) + port1;
 850         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 851         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 852
 853         return (uint16_t)(~cksum1);
 854 }
 855
 856 static __rte_always_inline uint16_t
 857 nat_ipv6_tcp_udp_checksum_update(uint16_t cksum0,
 858         uint16_t *ip0,
 859         uint16_t *ip1,
 860         uint16_t port0,
 861         uint16_t port1)
 862 {
 863         int32_t cksum1;
 864
 865         cksum1 = cksum0;
 866         cksum1 = ~cksum1 & 0xFFFF;
 867
 868         /* Subtract ip0 and port 0 (one's complement logic) */
 869         cksum1 -= ip0[0] + ip0[1] + ip0[2] + ip0[3] +
 870                 ip0[4] + ip0[5] + ip0[6] + ip0[7] + port0;
 871         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 872         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 873
 874         /* Add ip1 and port1 (one's complement logic) */
 875         cksum1 += ip1[0] + ip1[1] + ip1[2] + ip1[3] +
 876                 ip1[4] + ip1[5] + ip1[6] + ip1[7] + port1;
 877         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 878         cksum1 = (cksum1 & 0xFFFF) + (cksum1 >> 16);
 879
 880         return (uint16_t)(~cksum1);
 881 }
 882
 883 static __rte_always_inline void
 884 pkt_ipv4_work_nat(struct ipv4_hdr *ip,
 885         struct nat_ipv4_data *data,
 886         struct rte_table_action_nat_config *cfg)
 887 {
 888         if (cfg->source_nat) {
 889                 if (cfg->proto == 0x6) {
 890                         struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1];
 891                         uint16_t ip_cksum, tcp_cksum;
 892
 893                         ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum,
 894                                 ip->src_addr,
 895                                 data->addr);
 896
 897                         tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum,
 898                                 ip->src_addr,
 899                                 data->addr,
 900                                 tcp->src_port,
 901                                 data->port);
 902
 903                         ip->src_addr = data->addr;
 904                         ip->hdr_checksum = ip_cksum;
 905                         tcp->src_port = data->port;
 906                         tcp->cksum = tcp_cksum;
 907                 } else {
 908                         struct udp_hdr *udp = (struct udp_hdr *) &ip[1];
 909                         uint16_t ip_cksum, udp_cksum;
 910
 911                         ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum,
 912                                 ip->src_addr,
 913                                 data->addr);
 914
 915                         udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum,
 916                                 ip->src_addr,
 917                                 data->addr,
 918                                 udp->src_port,
 919                                 data->port);
 920
 921                         ip->src_addr = data->addr;
 922                         ip->hdr_checksum = ip_cksum;
 923                         udp->src_port = data->port;
 924                         if (udp->dgram_cksum)
 925                                 udp->dgram_cksum = udp_cksum;
 926                 }
 927         } else {
 928                 if (cfg->proto == 0x6) {
 929                         struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1];
 930                         uint16_t ip_cksum, tcp_cksum;
 931
 932                         ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum,
 933                                 ip->dst_addr,
 934                                 data->addr);
 935
 936                         tcp_cksum = nat_ipv4_tcp_udp_checksum_update(tcp->cksum,
 937                                 ip->dst_addr,
 938                                 data->addr,
 939                                 tcp->dst_port,
 940                                 data->port);
 941
 942                         ip->dst_addr = data->addr;
 943                         ip->hdr_checksum = ip_cksum;
 944                         tcp->dst_port = data->port;
 945                         tcp->cksum = tcp_cksum;
 946                 } else {
 947                         struct udp_hdr *udp = (struct udp_hdr *) &ip[1];
 948                         uint16_t ip_cksum, udp_cksum;
 949
 950                         ip_cksum = nat_ipv4_checksum_update(ip->hdr_checksum,
 951                                 ip->dst_addr,
 952                                 data->addr);
 953
 954                         udp_cksum = nat_ipv4_tcp_udp_checksum_update(udp->dgram_cksum,
 955                                 ip->dst_addr,
 956                                 data->addr,
 957                                 udp->dst_port,
 958                                 data->port);
 959
 960                         ip->dst_addr = data->addr;
 961                         ip->hdr_checksum = ip_cksum;
 962                         udp->dst_port = data->port;
 963                         if (udp->dgram_cksum)
 964                                 udp->dgram_cksum = udp_cksum;
 965                 }
 966         }
 967 }
 968
 969 static __rte_always_inline void
 970 pkt_ipv6_work_nat(struct ipv6_hdr *ip,
 971         struct nat_ipv6_data *data,
 972         struct rte_table_action_nat_config *cfg)
 973 {
 974         if (cfg->source_nat) {
 975                 if (cfg->proto == 0x6) {
 976                         struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1];
 977                         uint16_t tcp_cksum;
 978
 979                         tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum,
 980                                 (uint16_t *)ip->src_addr,
 981                                 (uint16_t *)data->addr,
 982                                 tcp->src_port,
 983                                 data->port);
 984
 985                         rte_memcpy(ip->src_addr, data->addr, 16);
 986                         tcp->src_port = data->port;
 987                         tcp->cksum = tcp_cksum;
 988                 } else {
 989                         struct udp_hdr *udp = (struct udp_hdr *) &ip[1];
 990                         uint16_t udp_cksum;
 991
 992                         udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum,
 993                                 (uint16_t *)ip->src_addr,
 994                                 (uint16_t *)data->addr,
 995                                 udp->src_port,
 996                                 data->port);
 997
 998                         rte_memcpy(ip->src_addr, data->addr, 16);
 999                         udp->src_port = data->port;
1000                         udp->dgram_cksum = udp_cksum;
1001                 }
1002         } else {
1003                 if (cfg->proto == 0x6) {
1004                         struct tcp_hdr *tcp = (struct tcp_hdr *) &ip[1];
1005                         uint16_t tcp_cksum;
1006
1007                         tcp_cksum = nat_ipv6_tcp_udp_checksum_update(tcp->cksum,
1008                                 (uint16_t *)ip->dst_addr,
1009                                 (uint16_t *)data->addr,
1010                                 tcp->dst_port,
1011                                 data->port);
1012
1013                         rte_memcpy(ip->dst_addr, data->addr, 16);
1014                         tcp->dst_port = data->port;
1015                         tcp->cksum = tcp_cksum;
1016                 } else {
1017                         struct udp_hdr *udp = (struct udp_hdr *) &ip[1];
1018                         uint16_t udp_cksum;
1019
1020                         udp_cksum = nat_ipv6_tcp_udp_checksum_update(udp->dgram_cksum,
1021                                 (uint16_t *)ip->dst_addr,
1022                                 (uint16_t *)data->addr,
1023                                 udp->dst_port,
1024                                 data->port);
1025
1026                         rte_memcpy(ip->dst_addr, data->addr, 16);
1027                         udp->dst_port = data->port;
1028                         udp->dgram_cksum = udp_cksum;
1029                 }
1030         }
1031 }
1032
1033 /**
1034  * Action profile
1035  */
1036 static int
1037 action_valid(enum rte_table_action_type action)
1038 {
1039         switch (action) {
1040         case RTE_TABLE_ACTION_FWD:
1041         case RTE_TABLE_ACTION_MTR:
1042         case RTE_TABLE_ACTION_TM:
1043         case RTE_TABLE_ACTION_ENCAP:
1044         case RTE_TABLE_ACTION_NAT:
1045                 return 1;
1046         default:
1047                 return 0;
1048         }
1049 }
1050
1051
1052 #define RTE_TABLE_ACTION_MAX                      64
1053
1054 struct ap_config {
1055         uint64_t action_mask;
1056         struct rte_table_action_common_config common;
1057         struct rte_table_action_mtr_config mtr;
1058         struct rte_table_action_tm_config tm;
1059         struct rte_table_action_encap_config encap;
1060         struct rte_table_action_nat_config nat;
1061 };
1062
1063 static size_t
1064 action_cfg_size(enum rte_table_action_type action)
1065 {
1066         switch (action) {
1067         case RTE_TABLE_ACTION_MTR:
1068                 return sizeof(struct rte_table_action_mtr_config);
1069         case RTE_TABLE_ACTION_TM:
1070                 return sizeof(struct rte_table_action_tm_config);
1071         case RTE_TABLE_ACTION_ENCAP:
1072                 return sizeof(struct rte_table_action_encap_config);
1073         case RTE_TABLE_ACTION_NAT:
1074                 return sizeof(struct rte_table_action_nat_config);
1075         default:
1076                 return 0;
1077         }
1078 }
1079
1080 static void*
1081 action_cfg_get(struct ap_config *ap_config,
1082         enum rte_table_action_type type)
1083 {
1084         switch (type) {
1085         case RTE_TABLE_ACTION_MTR:
1086                 return &ap_config->mtr;
1087
1088         case RTE_TABLE_ACTION_TM:
1089                 return &ap_config->tm;
1090
1091         case RTE_TABLE_ACTION_ENCAP:
1092                 return &ap_config->encap;
1093
1094         case RTE_TABLE_ACTION_NAT:
1095                 return &ap_config->nat;
1096
1097         default:
1098                 return NULL;
1099         }
1100 }
1101
1102 static void
1103 action_cfg_set(struct ap_config *ap_config,
1104         enum rte_table_action_type type,
1105         void *action_cfg)
1106 {
1107         void *dst = action_cfg_get(ap_config, type);
1108
1109         if (dst)
1110                 memcpy(dst, action_cfg, action_cfg_size(type));
1111
1112         ap_config->action_mask |= 1LLU << type;
1113 }
1114
1115 struct ap_data {
1116         size_t offset[RTE_TABLE_ACTION_MAX];
1117         size_t total_size;
1118 };
1119
1120 static size_t
1121 action_data_size(enum rte_table_action_type action,
1122         struct ap_config *ap_config)
1123 {
1124         switch (action) {
1125         case RTE_TABLE_ACTION_FWD:
1126                 return sizeof(struct fwd_data);
1127
1128         case RTE_TABLE_ACTION_MTR:
1129                 return mtr_data_size(&ap_config->mtr);
1130
1131         case RTE_TABLE_ACTION_TM:
1132                 return sizeof(struct tm_data);
1133
1134         case RTE_TABLE_ACTION_ENCAP:
1135                 return encap_data_size(&ap_config->encap);
1136
1137         case RTE_TABLE_ACTION_NAT:
1138                 return nat_data_size(&ap_config->nat,
1139                         &ap_config->common);
1140
1141         default:
1142                 return 0;
1143         }
1144 }
1145
1146
1147 static void
1148 action_data_offset_set(struct ap_data *ap_data,
1149         struct ap_config *ap_config)
1150 {
1151         uint64_t action_mask = ap_config->action_mask;
1152         size_t offset;
1153         uint32_t action;
1154
1155         memset(ap_data->offset, 0, sizeof(ap_data->offset));
1156
1157         offset = 0;
1158         for (action = 0; action < RTE_TABLE_ACTION_MAX; action++)
1159                 if (action_mask & (1LLU << action)) {
1160                         ap_data->offset[action] = offset;
1161                         offset += action_data_size((enum rte_table_action_type)action,
1162                                 ap_config);
1163                 }
1164
1165         ap_data->total_size = offset;
1166 }
1167
1168 struct rte_table_action_profile {
1169         struct ap_config cfg;
1170         struct ap_data data;
1171         int frozen;
1172 };
1173
1174 struct rte_table_action_profile *
1175 rte_table_action_profile_create(struct rte_table_action_common_config *common)
1176 {
1177         struct rte_table_action_profile *ap;
1178
1179         /* Check input arguments */
1180         if (common == NULL)
1181                 return NULL;
1182
1183         /* Memory allocation */
1184         ap = calloc(1, sizeof(struct rte_table_action_profile));
1185         if (ap == NULL)
1186                 return NULL;
1187
1188         /* Initialization */
1189         memcpy(&ap->cfg.common, common, sizeof(*common));
1190
1191         return ap;
1192 }
1193
1194
1195 int
1196 rte_table_action_profile_action_register(struct rte_table_action_profile *profile,
1197         enum rte_table_action_type type,
1198         void *action_config)
1199 {
1200         int status;
1201
1202         /* Check input arguments */
1203         if ((profile == NULL) ||
1204                 profile->frozen ||
1205                 (action_valid(type) == 0) ||
1206                 (profile->cfg.action_mask & (1LLU << type)) ||
1207                 ((action_cfg_size(type) == 0) && action_config) ||
1208                 (action_cfg_size(type) && (action_config == NULL)))
1209                 return -EINVAL;
1210
1211         switch (type) {
1212         case RTE_TABLE_ACTION_MTR:
1213                 status = mtr_cfg_check(action_config);
1214                 break;
1215
1216         case RTE_TABLE_ACTION_TM:
1217                 status = tm_cfg_check(action_config);
1218                 break;
1219
1220         case RTE_TABLE_ACTION_ENCAP:
1221                 status = encap_cfg_check(action_config);
1222                 break;
1223
1224         case RTE_TABLE_ACTION_NAT:
1225                 status = nat_cfg_check(action_config);
1226                 break;
1227
1228         default:
1229                 status = 0;
1230                 break;
1231         }
1232
1233         if (status)
1234                 return status;
1235
1236         /* Action enable */
1237         action_cfg_set(&profile->cfg, type, action_config);
1238
1239         return 0;
1240 }
1241
1242 int
1243 rte_table_action_profile_freeze(struct rte_table_action_profile *profile)
1244 {
1245         if (profile->frozen)
1246                 return -EBUSY;
1247
1248         profile->cfg.action_mask |= 1LLU << RTE_TABLE_ACTION_FWD;
1249         action_data_offset_set(&profile->data, &profile->cfg);
1250         profile->frozen = 1;
1251
1252         return 0;
1253 }
1254
1255 int
1256 rte_table_action_profile_free(struct rte_table_action_profile *profile)
1257 {
1258         if (profile == NULL)
1259                 return 0;
1260
1261         free(profile);
1262         return 0;
1263 }
1264
1265 /**
1266  * Action
1267  */
1268 #define METER_PROFILES_MAX                                 32
1269
1270 struct rte_table_action {
1271         struct ap_config cfg;
1272         struct ap_data data;
1273         struct dscp_table_data dscp_table;
1274         struct meter_profile_data mp[METER_PROFILES_MAX];
1275 };
1276
1277 struct rte_table_action *
1278 rte_table_action_create(struct rte_table_action_profile *profile,
1279         uint32_t socket_id)
1280 {
1281         struct rte_table_action *action;
1282
1283         /* Check input arguments */
1284         if ((profile == NULL) ||
1285                 (profile->frozen == 0))
1286                 return NULL;
1287
1288         /* Memory allocation */
1289         action = rte_zmalloc_socket(NULL,
1290                 sizeof(struct rte_table_action),
1291                 RTE_CACHE_LINE_SIZE,
1292                 socket_id);
1293         if (action == NULL)
1294                 return NULL;
1295
1296         /* Initialization */
1297         memcpy(&action->cfg, &profile->cfg, sizeof(profile->cfg));
1298         memcpy(&action->data, &profile->data, sizeof(profile->data));
1299
1300         return action;
1301 }
1302
1303 static __rte_always_inline void *
1304 action_data_get(void *data,
1305         struct rte_table_action *action,
1306         enum rte_table_action_type type)
1307 {
1308         size_t offset = action->data.offset[type];
1309         uint8_t *data_bytes = data;
1310
1311         return &data_bytes[offset];
1312 }
1313
1314 int
1315 rte_table_action_apply(struct rte_table_action *action,
1316         void *data,
1317         enum rte_table_action_type type,
1318         void *action_params)
1319 {
1320         void *action_data;
1321
1322         /* Check input arguments */
1323         if ((action == NULL) ||
1324                 (data == NULL) ||
1325                 (action_valid(type) == 0) ||
1326                 ((action->cfg.action_mask & (1LLU << type)) == 0) ||
1327                 (action_params == NULL))
1328                 return -EINVAL;
1329
1330         /* Data update */
1331         action_data = action_data_get(data, action, type);
1332
1333         switch (type) {
1334         case RTE_TABLE_ACTION_FWD:
1335                 return fwd_apply(action_data,
1336                         action_params);
1337
1338         case RTE_TABLE_ACTION_MTR:
1339                 return mtr_apply(action_data,
1340                         action_params,
1341                         &action->cfg.mtr,
1342                         action->mp,
1343                         RTE_DIM(action->mp));
1344
1345         case RTE_TABLE_ACTION_TM:
1346                 return tm_apply(action_data,
1347                         action_params,
1348                         &action->cfg.tm);
1349
1350         case RTE_TABLE_ACTION_ENCAP:
1351                 return encap_apply(action_data,
1352                         action_params,
1353                         &action->cfg.encap,
1354                         &action->cfg.common);
1355
1356         case RTE_TABLE_ACTION_NAT:
1357                 return nat_apply(action_data,
1358                         action_params,
1359                         &action->cfg.common);
1360
1361         default:
1362                 return -EINVAL;
1363         }
1364 }
1365
1366 int
1367 rte_table_action_dscp_table_update(struct rte_table_action *action,
1368         uint64_t dscp_mask,
1369         struct rte_table_action_dscp_table *table)
1370 {
1371         uint32_t i;
1372
1373         /* Check input arguments */
1374         if ((action == NULL) ||
1375                 ((action->cfg.action_mask & ((1LLU << RTE_TABLE_ACTION_MTR) |
1376                 (1LLU << RTE_TABLE_ACTION_TM))) == 0) ||
1377                 (dscp_mask == 0) ||
1378                 (table == NULL))
1379                 return -EINVAL;
1380
1381         for (i = 0; i < RTE_DIM(table->entry); i++) {
1382                 struct dscp_table_entry_data *data =
1383                         &action->dscp_table.entry[i];
1384                 struct rte_table_action_dscp_table_entry *entry =
1385                         &table->entry[i];
1386                 uint16_t queue_tc_color =
1387                         MBUF_SCHED_QUEUE_TC_COLOR(entry->tc_queue_id,
1388                                 entry->tc_id,
1389                                 entry->color);
1390
1391                 if ((dscp_mask & (1LLU << i)) == 0)
1392                         continue;
1393
1394                 data->color = entry->color;
1395                 data->tc = entry->tc_id;
1396                 data->queue_tc_color = queue_tc_color;
1397         }
1398
1399         return 0;
1400 }
1401
1402 int
1403 rte_table_action_meter_profile_add(struct rte_table_action *action,
1404         uint32_t meter_profile_id,
1405         struct rte_table_action_meter_profile *profile)
1406 {
1407         struct meter_profile_data *mp_data;
1408         uint32_t status;
1409
1410         /* Check input arguments */
1411         if ((action == NULL) ||
1412                 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) ||
1413                 (profile == NULL))
1414                 return -EINVAL;
1415
1416         if (profile->alg != RTE_TABLE_ACTION_METER_TRTCM)
1417                 return -ENOTSUP;
1418
1419         mp_data = meter_profile_data_find(action->mp,
1420                 RTE_DIM(action->mp),
1421                 meter_profile_id);
1422         if (mp_data)
1423                 return -EEXIST;
1424
1425         mp_data = meter_profile_data_find_unused(action->mp,
1426                 RTE_DIM(action->mp));
1427         if (!mp_data)
1428                 return -ENOSPC;
1429
1430         /* Install new profile */
1431         status = rte_meter_trtcm_profile_config(&mp_data->profile,
1432                 &profile->trtcm);
1433         if (status)
1434                 return status;
1435
1436         mp_data->profile_id = meter_profile_id;
1437         mp_data->valid = 1;
1438
1439         return 0;
1440 }
1441
1442 int
1443 rte_table_action_meter_profile_delete(struct rte_table_action *action,
1444         uint32_t meter_profile_id)
1445 {
1446         struct meter_profile_data *mp_data;
1447
1448         /* Check input arguments */
1449         if ((action == NULL) ||
1450                 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0))
1451                 return -EINVAL;
1452
1453         mp_data = meter_profile_data_find(action->mp,
1454                 RTE_DIM(action->mp),
1455                 meter_profile_id);
1456         if (!mp_data)
1457                 return 0;
1458
1459         /* Uninstall profile */
1460         mp_data->valid = 0;
1461
1462         return 0;
1463 }
1464
1465 int
1466 rte_table_action_meter_read(struct rte_table_action *action,
1467         void *data,
1468         uint32_t tc_mask,
1469         struct rte_table_action_mtr_counters *stats,
1470         int clear)
1471 {
1472         struct mtr_trtcm_data *mtr_data;
1473         uint32_t i;
1474
1475         /* Check input arguments */
1476         if ((action == NULL) ||
1477                 ((action->cfg.action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) == 0) ||
1478                 (data == NULL) ||
1479                 (tc_mask > RTE_LEN2MASK(action->cfg.mtr.n_tc, uint32_t)))
1480                 return -EINVAL;
1481
1482         mtr_data = action_data_get(data, action, RTE_TABLE_ACTION_MTR);
1483
1484         /* Read */
1485         if (stats) {
1486                 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) {
1487                         struct rte_table_action_mtr_counters_tc *dst =
1488                                 &stats->stats[i];
1489                         struct mtr_trtcm_data *src = &mtr_data[i];
1490
1491                         if ((tc_mask & (1 << i)) == 0)
1492                                 continue;
1493
1494                         dst->n_packets[e_RTE_METER_GREEN] =
1495                                 mtr_trtcm_data_stats_get(src, e_RTE_METER_GREEN);
1496
1497                         dst->n_packets[e_RTE_METER_YELLOW] =
1498                                 mtr_trtcm_data_stats_get(src, e_RTE_METER_YELLOW);
1499
1500                         dst->n_packets[e_RTE_METER_RED] =
1501                                 mtr_trtcm_data_stats_get(src, e_RTE_METER_RED);
1502
1503                         dst->n_packets_valid = 1;
1504                         dst->n_bytes_valid = 0;
1505                 }
1506
1507                 stats->tc_mask = tc_mask;
1508         }
1509
1510         /* Clear */
1511         if (clear)
1512                 for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++) {
1513                         struct mtr_trtcm_data *src = &mtr_data[i];
1514
1515                         if ((tc_mask & (1 << i)) == 0)
1516                                 continue;
1517
1518                         mtr_trtcm_data_stats_reset(src, e_RTE_METER_GREEN);
1519                         mtr_trtcm_data_stats_reset(src, e_RTE_METER_YELLOW);
1520                         mtr_trtcm_data_stats_reset(src, e_RTE_METER_RED);
1521                 }
1522
1523
1524         return 0;
1525 }
1526
1527 static __rte_always_inline uint64_t
1528 pkt_work(struct rte_mbuf *mbuf,
1529         struct rte_pipeline_table_entry *table_entry,
1530         uint64_t time,
1531         struct rte_table_action *action,
1532         struct ap_config *cfg)
1533 {
1534         uint64_t drop_mask = 0;
1535
1536         uint32_t ip_offset = action->cfg.common.ip_offset;
1537         void *ip = RTE_MBUF_METADATA_UINT32_PTR(mbuf, ip_offset);
1538
1539         uint32_t dscp;
1540         uint16_t total_length;
1541
1542         if (cfg->common.ip_version) {
1543                 struct ipv4_hdr *hdr = ip;
1544
1545                 dscp = hdr->type_of_service >> 2;
1546                 total_length = rte_ntohs(hdr->total_length);
1547         } else {
1548                 struct ipv6_hdr *hdr = ip;
1549
1550                 dscp = (rte_ntohl(hdr->vtc_flow) & 0x0F600000) >> 18;
1551                 total_length =
1552                         rte_ntohs(hdr->payload_len) + sizeof(struct ipv6_hdr);
1553         }
1554
1555         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1556                 void *data =
1557                         action_data_get(table_entry, action, RTE_TABLE_ACTION_MTR);
1558
1559                 drop_mask |= pkt_work_mtr(mbuf,
1560                         data,
1561                         &action->dscp_table,
1562                         action->mp,
1563                         time,
1564                         dscp,
1565                         total_length);
1566         }
1567
1568         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1569                 void *data =
1570                         action_data_get(table_entry, action, RTE_TABLE_ACTION_TM);
1571
1572                 pkt_work_tm(mbuf,
1573                         data,
1574                         &action->dscp_table,
1575                         dscp);
1576         }
1577
1578         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1579                 void *data =
1580                         action_data_get(table_entry, action, RTE_TABLE_ACTION_ENCAP);
1581
1582                 pkt_work_encap(mbuf,
1583                         data,
1584                         &cfg->encap,
1585                         ip,
1586                         total_length,
1587                         ip_offset);
1588         }
1589
1590         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1591                 void *data =
1592                         action_data_get(table_entry, action, RTE_TABLE_ACTION_NAT);
1593
1594                 if (cfg->common.ip_version)
1595                         pkt_ipv4_work_nat(ip, data, &cfg->nat);
1596                 else
1597                         pkt_ipv6_work_nat(ip, data, &cfg->nat);
1598         }
1599
1600         return drop_mask;
1601 }
1602
1603 static __rte_always_inline uint64_t
1604 pkt4_work(struct rte_mbuf **mbufs,
1605         struct rte_pipeline_table_entry **table_entries,
1606         uint64_t time,
1607         struct rte_table_action *action,
1608         struct ap_config *cfg)
1609 {
1610         uint64_t drop_mask0 = 0;
1611         uint64_t drop_mask1 = 0;
1612         uint64_t drop_mask2 = 0;
1613         uint64_t drop_mask3 = 0;
1614
1615         struct rte_mbuf *mbuf0 = mbufs[0];
1616         struct rte_mbuf *mbuf1 = mbufs[1];
1617         struct rte_mbuf *mbuf2 = mbufs[2];
1618         struct rte_mbuf *mbuf3 = mbufs[3];
1619
1620         struct rte_pipeline_table_entry *table_entry0 = table_entries[0];
1621         struct rte_pipeline_table_entry *table_entry1 = table_entries[1];
1622         struct rte_pipeline_table_entry *table_entry2 = table_entries[2];
1623         struct rte_pipeline_table_entry *table_entry3 = table_entries[3];
1624
1625         uint32_t ip_offset = action->cfg.common.ip_offset;
1626         void *ip0 = RTE_MBUF_METADATA_UINT32_PTR(mbuf0, ip_offset);
1627         void *ip1 = RTE_MBUF_METADATA_UINT32_PTR(mbuf1, ip_offset);
1628         void *ip2 = RTE_MBUF_METADATA_UINT32_PTR(mbuf2, ip_offset);
1629         void *ip3 = RTE_MBUF_METADATA_UINT32_PTR(mbuf3, ip_offset);
1630
1631         uint32_t dscp0, dscp1, dscp2, dscp3;
1632         uint16_t total_length0, total_length1, total_length2, total_length3;
1633
1634         if (cfg->common.ip_version) {
1635                 struct ipv4_hdr *hdr0 = ip0;
1636                 struct ipv4_hdr *hdr1 = ip1;
1637                 struct ipv4_hdr *hdr2 = ip2;
1638                 struct ipv4_hdr *hdr3 = ip3;
1639
1640                 dscp0 = hdr0->type_of_service >> 2;
1641                 dscp1 = hdr1->type_of_service >> 2;
1642                 dscp2 = hdr2->type_of_service >> 2;
1643                 dscp3 = hdr3->type_of_service >> 2;
1644
1645                 total_length0 = rte_ntohs(hdr0->total_length);
1646                 total_length1 = rte_ntohs(hdr1->total_length);
1647                 total_length2 = rte_ntohs(hdr2->total_length);
1648                 total_length3 = rte_ntohs(hdr3->total_length);
1649         } else {
1650                 struct ipv6_hdr *hdr0 = ip0;
1651                 struct ipv6_hdr *hdr1 = ip1;
1652                 struct ipv6_hdr *hdr2 = ip2;
1653                 struct ipv6_hdr *hdr3 = ip3;
1654
1655                 dscp0 = (rte_ntohl(hdr0->vtc_flow) & 0x0F600000) >> 18;
1656                 dscp1 = (rte_ntohl(hdr1->vtc_flow) & 0x0F600000) >> 18;
1657                 dscp2 = (rte_ntohl(hdr2->vtc_flow) & 0x0F600000) >> 18;
1658                 dscp3 = (rte_ntohl(hdr3->vtc_flow) & 0x0F600000) >> 18;
1659
1660                 total_length0 =
1661                         rte_ntohs(hdr0->payload_len) + sizeof(struct ipv6_hdr);
1662                 total_length1 =
1663                         rte_ntohs(hdr1->payload_len) + sizeof(struct ipv6_hdr);
1664                 total_length2 =
1665                         rte_ntohs(hdr2->payload_len) + sizeof(struct ipv6_hdr);
1666                 total_length3 =
1667                         rte_ntohs(hdr3->payload_len) + sizeof(struct ipv6_hdr);
1668         }
1669
1670         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
1671                 void *data0 =
1672                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_MTR);
1673                 void *data1 =
1674                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_MTR);
1675                 void *data2 =
1676                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_MTR);
1677                 void *data3 =
1678                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_MTR);
1679
1680                 drop_mask0 |= pkt_work_mtr(mbuf0,
1681                         data0,
1682                         &action->dscp_table,
1683                         action->mp,
1684                         time,
1685                         dscp0,
1686                         total_length0);
1687
1688                 drop_mask1 |= pkt_work_mtr(mbuf1,
1689                         data1,
1690                         &action->dscp_table,
1691                         action->mp,
1692                         time,
1693                         dscp1,
1694                         total_length1);
1695
1696                 drop_mask2 |= pkt_work_mtr(mbuf2,
1697                         data2,
1698                         &action->dscp_table,
1699                         action->mp,
1700                         time,
1701                         dscp2,
1702                         total_length2);
1703
1704                 drop_mask3 |= pkt_work_mtr(mbuf3,
1705                         data3,
1706                         &action->dscp_table,
1707                         action->mp,
1708                         time,
1709                         dscp3,
1710                         total_length3);
1711         }
1712
1713         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_TM)) {
1714                 void *data0 =
1715                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_TM);
1716                 void *data1 =
1717                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_TM);
1718                 void *data2 =
1719                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_TM);
1720                 void *data3 =
1721                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_TM);
1722
1723                 pkt_work_tm(mbuf0,
1724                         data0,
1725                         &action->dscp_table,
1726                         dscp0);
1727
1728                 pkt_work_tm(mbuf1,
1729                         data1,
1730                         &action->dscp_table,
1731                         dscp1);
1732
1733                 pkt_work_tm(mbuf2,
1734                         data2,
1735                         &action->dscp_table,
1736                         dscp2);
1737
1738                 pkt_work_tm(mbuf3,
1739                         data3,
1740                         &action->dscp_table,
1741                         dscp3);
1742         }
1743
1744         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
1745                 void *data0 =
1746                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_ENCAP);
1747                 void *data1 =
1748                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_ENCAP);
1749                 void *data2 =
1750                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_ENCAP);
1751                 void *data3 =
1752                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_ENCAP);
1753
1754                 pkt_work_encap(mbuf0,
1755                         data0,
1756                         &cfg->encap,
1757                         ip0,
1758                         total_length0,
1759                         ip_offset);
1760
1761                 pkt_work_encap(mbuf1,
1762                         data1,
1763                         &cfg->encap,
1764                         ip1,
1765                         total_length1,
1766                         ip_offset);
1767
1768                 pkt_work_encap(mbuf2,
1769                         data2,
1770                         &cfg->encap,
1771                         ip2,
1772                         total_length2,
1773                         ip_offset);
1774
1775                 pkt_work_encap(mbuf3,
1776                         data3,
1777                         &cfg->encap,
1778                         ip3,
1779                         total_length3,
1780                         ip_offset);
1781         }
1782
1783         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
1784                 void *data0 =
1785                         action_data_get(table_entry0, action, RTE_TABLE_ACTION_NAT);
1786                 void *data1 =
1787                         action_data_get(table_entry1, action, RTE_TABLE_ACTION_NAT);
1788                 void *data2 =
1789                         action_data_get(table_entry2, action, RTE_TABLE_ACTION_NAT);
1790                 void *data3 =
1791                         action_data_get(table_entry3, action, RTE_TABLE_ACTION_NAT);
1792
1793                 if (cfg->common.ip_version) {
1794                         pkt_ipv4_work_nat(ip0, data0, &cfg->nat);
1795                         pkt_ipv4_work_nat(ip1, data1, &cfg->nat);
1796                         pkt_ipv4_work_nat(ip2, data2, &cfg->nat);
1797                         pkt_ipv4_work_nat(ip3, data3, &cfg->nat);
1798                 } else {
1799                         pkt_ipv6_work_nat(ip0, data0, &cfg->nat);
1800                         pkt_ipv6_work_nat(ip1, data1, &cfg->nat);
1801                         pkt_ipv6_work_nat(ip2, data2, &cfg->nat);
1802                         pkt_ipv6_work_nat(ip3, data3, &cfg->nat);
1803                 }
1804         }
1805
1806         return drop_mask0 |
1807                 (drop_mask1 << 1) |
1808                 (drop_mask2 << 2) |
1809                 (drop_mask3 << 3);
1810 }
1811
1812 static __rte_always_inline int
1813 ah(struct rte_pipeline *p,
1814         struct rte_mbuf **pkts,
1815         uint64_t pkts_mask,
1816         struct rte_pipeline_table_entry **entries,
1817         struct rte_table_action *action,
1818         struct ap_config *cfg)
1819 {
1820         uint64_t pkts_drop_mask = 0;
1821         uint64_t time = 0;
1822
1823         if (cfg->action_mask & (1LLU << RTE_TABLE_ACTION_MTR))
1824                 time = rte_rdtsc();
1825
1826         if ((pkts_mask & (pkts_mask + 1)) == 0) {
1827                 uint64_t n_pkts = __builtin_popcountll(pkts_mask);
1828                 uint32_t i;
1829
1830                 for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4) {
1831                         uint64_t drop_mask;
1832
1833                         drop_mask = pkt4_work(&pkts[i],
1834                                 &entries[i],
1835                                 time,
1836                                 action,
1837                                 cfg);
1838
1839                         pkts_drop_mask |= drop_mask << i;
1840                 }
1841
1842                 for ( ; i < n_pkts; i++) {
1843                         uint64_t drop_mask;
1844
1845                         drop_mask = pkt_work(pkts[i],
1846                                 entries[i],
1847                                 time,
1848                                 action,
1849                                 cfg);
1850
1851                         pkts_drop_mask |= drop_mask << i;
1852                 }
1853         } else
1854                 for ( ; pkts_mask; ) {
1855                         uint32_t pos = __builtin_ctzll(pkts_mask);
1856                         uint64_t pkt_mask = 1LLU << pos;
1857                         uint64_t drop_mask;
1858
1859                         drop_mask = pkt_work(pkts[pos],
1860                                 entries[pos],
1861                                 time,
1862                                 action,
1863                                 cfg);
1864
1865                         pkts_mask &= ~pkt_mask;
1866                         pkts_drop_mask |= drop_mask << pos;
1867                 }
1868
1869         rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
1870
1871         return 0;
1872 }
1873
1874 static int
1875 ah_default(struct rte_pipeline *p,
1876         struct rte_mbuf **pkts,
1877         uint64_t pkts_mask,
1878         struct rte_pipeline_table_entry **entries,
1879         void *arg)
1880 {
1881         struct rte_table_action *action = arg;
1882
1883         return ah(p,
1884                 pkts,
1885                 pkts_mask,
1886                 entries,
1887                 action,
1888                 &action->cfg);
1889 }
1890
1891 static rte_pipeline_table_action_handler_hit
1892 ah_selector(struct rte_table_action *action)
1893 {
1894         if (action->cfg.action_mask == (1LLU << RTE_TABLE_ACTION_FWD))
1895                 return NULL;
1896
1897         return ah_default;
1898 }
1899
1900 int
1901 rte_table_action_table_params_get(struct rte_table_action *action,
1902         struct rte_pipeline_table_params *params)
1903 {
1904         rte_pipeline_table_action_handler_hit f_action_hit;
1905         uint32_t total_size;
1906
1907         /* Check input arguments */
1908         if ((action == NULL) ||
1909                 (params == NULL))
1910                 return -EINVAL;
1911
1912         f_action_hit = ah_selector(action);
1913         total_size = rte_align32pow2(action->data.total_size);
1914
1915         /* Fill in params */
1916         params->f_action_hit = f_action_hit;
1917         params->f_action_miss = NULL;
1918         params->arg_ah = (f_action_hit) ? action : NULL;
1919         params->action_data_size = total_size -
1920                 sizeof(struct rte_pipeline_table_entry);
1921
1922         return 0;
1923 }
1924
1925 int
1926 rte_table_action_free(struct rte_table_action *action)
1927 {
1928         if (action == NULL)
1929                 return 0;
1930
1931         rte_free(action);
1932
1933         return 0;
1934 }