examples/ip_pipeline/pipeline/pipeline_routing_be.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_malloc.h>
  42 #include <rte_ip.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_table_lpm.h>
  45 #include <rte_table_hash.h>
  46 #include <rte_pipeline.h>
  47
  48 #include "pipeline_routing_be.h"
  49 #include "pipeline_actions_common.h"
  50 #include "parser.h"
  51 #include "hash_func.h"
  52
  53 #define MPLS_LABEL(label, exp, s, ttl)                                  \
  54         (((((uint64_t) (label)) & 0xFFFFFLLU) << 12) |          \
  55         ((((uint64_t) (exp)) & 0x7LLU) << 9) |                          \
  56         ((((uint64_t) (s)) & 0x1LLU) << 8) |                            \
  57         (((uint64_t) (ttl)) & 0xFFLU))
  58
  59 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe,         \
  60         traffic_class, queue, color)                            \
  61         ((((uint64_t) (queue)) & 0x3) |                \
  62         ((((uint64_t) (traffic_class)) & 0x3) << 2) |  \
  63         ((((uint64_t) (color)) & 0x3) << 4) |          \
  64         ((((uint64_t) (subport)) & 0xFFFF) << 16) |    \
  65         ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
  66
  67
  68 /* Network Byte Order (NBO) */
  69 #define SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr, ethertype)       \
  70         (((uint64_t) macaddr) | (((uint64_t) rte_cpu_to_be_16(ethertype)) << 48))
  71
  72 #ifndef PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s
  73 #define PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s 256
  74 #endif
  75
  76 struct pipeline_routing {
  77         struct pipeline p;
  78         struct pipeline_routing_params params;
  79         pipeline_msg_req_handler custom_handlers[PIPELINE_ROUTING_MSG_REQS];
  80         uint64_t macaddr[PIPELINE_MAX_PORT_OUT];
  81 } __rte_cache_aligned;
  82
  83 /*
  84  * Message handlers
  85  */
  86 static void *
  87 pipeline_routing_msg_req_custom_handler(struct pipeline *p, void *msg);
  88
  89 static pipeline_msg_req_handler handlers[] = {
  90         [PIPELINE_MSG_REQ_PING] =
  91                 pipeline_msg_req_ping_handler,
  92         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  93                 pipeline_msg_req_stats_port_in_handler,
  94         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  95                 pipeline_msg_req_stats_port_out_handler,
  96         [PIPELINE_MSG_REQ_STATS_TABLE] =
  97                 pipeline_msg_req_stats_table_handler,
  98         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  99                 pipeline_msg_req_port_in_enable_handler,
 100         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
 101                 pipeline_msg_req_port_in_disable_handler,
 102         [PIPELINE_MSG_REQ_CUSTOM] =
 103                 pipeline_routing_msg_req_custom_handler,
 104 };
 105
 106 static void *
 107 pipeline_routing_msg_req_route_add_handler(struct pipeline *p,
 108         void *msg);
 109
 110 static void *
 111 pipeline_routing_msg_req_route_del_handler(struct pipeline *p,
 112         void *msg);
 113
 114 static void *
 115 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
 116         void *msg);
 117
 118 static void *
 119 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
 120         void *msg);
 121
 122 static void *
 123 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p,
 124         void *msg);
 125
 126 static void *
 127 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p,
 128         void *msg);
 129
 130 static void *
 131 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p,
 132         void *msg);
 133
 134 static void *
 135 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p,
 136         void *msg);
 137
 138 static void *
 139 pipeline_routing_msg_req_set_macaddr_handler(struct pipeline *p,
 140         void *msg);
 141
 142 static pipeline_msg_req_handler custom_handlers[] = {
 143         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD] =
 144                 pipeline_routing_msg_req_route_add_handler,
 145         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL] =
 146                 pipeline_routing_msg_req_route_del_handler,
 147         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD_DEFAULT] =
 148                 pipeline_routing_msg_req_route_add_default_handler,
 149         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL_DEFAULT] =
 150                 pipeline_routing_msg_req_route_del_default_handler,
 151         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD] =
 152                 pipeline_routing_msg_req_arp_add_handler,
 153         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL] =
 154                 pipeline_routing_msg_req_arp_del_handler,
 155         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD_DEFAULT] =
 156                 pipeline_routing_msg_req_arp_add_default_handler,
 157         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL_DEFAULT] =
 158                 pipeline_routing_msg_req_arp_del_default_handler,
 159         [PIPELINE_ROUTING_MSG_REQ_SET_MACADDR] =
 160                 pipeline_routing_msg_req_set_macaddr_handler,
 161 };
 162
 163 /*
 164  * Routing table
 165  */
 166 struct routing_table_entry {
 167         struct rte_pipeline_table_entry head;
 168         uint32_t flags;
 169         uint32_t port_id; /* Output port ID */
 170         uint32_t ip; /* Next hop IP address (only valid for remote routes) */
 171
 172         /* ether_l2 */
 173         uint16_t data_offset;
 174         uint16_t ether_l2_length;
 175         uint64_t slab[4];
 176         uint16_t slab_offset[4];
 177 };
 178
 179 struct layout {
 180         uint16_t a;
 181         uint32_t b;
 182         uint16_t c;
 183 } __attribute__((__packed__));
 184
 185 #define MACADDR_DST_WRITE(slab_ptr, slab)                       \
 186 {                                                               \
 187         struct layout *dst = (struct layout *) (slab_ptr);      \
 188         struct layout *src = (struct layout *) &(slab);         \
 189                                                                 \
 190         dst->b = src->b;                                        \
 191         dst->c = src->c;                                        \
 192 }
 193
 194 static __rte_always_inline void
 195 pkt_work_routing(
 196         struct rte_mbuf *pkt,
 197         struct rte_pipeline_table_entry *table_entry,
 198         void *arg,
 199         int arp,
 200         int qinq,
 201         int qinq_sched,
 202         int mpls,
 203         int mpls_color_mark)
 204 {
 205         struct pipeline_routing *p_rt = arg;
 206
 207         struct routing_table_entry *entry =
 208                 (struct routing_table_entry *) table_entry;
 209
 210         struct ipv4_hdr *ip = (struct ipv4_hdr *)
 211                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.ip_hdr_offset);
 212
 213         enum rte_meter_color pkt_color = (enum rte_meter_color)
 214                 RTE_MBUF_METADATA_UINT32(pkt, p_rt->params.color_offset);
 215
 216         struct pipeline_routing_arp_key_ipv4 *arp_key =
 217                 (struct pipeline_routing_arp_key_ipv4 *)
 218                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.arp_key_offset);
 219
 220         uint64_t *slab0_ptr, *slab1_ptr, *slab2_ptr, *slab3_ptr, sched;
 221         uint32_t ip_da, nh_ip, port_id;
 222         uint16_t total_length, data_offset, ether_l2_length;
 223
 224         /* Read */
 225         total_length = rte_bswap16(ip->total_length);
 226         ip_da = ip->dst_addr;
 227         data_offset = entry->data_offset;
 228         ether_l2_length = entry->ether_l2_length;
 229         slab0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[0]);
 230         slab1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[1]);
 231         slab2_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[2]);
 232         slab3_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[3]);
 233
 234         if (arp) {
 235                 port_id = entry->port_id;
 236                 nh_ip = entry->ip;
 237                 if (entry->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 238                         nh_ip = ip_da;
 239         }
 240
 241         /* Compute */
 242         total_length += ether_l2_length;
 243
 244         if (qinq && qinq_sched) {
 245                 uint32_t dscp = ip->type_of_service >> 2;
 246                 uint32_t svlan, cvlan, tc, tc_q;
 247
 248                 if (qinq_sched == 1) {
 249                         uint64_t slab_qinq = rte_bswap64(entry->slab[0]);
 250
 251                         svlan = (slab_qinq >> 48) & 0xFFF;
 252                         cvlan = (slab_qinq >> 16) & 0xFFF;
 253                         tc = (dscp >> 2) & 0x3;
 254                         tc_q = dscp & 0x3;
 255                 } else {
 256                         uint32_t ip_src = rte_bswap32(ip->src_addr);
 257
 258                         svlan = 0;
 259                         cvlan = (ip_src >> 16) & 0xFFF;
 260                         tc = (ip_src >> 2) & 0x3;
 261                         tc_q = ip_src & 0x3;
 262                 }
 263                 sched = RTE_SCHED_PORT_HIERARCHY(svlan,
 264                         cvlan,
 265                         tc,
 266                         tc_q,
 267                         e_RTE_METER_GREEN);
 268         }
 269
 270         /* Write */
 271         pkt->data_off = data_offset;
 272         pkt->data_len = total_length;
 273         pkt->pkt_len = total_length;
 274
 275         if ((qinq == 0) && (mpls == 0)) {
 276                 *slab0_ptr = entry->slab[0];
 277
 278                 if (arp == 0)
 279                         MACADDR_DST_WRITE(slab1_ptr, entry->slab[1]);
 280         }
 281
 282         if (qinq) {
 283                 *slab0_ptr = entry->slab[0];
 284                 *slab1_ptr = entry->slab[1];
 285
 286                 if (arp == 0)
 287                         MACADDR_DST_WRITE(slab2_ptr, entry->slab[2]);
 288
 289                 if (qinq_sched) {
 290                         pkt->hash.sched.lo = sched & 0xFFFFFFFF;
 291                         pkt->hash.sched.hi = sched >> 32;
 292                 }
 293         }
 294
 295         if (mpls) {
 296                 if (mpls_color_mark) {
 297                         uint64_t mpls_exp = rte_bswap64(
 298                                 (MPLS_LABEL(0, pkt_color, 0, 0) << 32) |
 299                                 MPLS_LABEL(0, pkt_color, 0, 0));
 300
 301                         *slab0_ptr = entry->slab[0] | mpls_exp;
 302                         *slab1_ptr = entry->slab[1] | mpls_exp;
 303                         *slab2_ptr = entry->slab[2];
 304                 } else {
 305                         *slab0_ptr = entry->slab[0];
 306                         *slab1_ptr = entry->slab[1];
 307                         *slab2_ptr = entry->slab[2];
 308                 }
 309
 310                 if (arp == 0)
 311                         MACADDR_DST_WRITE(slab3_ptr, entry->slab[3]);
 312         }
 313
 314         if (arp) {
 315                 arp_key->port_id = port_id;
 316                 arp_key->ip = nh_ip;
 317         }
 318 }
 319
 320 static __rte_always_inline void
 321 pkt4_work_routing(
 322         struct rte_mbuf **pkts,
 323         struct rte_pipeline_table_entry **table_entries,
 324         void *arg,
 325         int arp,
 326         int qinq,
 327         int qinq_sched,
 328         int mpls,
 329         int mpls_color_mark)
 330 {
 331         struct pipeline_routing *p_rt = arg;
 332
 333         struct routing_table_entry *entry0 =
 334                 (struct routing_table_entry *) table_entries[0];
 335         struct routing_table_entry *entry1 =
 336                 (struct routing_table_entry *) table_entries[1];
 337         struct routing_table_entry *entry2 =
 338                 (struct routing_table_entry *) table_entries[2];
 339         struct routing_table_entry *entry3 =
 340                 (struct routing_table_entry *) table_entries[3];
 341
 342         struct ipv4_hdr *ip0 = (struct ipv4_hdr *)
 343                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 344                         p_rt->params.ip_hdr_offset);
 345         struct ipv4_hdr *ip1 = (struct ipv4_hdr *)
 346                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 347                         p_rt->params.ip_hdr_offset);
 348         struct ipv4_hdr *ip2 = (struct ipv4_hdr *)
 349                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 350                         p_rt->params.ip_hdr_offset);
 351         struct ipv4_hdr *ip3 = (struct ipv4_hdr *)
 352                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 353                         p_rt->params.ip_hdr_offset);
 354
 355         enum rte_meter_color pkt0_color = (enum rte_meter_color)
 356                 RTE_MBUF_METADATA_UINT32(pkts[0], p_rt->params.color_offset);
 357         enum rte_meter_color pkt1_color = (enum rte_meter_color)
 358                 RTE_MBUF_METADATA_UINT32(pkts[1], p_rt->params.color_offset);
 359         enum rte_meter_color pkt2_color = (enum rte_meter_color)
 360                 RTE_MBUF_METADATA_UINT32(pkts[2], p_rt->params.color_offset);
 361         enum rte_meter_color pkt3_color = (enum rte_meter_color)
 362                 RTE_MBUF_METADATA_UINT32(pkts[3], p_rt->params.color_offset);
 363
 364         struct pipeline_routing_arp_key_ipv4 *arp_key0 =
 365                 (struct pipeline_routing_arp_key_ipv4 *)
 366                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 367                         p_rt->params.arp_key_offset);
 368         struct pipeline_routing_arp_key_ipv4 *arp_key1 =
 369                 (struct pipeline_routing_arp_key_ipv4 *)
 370                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 371                         p_rt->params.arp_key_offset);
 372         struct pipeline_routing_arp_key_ipv4 *arp_key2 =
 373                 (struct pipeline_routing_arp_key_ipv4 *)
 374                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 375                         p_rt->params.arp_key_offset);
 376         struct pipeline_routing_arp_key_ipv4 *arp_key3 =
 377                 (struct pipeline_routing_arp_key_ipv4 *)
 378                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 379                         p_rt->params.arp_key_offset);
 380
 381         uint64_t *slab0_ptr0, *slab1_ptr0, *slab2_ptr0, *slab3_ptr0;
 382         uint64_t *slab0_ptr1, *slab1_ptr1, *slab2_ptr1, *slab3_ptr1;
 383         uint64_t *slab0_ptr2, *slab1_ptr2, *slab2_ptr2, *slab3_ptr2;
 384         uint64_t *slab0_ptr3, *slab1_ptr3, *slab2_ptr3, *slab3_ptr3;
 385         uint64_t sched0, sched1, sched2, sched3;
 386
 387         uint32_t ip_da0, nh_ip0, port_id0;
 388         uint32_t ip_da1, nh_ip1, port_id1;
 389         uint32_t ip_da2, nh_ip2, port_id2;
 390         uint32_t ip_da3, nh_ip3, port_id3;
 391
 392         uint16_t total_length0, data_offset0, ether_l2_length0;
 393         uint16_t total_length1, data_offset1, ether_l2_length1;
 394         uint16_t total_length2, data_offset2, ether_l2_length2;
 395         uint16_t total_length3, data_offset3, ether_l2_length3;
 396
 397         /* Read */
 398         total_length0 = rte_bswap16(ip0->total_length);
 399         total_length1 = rte_bswap16(ip1->total_length);
 400         total_length2 = rte_bswap16(ip2->total_length);
 401         total_length3 = rte_bswap16(ip3->total_length);
 402
 403         ip_da0 = ip0->dst_addr;
 404         ip_da1 = ip1->dst_addr;
 405         ip_da2 = ip2->dst_addr;
 406         ip_da3 = ip3->dst_addr;
 407
 408         data_offset0 = entry0->data_offset;
 409         data_offset1 = entry1->data_offset;
 410         data_offset2 = entry2->data_offset;
 411         data_offset3 = entry3->data_offset;
 412
 413         ether_l2_length0 = entry0->ether_l2_length;
 414         ether_l2_length1 = entry1->ether_l2_length;
 415         ether_l2_length2 = entry2->ether_l2_length;
 416         ether_l2_length3 = entry3->ether_l2_length;
 417
 418         slab0_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 419                 entry0->slab_offset[0]);
 420         slab1_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 421                 entry0->slab_offset[1]);
 422         slab2_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 423                 entry0->slab_offset[2]);
 424         slab3_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 425                 entry0->slab_offset[3]);
 426
 427         slab0_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 428                 entry1->slab_offset[0]);
 429         slab1_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 430                 entry1->slab_offset[1]);
 431         slab2_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 432                 entry1->slab_offset[2]);
 433         slab3_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 434                 entry1->slab_offset[3]);
 435
 436         slab0_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 437                 entry2->slab_offset[0]);
 438         slab1_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 439                 entry2->slab_offset[1]);
 440         slab2_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 441                 entry2->slab_offset[2]);
 442         slab3_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 443                 entry2->slab_offset[3]);
 444
 445         slab0_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 446                 entry3->slab_offset[0]);
 447         slab1_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 448                 entry3->slab_offset[1]);
 449         slab2_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 450                 entry3->slab_offset[2]);
 451         slab3_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 452                 entry3->slab_offset[3]);
 453
 454         if (arp) {
 455                 port_id0 = entry0->port_id;
 456                 nh_ip0 = entry0->ip;
 457                 if (entry0->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 458                         nh_ip0 = ip_da0;
 459
 460                 port_id1 = entry1->port_id;
 461                 nh_ip1 = entry1->ip;
 462                 if (entry1->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 463                         nh_ip1 = ip_da1;
 464
 465                 port_id2 = entry2->port_id;
 466                 nh_ip2 = entry2->ip;
 467                 if (entry2->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 468                         nh_ip2 = ip_da2;
 469
 470                 port_id3 = entry3->port_id;
 471                 nh_ip3 = entry3->ip;
 472                 if (entry3->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 473                         nh_ip3 = ip_da3;
 474         }
 475
 476         /* Compute */
 477         total_length0 += ether_l2_length0;
 478         total_length1 += ether_l2_length1;
 479         total_length2 += ether_l2_length2;
 480         total_length3 += ether_l2_length3;
 481
 482         if (qinq && qinq_sched) {
 483                 uint32_t dscp0 = ip0->type_of_service >> 2;
 484                 uint32_t dscp1 = ip1->type_of_service >> 2;
 485                 uint32_t dscp2 = ip2->type_of_service >> 2;
 486                 uint32_t dscp3 = ip3->type_of_service >> 2;
 487                 uint32_t svlan0, cvlan0, tc0, tc_q0;
 488                 uint32_t svlan1, cvlan1, tc1, tc_q1;
 489                 uint32_t svlan2, cvlan2, tc2, tc_q2;
 490                 uint32_t svlan3, cvlan3, tc3, tc_q3;
 491
 492                 if (qinq_sched == 1) {
 493                         uint64_t slab_qinq0 = rte_bswap64(entry0->slab[0]);
 494                         uint64_t slab_qinq1 = rte_bswap64(entry1->slab[0]);
 495                         uint64_t slab_qinq2 = rte_bswap64(entry2->slab[0]);
 496                         uint64_t slab_qinq3 = rte_bswap64(entry3->slab[0]);
 497
 498                         svlan0 = (slab_qinq0 >> 48) & 0xFFF;
 499                         svlan1 = (slab_qinq1 >> 48) & 0xFFF;
 500                         svlan2 = (slab_qinq2 >> 48) & 0xFFF;
 501                         svlan3 = (slab_qinq3 >> 48) & 0xFFF;
 502
 503                         cvlan0 = (slab_qinq0 >> 16) & 0xFFF;
 504                         cvlan1 = (slab_qinq1 >> 16) & 0xFFF;
 505                         cvlan2 = (slab_qinq2 >> 16) & 0xFFF;
 506                         cvlan3 = (slab_qinq3 >> 16) & 0xFFF;
 507
 508                         tc0 = (dscp0 >> 2) & 0x3;
 509                         tc1 = (dscp1 >> 2) & 0x3;
 510                         tc2 = (dscp2 >> 2) & 0x3;
 511                         tc3 = (dscp3 >> 2) & 0x3;
 512
 513                         tc_q0 = dscp0 & 0x3;
 514                         tc_q1 = dscp1 & 0x3;
 515                         tc_q2 = dscp2 & 0x3;
 516                         tc_q3 = dscp3 & 0x3;
 517                 } else {
 518                         uint32_t ip_src0 = rte_bswap32(ip0->src_addr);
 519                         uint32_t ip_src1 = rte_bswap32(ip1->src_addr);
 520                         uint32_t ip_src2 = rte_bswap32(ip2->src_addr);
 521                         uint32_t ip_src3 = rte_bswap32(ip3->src_addr);
 522
 523                         svlan0 = 0;
 524                         svlan1 = 0;
 525                         svlan2 = 0;
 526                         svlan3 = 0;
 527
 528                         cvlan0 = (ip_src0 >> 16) & 0xFFF;
 529                         cvlan1 = (ip_src1 >> 16) & 0xFFF;
 530                         cvlan2 = (ip_src2 >> 16) & 0xFFF;
 531                         cvlan3 = (ip_src3 >> 16) & 0xFFF;
 532
 533                         tc0 = (ip_src0 >> 2) & 0x3;
 534                         tc1 = (ip_src1 >> 2) & 0x3;
 535                         tc2 = (ip_src2 >> 2) & 0x3;
 536                         tc3 = (ip_src3 >> 2) & 0x3;
 537
 538                         tc_q0 = ip_src0 & 0x3;
 539                         tc_q1 = ip_src1 & 0x3;
 540                         tc_q2 = ip_src2 & 0x3;
 541                         tc_q3 = ip_src3 & 0x3;
 542                 }
 543
 544                 sched0 = RTE_SCHED_PORT_HIERARCHY(svlan0,
 545                         cvlan0,
 546                         tc0,
 547                         tc_q0,
 548                         e_RTE_METER_GREEN);
 549                 sched1 = RTE_SCHED_PORT_HIERARCHY(svlan1,
 550                         cvlan1,
 551                         tc1,
 552                         tc_q1,
 553                         e_RTE_METER_GREEN);
 554                 sched2 = RTE_SCHED_PORT_HIERARCHY(svlan2,
 555                         cvlan2,
 556                         tc2,
 557                         tc_q2,
 558                         e_RTE_METER_GREEN);
 559                 sched3 = RTE_SCHED_PORT_HIERARCHY(svlan3,
 560                         cvlan3,
 561                         tc3,
 562                         tc_q3,
 563                         e_RTE_METER_GREEN);
 564
 565         }
 566
 567         /* Write */
 568         pkts[0]->data_off = data_offset0;
 569         pkts[1]->data_off = data_offset1;
 570         pkts[2]->data_off = data_offset2;
 571         pkts[3]->data_off = data_offset3;
 572
 573         pkts[0]->data_len = total_length0;
 574         pkts[1]->data_len = total_length1;
 575         pkts[2]->data_len = total_length2;
 576         pkts[3]->data_len = total_length3;
 577
 578         pkts[0]->pkt_len = total_length0;
 579         pkts[1]->pkt_len = total_length1;
 580         pkts[2]->pkt_len = total_length2;
 581         pkts[3]->pkt_len = total_length3;
 582
 583         if ((qinq == 0) && (mpls == 0)) {
 584                 *slab0_ptr0 = entry0->slab[0];
 585                 *slab0_ptr1 = entry1->slab[0];
 586                 *slab0_ptr2 = entry2->slab[0];
 587                 *slab0_ptr3 = entry3->slab[0];
 588
 589                 if (arp == 0) {
 590                         MACADDR_DST_WRITE(slab1_ptr0, entry0->slab[1]);
 591                         MACADDR_DST_WRITE(slab1_ptr1, entry1->slab[1]);
 592                         MACADDR_DST_WRITE(slab1_ptr2, entry2->slab[1]);
 593                         MACADDR_DST_WRITE(slab1_ptr3, entry3->slab[1]);
 594                 }
 595         }
 596
 597         if (qinq) {
 598                 *slab0_ptr0 = entry0->slab[0];
 599                 *slab0_ptr1 = entry1->slab[0];
 600                 *slab0_ptr2 = entry2->slab[0];
 601                 *slab0_ptr3 = entry3->slab[0];
 602
 603                 *slab1_ptr0 = entry0->slab[1];
 604                 *slab1_ptr1 = entry1->slab[1];
 605                 *slab1_ptr2 = entry2->slab[1];
 606                 *slab1_ptr3 = entry3->slab[1];
 607
 608                 if (arp == 0) {
 609                         MACADDR_DST_WRITE(slab2_ptr0, entry0->slab[2]);
 610                         MACADDR_DST_WRITE(slab2_ptr1, entry1->slab[2]);
 611                         MACADDR_DST_WRITE(slab2_ptr2, entry2->slab[2]);
 612                         MACADDR_DST_WRITE(slab2_ptr3, entry3->slab[2]);
 613                 }
 614
 615                 if (qinq_sched) {
 616                         pkts[0]->hash.sched.lo = sched0 & 0xFFFFFFFF;
 617                         pkts[0]->hash.sched.hi = sched0 >> 32;
 618                         pkts[1]->hash.sched.lo = sched1 & 0xFFFFFFFF;
 619                         pkts[1]->hash.sched.hi = sched1 >> 32;
 620                         pkts[2]->hash.sched.lo = sched2 & 0xFFFFFFFF;
 621                         pkts[2]->hash.sched.hi = sched2 >> 32;
 622                         pkts[3]->hash.sched.lo = sched3 & 0xFFFFFFFF;
 623                         pkts[3]->hash.sched.hi = sched3 >> 32;
 624                 }
 625         }
 626
 627         if (mpls) {
 628                 if (mpls_color_mark) {
 629                         uint64_t mpls_exp0 = rte_bswap64(
 630                                 (MPLS_LABEL(0, pkt0_color, 0, 0) << 32) |
 631                                 MPLS_LABEL(0, pkt0_color, 0, 0));
 632                         uint64_t mpls_exp1 = rte_bswap64(
 633                                 (MPLS_LABEL(0, pkt1_color, 0, 0) << 32) |
 634                                 MPLS_LABEL(0, pkt1_color, 0, 0));
 635                         uint64_t mpls_exp2 = rte_bswap64(
 636                                 (MPLS_LABEL(0, pkt2_color, 0, 0) << 32) |
 637                                 MPLS_LABEL(0, pkt2_color, 0, 0));
 638                         uint64_t mpls_exp3 = rte_bswap64(
 639                                 (MPLS_LABEL(0, pkt3_color, 0, 0) << 32) |
 640                                 MPLS_LABEL(0, pkt3_color, 0, 0));
 641
 642                         *slab0_ptr0 = entry0->slab[0] | mpls_exp0;
 643                         *slab0_ptr1 = entry1->slab[0] | mpls_exp1;
 644                         *slab0_ptr2 = entry2->slab[0] | mpls_exp2;
 645                         *slab0_ptr3 = entry3->slab[0] | mpls_exp3;
 646
 647                         *slab1_ptr0 = entry0->slab[1] | mpls_exp0;
 648                         *slab1_ptr1 = entry1->slab[1] | mpls_exp1;
 649                         *slab1_ptr2 = entry2->slab[1] | mpls_exp2;
 650                         *slab1_ptr3 = entry3->slab[1] | mpls_exp3;
 651
 652                         *slab2_ptr0 = entry0->slab[2];
 653                         *slab2_ptr1 = entry1->slab[2];
 654                         *slab2_ptr2 = entry2->slab[2];
 655                         *slab2_ptr3 = entry3->slab[2];
 656                 } else {
 657                         *slab0_ptr0 = entry0->slab[0];
 658                         *slab0_ptr1 = entry1->slab[0];
 659                         *slab0_ptr2 = entry2->slab[0];
 660                         *slab0_ptr3 = entry3->slab[0];
 661
 662                         *slab1_ptr0 = entry0->slab[1];
 663                         *slab1_ptr1 = entry1->slab[1];
 664                         *slab1_ptr2 = entry2->slab[1];
 665                         *slab1_ptr3 = entry3->slab[1];
 666
 667                         *slab2_ptr0 = entry0->slab[2];
 668                         *slab2_ptr1 = entry1->slab[2];
 669                         *slab2_ptr2 = entry2->slab[2];
 670                         *slab2_ptr3 = entry3->slab[2];
 671                 }
 672
 673                 if (arp == 0) {
 674                         MACADDR_DST_WRITE(slab3_ptr0, entry0->slab[3]);
 675                         MACADDR_DST_WRITE(slab3_ptr1, entry1->slab[3]);
 676                         MACADDR_DST_WRITE(slab3_ptr2, entry2->slab[3]);
 677                         MACADDR_DST_WRITE(slab3_ptr3, entry3->slab[3]);
 678                 }
 679         }
 680
 681         if (arp) {
 682                 arp_key0->port_id = port_id0;
 683                 arp_key1->port_id = port_id1;
 684                 arp_key2->port_id = port_id2;
 685                 arp_key3->port_id = port_id3;
 686
 687                 arp_key0->ip = nh_ip0;
 688                 arp_key1->ip = nh_ip1;
 689                 arp_key2->ip = nh_ip2;
 690                 arp_key3->ip = nh_ip3;
 691         }
 692 }
 693
 694 #define PKT_WORK_ROUTING_ETHERNET(arp)                          \
 695 static inline void                                              \
 696 pkt_work_routing_ether_arp##arp(                                \
 697         struct rte_mbuf *pkt,                                   \
 698         struct rte_pipeline_table_entry *table_entry,           \
 699         void *arg)                                              \
 700 {                                                               \
 701         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 0, 0);\
 702 }
 703
 704 #define PKT4_WORK_ROUTING_ETHERNET(arp)                         \
 705 static inline void                                              \
 706 pkt4_work_routing_ether_arp##arp(                               \
 707         struct rte_mbuf **pkts,                                 \
 708         struct rte_pipeline_table_entry **table_entries,        \
 709         void *arg)                                              \
 710 {                                                               \
 711         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 0, 0);\
 712 }
 713
 714 #define routing_table_ah_hit_ether(arp)                         \
 715 PKT_WORK_ROUTING_ETHERNET(arp)                                  \
 716 PKT4_WORK_ROUTING_ETHERNET(arp)                                 \
 717 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_arp##arp,      \
 718         pkt_work_routing_ether_arp##arp,                        \
 719         pkt4_work_routing_ether_arp##arp)
 720
 721 routing_table_ah_hit_ether(0)
 722 routing_table_ah_hit_ether(1)
 723
 724 #define PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)              \
 725 static inline void                                              \
 726 pkt_work_routing_ether_qinq_sched##sched##_arp##arp(            \
 727         struct rte_mbuf *pkt,                                   \
 728         struct rte_pipeline_table_entry *table_entry,           \
 729         void *arg)                                              \
 730 {                                                               \
 731         pkt_work_routing(pkt, table_entry, arg, arp, 1, sched, 0, 0);\
 732 }
 733
 734 #define PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)             \
 735 static inline void                                              \
 736 pkt4_work_routing_ether_qinq_sched##sched##_arp##arp(           \
 737         struct rte_mbuf **pkts,                                 \
 738         struct rte_pipeline_table_entry **table_entries,        \
 739         void *arg)                                              \
 740 {                                                               \
 741         pkt4_work_routing(pkts, table_entries, arg, arp, 1, sched, 0, 0);\
 742 }
 743
 744 #define routing_table_ah_hit_ether_qinq(sched, arp)             \
 745 PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                      \
 746 PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                     \
 747 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_qinq_sched##sched##_arp##arp,\
 748         pkt_work_routing_ether_qinq_sched##sched##_arp##arp,    \
 749         pkt4_work_routing_ether_qinq_sched##sched##_arp##arp)
 750
 751 routing_table_ah_hit_ether_qinq(0, 0)
 752 routing_table_ah_hit_ether_qinq(1, 0)
 753 routing_table_ah_hit_ether_qinq(2, 0)
 754 routing_table_ah_hit_ether_qinq(0, 1)
 755 routing_table_ah_hit_ether_qinq(1, 1)
 756 routing_table_ah_hit_ether_qinq(2, 1)
 757
 758 #define PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)              \
 759 static inline void                                              \
 760 pkt_work_routing_ether_mpls_color##color##_arp##arp(            \
 761         struct rte_mbuf *pkt,                                   \
 762         struct rte_pipeline_table_entry *table_entry,           \
 763         void *arg)                                              \
 764 {                                                               \
 765         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 1, color);\
 766 }
 767
 768 #define PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)             \
 769 static inline void                                              \
 770 pkt4_work_routing_ether_mpls_color##color##_arp##arp(           \
 771         struct rte_mbuf **pkts,                                 \
 772         struct rte_pipeline_table_entry **table_entries,        \
 773         void *arg)                                              \
 774 {                                                               \
 775         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 1, color);\
 776 }
 777
 778 #define routing_table_ah_hit_ether_mpls(color, arp)             \
 779 PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)                      \
 780 PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)                     \
 781 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_mpls_color##color##_arp##arp,\
 782         pkt_work_routing_ether_mpls_color##color##_arp##arp,    \
 783         pkt4_work_routing_ether_mpls_color##color##_arp##arp)
 784
 785 routing_table_ah_hit_ether_mpls(0, 0)
 786 routing_table_ah_hit_ether_mpls(1, 0)
 787 routing_table_ah_hit_ether_mpls(0, 1)
 788 routing_table_ah_hit_ether_mpls(1, 1)
 789
 790 static rte_pipeline_table_action_handler_hit
 791 get_routing_table_ah_hit(struct pipeline_routing *p)
 792 {
 793         if (p->params.dbg_ah_disable)
 794                 return NULL;
 795
 796         switch (p->params.encap) {
 797         case PIPELINE_ROUTING_ENCAP_ETHERNET:
 798                 return (p->params.n_arp_entries) ?
 799                         routing_table_ah_hit_ether_arp1 :
 800                         routing_table_ah_hit_ether_arp0;
 801
 802         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
 803                 if (p->params.n_arp_entries)
 804                         switch (p->params.qinq_sched) {
 805                         case 0:
 806                                 return routing_table_ah_hit_ether_qinq_sched0_arp1;
 807                         case 1:
 808                                 return routing_table_ah_hit_ether_qinq_sched1_arp1;
 809                         case 2:
 810                                 return routing_table_ah_hit_ether_qinq_sched2_arp1;
 811                         default:
 812                                 return NULL;
 813                         }
 814                  else
 815                         switch (p->params.qinq_sched) {
 816                         case 0:
 817                                 return routing_table_ah_hit_ether_qinq_sched0_arp0;
 818                         case 1:
 819                                 return routing_table_ah_hit_ether_qinq_sched1_arp0;
 820                         case 2:
 821                                 return routing_table_ah_hit_ether_qinq_sched2_arp0;
 822                         default:
 823                                 return NULL;
 824                         }
 825
 826         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
 827                 if (p->params.n_arp_entries)
 828                         if (p->params.mpls_color_mark)
 829                                 return routing_table_ah_hit_ether_mpls_color1_arp1;
 830                         else
 831                                 return routing_table_ah_hit_ether_mpls_color0_arp1;
 832                 else
 833                         if (p->params.mpls_color_mark)
 834                                 return routing_table_ah_hit_ether_mpls_color1_arp0;
 835                         else
 836                                 return routing_table_ah_hit_ether_mpls_color0_arp0;
 837
 838         default:
 839                 return NULL;
 840         }
 841 }
 842
 843 /*
 844  * ARP table
 845  */
 846 struct arp_table_entry {
 847         struct rte_pipeline_table_entry head;
 848         uint64_t macaddr;
 849 };
 850
 851 /**
 852  * ARP table AH
 853  */
 854 static inline void
 855 pkt_work_arp(
 856         struct rte_mbuf *pkt,
 857         struct rte_pipeline_table_entry *table_entry,
 858         __rte_unused void *arg)
 859 {
 860         struct arp_table_entry *entry = (struct arp_table_entry *) table_entry;
 861
 862         /* Read */
 863         uint64_t macaddr_dst = entry->macaddr;
 864         uint64_t *slab_ptr = (uint64_t *) ((char *) pkt->buf_addr +
 865                 (pkt->data_off - 2));
 866
 867         /* Compute */
 868
 869         /* Write */
 870         MACADDR_DST_WRITE(slab_ptr, macaddr_dst);
 871 }
 872
 873 static inline void
 874 pkt4_work_arp(
 875         struct rte_mbuf **pkts,
 876         struct rte_pipeline_table_entry **table_entries,
 877         __rte_unused void *arg)
 878 {
 879         struct arp_table_entry *entry0 =
 880                 (struct arp_table_entry *) table_entries[0];
 881         struct arp_table_entry *entry1 =
 882                 (struct arp_table_entry *) table_entries[1];
 883         struct arp_table_entry *entry2 =
 884                 (struct arp_table_entry *) table_entries[2];
 885         struct arp_table_entry *entry3 =
 886                 (struct arp_table_entry *) table_entries[3];
 887
 888         /* Read */
 889         uint64_t macaddr_dst0 = entry0->macaddr;
 890         uint64_t macaddr_dst1 = entry1->macaddr;
 891         uint64_t macaddr_dst2 = entry2->macaddr;
 892         uint64_t macaddr_dst3 = entry3->macaddr;
 893
 894         uint64_t *slab_ptr0 = (uint64_t *) ((char *) pkts[0]->buf_addr +
 895                 (pkts[0]->data_off - 2));
 896         uint64_t *slab_ptr1 = (uint64_t *) ((char *) pkts[1]->buf_addr +
 897                 (pkts[1]->data_off - 2));
 898         uint64_t *slab_ptr2 = (uint64_t *) ((char *) pkts[2]->buf_addr +
 899                 (pkts[2]->data_off - 2));
 900         uint64_t *slab_ptr3 = (uint64_t *) ((char *) pkts[3]->buf_addr +
 901                 (pkts[3]->data_off - 2));
 902
 903         /* Compute */
 904
 905         /* Write */
 906         MACADDR_DST_WRITE(slab_ptr0, macaddr_dst0);
 907         MACADDR_DST_WRITE(slab_ptr1, macaddr_dst1);
 908         MACADDR_DST_WRITE(slab_ptr2, macaddr_dst2);
 909         MACADDR_DST_WRITE(slab_ptr3, macaddr_dst3);
 910 }
 911
 912 PIPELINE_TABLE_AH_HIT(arp_table_ah_hit,
 913         pkt_work_arp,
 914         pkt4_work_arp);
 915
 916 static rte_pipeline_table_action_handler_hit
 917 get_arp_table_ah_hit(struct pipeline_routing *p)
 918 {
 919         if (p->params.dbg_ah_disable)
 920                 return NULL;
 921
 922         return arp_table_ah_hit;
 923 }
 924
 925 /*
 926  * Argument parsing
 927  */
 928 int
 929 pipeline_routing_parse_args(struct pipeline_routing_params *p,
 930         struct pipeline_params *params)
 931 {
 932         uint32_t n_routes_present = 0;
 933         uint32_t port_local_dest_present = 0;
 934         uint32_t encap_present = 0;
 935         uint32_t qinq_sched_present = 0;
 936         uint32_t mpls_color_mark_present = 0;
 937         uint32_t n_arp_entries_present = 0;
 938         uint32_t ip_hdr_offset_present = 0;
 939         uint32_t arp_key_offset_present = 0;
 940         uint32_t color_offset_present = 0;
 941         uint32_t dbg_ah_disable_present = 0;
 942         uint32_t i;
 943
 944         /* default values */
 945         p->n_routes = PIPELINE_ROUTING_N_ROUTES_DEFAULT;
 946         p->port_local_dest = params->n_ports_out - 1;
 947         p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 948         p->qinq_sched = 0;
 949         p->mpls_color_mark = 0;
 950         p->n_arp_entries = 0;
 951         p->dbg_ah_disable = 0;
 952
 953         for (i = 0; i < params->n_args; i++) {
 954                 char *arg_name = params->args_name[i];
 955                 char *arg_value = params->args_value[i];
 956
 957                 /* n_routes */
 958                 if (strcmp(arg_name, "n_routes") == 0) {
 959                         int status;
 960
 961                         PIPELINE_PARSE_ERR_DUPLICATE(
 962                                 n_routes_present == 0, params->name,
 963                                 arg_name);
 964                         n_routes_present = 1;
 965
 966                         status = parser_read_uint32(&p->n_routes,
 967                                 arg_value);
 968                         PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
 969                                 (p->n_routes != 0)), params->name,
 970                                 arg_name, arg_value);
 971                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 972                                 params->name, arg_name, arg_value);
 973
 974                         continue;
 975                 }
 976                 /* port_local_dest */
 977                 if (strcmp(arg_name, "port_local_dest") == 0) {
 978                         int status;
 979
 980                         PIPELINE_PARSE_ERR_DUPLICATE(
 981                                 port_local_dest_present == 0, params->name,
 982                                 arg_name);
 983                         port_local_dest_present = 1;
 984
 985                         status = parser_read_uint32(&p->port_local_dest,
 986                                 arg_value);
 987                         PIPELINE_PARSE_ERR_INV_VAL(((status == 0) &&
 988                                 (p->port_local_dest < params->n_ports_out)),
 989                                 params->name, arg_name, arg_value);
 990
 991                         continue;
 992                 }
 993
 994                 /* encap */
 995                 if (strcmp(arg_name, "encap") == 0) {
 996                         PIPELINE_PARSE_ERR_DUPLICATE(encap_present == 0,
 997                                 params->name, arg_name);
 998                         encap_present = 1;
 999
1000                         /* ethernet */
1001                         if (strcmp(arg_value, "ethernet") == 0) {
1002                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
1003                                 continue;
1004                         }
1005
1006                         /* ethernet_qinq */
1007                         if (strcmp(arg_value, "ethernet_qinq") == 0) {
1008                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ;
1009                                 continue;
1010                         }
1011
1012                         /* ethernet_mpls */
1013                         if (strcmp(arg_value, "ethernet_mpls") == 0) {
1014                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS;
1015                                 continue;
1016                         }
1017
1018                         /* any other */
1019                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1020                                 arg_name, arg_value);
1021                 }
1022
1023                 /* qinq_sched */
1024                 if (strcmp(arg_name, "qinq_sched") == 0) {
1025                         int status;
1026
1027                         PIPELINE_PARSE_ERR_DUPLICATE(
1028                                 qinq_sched_present == 0, params->name,
1029                                 arg_name);
1030                         qinq_sched_present = 1;
1031
1032                         status = parser_read_arg_bool(arg_value);
1033                         if (status == -EINVAL) {
1034                                 if (strcmp(arg_value, "test") == 0) {
1035                                         p->qinq_sched = 2;
1036                                         continue;
1037                                 }
1038                         } else {
1039                                 p->qinq_sched = status;
1040                                 continue;
1041                         }
1042
1043                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1044                                 arg_name, arg_value);
1045                 }
1046
1047                 /* mpls_color_mark */
1048                 if (strcmp(arg_name, "mpls_color_mark") == 0) {
1049                         int status;
1050
1051                         PIPELINE_PARSE_ERR_DUPLICATE(
1052                                 mpls_color_mark_present == 0,
1053                                 params->name, arg_name);
1054                         mpls_color_mark_present = 1;
1055
1056
1057                         status = parser_read_arg_bool(arg_value);
1058                         if (status >= 0) {
1059                                 p->mpls_color_mark = status;
1060                                 continue;
1061                         }
1062
1063                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1064                                 arg_name, arg_value);
1065                 }
1066
1067                 /* n_arp_entries */
1068                 if (strcmp(arg_name, "n_arp_entries") == 0) {
1069                         int status;
1070
1071                         PIPELINE_PARSE_ERR_DUPLICATE(
1072                                 n_arp_entries_present == 0, params->name,
1073                                 arg_name);
1074                         n_arp_entries_present = 1;
1075
1076                         status = parser_read_uint32(&p->n_arp_entries,
1077                                 arg_value);
1078                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1079                                 params->name, arg_name, arg_value);
1080                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1081                                 params->name, arg_name, arg_value);
1082
1083                         continue;
1084                 }
1085
1086                 /* ip_hdr_offset */
1087                 if (strcmp(arg_name, "ip_hdr_offset") == 0) {
1088                         int status;
1089
1090                         PIPELINE_PARSE_ERR_DUPLICATE(
1091                                 ip_hdr_offset_present == 0, params->name,
1092                                 arg_name);
1093                         ip_hdr_offset_present = 1;
1094
1095                         status = parser_read_uint32(&p->ip_hdr_offset,
1096                                 arg_value);
1097                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1098                                 params->name, arg_name, arg_value);
1099                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1100                                 params->name, arg_name, arg_value);
1101
1102                         continue;
1103                 }
1104
1105                 /* arp_key_offset */
1106                 if (strcmp(arg_name, "arp_key_offset") == 0) {
1107                         int status;
1108
1109                         PIPELINE_PARSE_ERR_DUPLICATE(
1110                                 arp_key_offset_present == 0, params->name,
1111                                 arg_name);
1112                         arp_key_offset_present = 1;
1113
1114                         status = parser_read_uint32(&p->arp_key_offset,
1115                                 arg_value);
1116                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1117                                 params->name, arg_name, arg_value);
1118                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1119                                 params->name, arg_name, arg_value);
1120
1121                         continue;
1122                 }
1123
1124                 /* color_offset */
1125                 if (strcmp(arg_name, "color_offset") == 0) {
1126                         int status;
1127
1128                         PIPELINE_PARSE_ERR_DUPLICATE(
1129                                 color_offset_present == 0, params->name,
1130                                 arg_name);
1131                         color_offset_present = 1;
1132
1133                         status = parser_read_uint32(&p->color_offset,
1134                                 arg_value);
1135                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1136                                 params->name, arg_name, arg_value);
1137                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1138                                 params->name, arg_name, arg_value);
1139
1140                         continue;
1141                 }
1142
1143                 /* debug */
1144                 if (strcmp(arg_name, "dbg_ah_disable") == 0) {
1145                         int status;
1146
1147                         PIPELINE_PARSE_ERR_DUPLICATE(
1148                                 dbg_ah_disable_present == 0, params->name,
1149                                 arg_name);
1150                         dbg_ah_disable_present = 1;
1151
1152                         status = parser_read_arg_bool(arg_value);
1153                         if (status >= 0) {
1154                                 p->dbg_ah_disable = status;
1155                                 continue;
1156                         }
1157
1158                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1159                                 arg_name, arg_value);
1160
1161                         continue;
1162                 }
1163
1164                 /* any other */
1165                 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
1166         }
1167
1168         /* Check that mandatory arguments are present */
1169         PIPELINE_PARSE_ERR_MANDATORY(ip_hdr_offset_present, params->name,
1170                 "ip_hdr_offset");
1171
1172         /* Check relations between arguments */
1173         switch (p->encap) {
1174         case PIPELINE_ROUTING_ENCAP_ETHERNET:
1175                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1176                         "section \"%s\": encap = ethernet, therefore "
1177                         "qinq_sched = yes/test is not allowed",
1178                         params->name);
1179                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1180                         "in section \"%s\": encap = ethernet, therefore "
1181                         "mpls_color_mark = yes is not allowed",
1182                         params->name);
1183                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1184                         "in section \"%s\": encap = ethernet, therefore "
1185                         "color_offset is not allowed",
1186                         params->name);
1187                 break;
1188
1189         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
1190                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1191                         "in section \"%s\": encap = ethernet_qinq, "
1192                         "therefore mpls_color_mark = yes is not allowed",
1193                         params->name);
1194                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1195                         "in section \"%s\": encap = ethernet_qinq, "
1196                         "therefore color_offset is not allowed",
1197                         params->name);
1198                 break;
1199
1200         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
1201                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1202                         "section \"%s\": encap = ethernet_mpls, therefore "
1203                         "qinq_sched  = yes/test is not allowed",
1204                         params->name);
1205                 break;
1206         }
1207
1208         PIPELINE_ARG_CHECK((!(p->n_arp_entries &&
1209                 (!arp_key_offset_present))), "Parse error in section "
1210                         "\"%s\": n_arp_entries is set while "
1211                         "arp_key_offset is not set", params->name);
1212
1213         PIPELINE_ARG_CHECK((!((p->n_arp_entries == 0) &&
1214                 arp_key_offset_present)), "Parse error in section "
1215                         "\"%s\": arp_key_offset present while "
1216                         "n_arp_entries is not set", params->name);
1217
1218         return 0;
1219 }
1220
1221 static void *
1222 pipeline_routing_init(struct pipeline_params *params,
1223         __rte_unused void *arg)
1224 {
1225         struct pipeline *p;
1226         struct pipeline_routing *p_rt;
1227         uint32_t size, i;
1228
1229         /* Check input arguments */
1230         if ((params == NULL) ||
1231                 (params->n_ports_in == 0) ||
1232                 (params->n_ports_out == 0))
1233                 return NULL;
1234
1235         /* Memory allocation */
1236         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_routing));
1237         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
1238         p_rt = (struct pipeline_routing *) p;
1239         if (p == NULL)
1240                 return NULL;
1241
1242         strcpy(p->name, params->name);
1243         p->log_level = params->log_level;
1244
1245         PLOG(p, HIGH, "Routing");
1246
1247         /* Parse arguments */
1248         if (pipeline_routing_parse_args(&p_rt->params, params))
1249                 return NULL;
1250
1251         /* Pipeline */
1252         {
1253                 struct rte_pipeline_params pipeline_params = {
1254                         .name = params->name,
1255                         .socket_id = params->socket_id,
1256                         .offset_port_id = 0,
1257                 };
1258
1259                 p->p = rte_pipeline_create(&pipeline_params);
1260                 if (p->p == NULL) {
1261                         rte_free(p);
1262                         return NULL;
1263                 }
1264         }
1265
1266         /* Input ports */
1267         p->n_ports_in = params->n_ports_in;
1268         for (i = 0; i < p->n_ports_in; i++) {
1269                 struct rte_pipeline_port_in_params port_params = {
1270                         .ops = pipeline_port_in_params_get_ops(
1271                                 &params->port_in[i]),
1272                         .arg_create = pipeline_port_in_params_convert(
1273                                 &params->port_in[i]),
1274                         .f_action = NULL,
1275                         .arg_ah = NULL,
1276                         .burst_size = params->port_in[i].burst_size,
1277                 };
1278
1279                 int status = rte_pipeline_port_in_create(p->p,
1280                         &port_params,
1281                         &p->port_in_id[i]);
1282
1283                 if (status) {
1284                         rte_pipeline_free(p->p);
1285                         rte_free(p);
1286                         return NULL;
1287                 }
1288         }
1289
1290         /* Output ports */
1291         p->n_ports_out = params->n_ports_out;
1292         for (i = 0; i < p->n_ports_out; i++) {
1293                 struct rte_pipeline_port_out_params port_params = {
1294                         .ops = pipeline_port_out_params_get_ops(
1295                                 &params->port_out[i]),
1296                         .arg_create = pipeline_port_out_params_convert(
1297                                 &params->port_out[i]),
1298                         .f_action = NULL,
1299                         .arg_ah = NULL,
1300                 };
1301
1302                 int status = rte_pipeline_port_out_create(p->p,
1303                         &port_params,
1304                         &p->port_out_id[i]);
1305
1306                 if (status) {
1307                         rte_pipeline_free(p->p);
1308                         rte_free(p);
1309                         return NULL;
1310                 }
1311         }
1312
1313         /* Routing table */
1314         p->n_tables = 1;
1315         {
1316                 struct rte_table_lpm_params table_lpm_params = {
1317                         .name = p->name,
1318                         .n_rules = p_rt->params.n_routes,
1319                         .number_tbl8s = PIPELINE_ROUTING_LPM_TABLE_NUMBER_TABLE8s,
1320                         .flags = 0,
1321                         .entry_unique_size = sizeof(struct routing_table_entry),
1322                         .offset = p_rt->params.ip_hdr_offset +
1323                                 __builtin_offsetof(struct ipv4_hdr, dst_addr),
1324                 };
1325
1326                 struct rte_pipeline_table_params table_params = {
1327                                 .ops = &rte_table_lpm_ops,
1328                                 .arg_create = &table_lpm_params,
1329                                 .f_action_hit = get_routing_table_ah_hit(p_rt),
1330                                 .f_action_miss = NULL,
1331                                 .arg_ah = p_rt,
1332                                 .action_data_size =
1333                                         sizeof(struct routing_table_entry) -
1334                                         sizeof(struct rte_pipeline_table_entry),
1335                         };
1336
1337                 int status;
1338
1339                 status = rte_pipeline_table_create(p->p,
1340                         &table_params,
1341                         &p->table_id[0]);
1342
1343                 if (status) {
1344                         rte_pipeline_free(p->p);
1345                         rte_free(p);
1346                         return NULL;
1347                 }
1348         }
1349
1350         /* ARP table configuration */
1351         if (p_rt->params.n_arp_entries) {
1352                 struct rte_table_hash_params table_arp_params = {
1353                         .name = p->name,
1354                         .key_size = 8,
1355                         .key_offset = p_rt->params.arp_key_offset,
1356                         .key_mask = NULL,
1357                         .n_keys = p_rt->params.n_arp_entries,
1358                         .n_buckets = p_rt->params.n_arp_entries / 4,
1359                         .f_hash = hash_default_key8,
1360                         .seed = 0,
1361                 };
1362
1363                 struct rte_pipeline_table_params table_params = {
1364                         .ops = &rte_table_hash_key8_ext_ops,
1365                         .arg_create = &table_arp_params,
1366                         .f_action_hit = get_arp_table_ah_hit(p_rt),
1367                         .f_action_miss = NULL,
1368                         .arg_ah = p_rt,
1369                         .action_data_size = sizeof(struct arp_table_entry) -
1370                                 sizeof(struct rte_pipeline_table_entry),
1371                 };
1372
1373                 int status;
1374
1375                 status = rte_pipeline_table_create(p->p,
1376                         &table_params,
1377                         &p->table_id[1]);
1378
1379                 if (status) {
1380                         rte_pipeline_free(p->p);
1381                         rte_free(p);
1382                         return NULL;
1383                 }
1384
1385                 p->n_tables++;
1386         }
1387
1388         /* Connecting input ports to tables */
1389         for (i = 0; i < p->n_ports_in; i++) {
1390                 int status = rte_pipeline_port_in_connect_to_table(p->p,
1391                         p->port_in_id[i],
1392                         p->table_id[0]);
1393
1394                 if (status) {
1395                         rte_pipeline_free(p->p);
1396                         rte_free(p);
1397                         return NULL;
1398                 }
1399         }
1400
1401         /* Enable input ports */
1402         for (i = 0; i < p->n_ports_in; i++) {
1403                 int status = rte_pipeline_port_in_enable(p->p,
1404                         p->port_in_id[i]);
1405
1406                 if (status) {
1407                         rte_pipeline_free(p->p);
1408                         rte_free(p);
1409                         return NULL;
1410                 }
1411         }
1412
1413         /* Check pipeline consistency */
1414         if (rte_pipeline_check(p->p) < 0) {
1415                 rte_pipeline_free(p->p);
1416                 rte_free(p);
1417                 return NULL;
1418         }
1419
1420         /* Message queues */
1421         p->n_msgq = params->n_msgq;
1422         for (i = 0; i < p->n_msgq; i++)
1423                 p->msgq_in[i] = params->msgq_in[i];
1424         for (i = 0; i < p->n_msgq; i++)
1425                 p->msgq_out[i] = params->msgq_out[i];
1426
1427         /* Message handlers */
1428         memcpy(p->handlers, handlers, sizeof(p->handlers));
1429         memcpy(p_rt->custom_handlers,
1430                 custom_handlers,
1431                 sizeof(p_rt->custom_handlers));
1432
1433         return p;
1434 }
1435
1436 static int
1437 pipeline_routing_free(void *pipeline)
1438 {
1439         struct pipeline *p = (struct pipeline *) pipeline;
1440
1441         /* Check input arguments */
1442         if (p == NULL)
1443                 return -1;
1444
1445         /* Free resources */
1446         rte_pipeline_free(p->p);
1447         rte_free(p);
1448         return 0;
1449 }
1450
1451 static int
1452 pipeline_routing_timer(void *pipeline)
1453 {
1454         struct pipeline *p = (struct pipeline *) pipeline;
1455
1456         pipeline_msg_req_handle(p);
1457         rte_pipeline_flush(p->p);
1458
1459         return 0;
1460 }
1461
1462 void *
1463 pipeline_routing_msg_req_custom_handler(struct pipeline *p,
1464         void *msg)
1465 {
1466         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1467         struct pipeline_custom_msg_req *req = msg;
1468         pipeline_msg_req_handler f_handle;
1469
1470         f_handle = (req->subtype < PIPELINE_ROUTING_MSG_REQS) ?
1471                 p_rt->custom_handlers[req->subtype] :
1472                 pipeline_msg_req_invalid_handler;
1473
1474         if (f_handle == NULL)
1475                 f_handle = pipeline_msg_req_invalid_handler;
1476
1477         return f_handle(p, req);
1478 }
1479
1480 void *
1481 pipeline_routing_msg_req_route_add_handler(struct pipeline *p, void *msg)
1482 {
1483         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1484         struct pipeline_routing_route_add_msg_req *req = msg;
1485         struct pipeline_routing_route_add_msg_rsp *rsp = msg;
1486
1487         struct rte_table_lpm_key key = {
1488                 .ip = req->key.key.ipv4.ip,
1489                 .depth = req->key.key.ipv4.depth,
1490         };
1491
1492         struct routing_table_entry entry_arp0 = {
1493                 .head = {
1494                         .action = RTE_PIPELINE_ACTION_PORT,
1495                         {.port_id = p->port_out_id[req->data.port_id]},
1496                 },
1497
1498                 .flags = req->data.flags,
1499                 .port_id = req->data.port_id,
1500                 .ip = 0,
1501                 .data_offset = 0,
1502                 .ether_l2_length = 0,
1503                 .slab = {0},
1504                 .slab_offset = {0},
1505         };
1506
1507         struct routing_table_entry entry_arp1 = {
1508                 .head = {
1509                         .action = RTE_PIPELINE_ACTION_TABLE,
1510                         {.table_id = p->table_id[1]},
1511                 },
1512
1513                 .flags = req->data.flags,
1514                 .port_id = req->data.port_id,
1515                 .ip = rte_bswap32(req->data.ethernet.ip),
1516                 .data_offset = 0,
1517                 .ether_l2_length = 0,
1518                 .slab = {0},
1519                 .slab_offset = {0},
1520         };
1521
1522         struct rte_pipeline_table_entry *entry = (p_rt->params.n_arp_entries) ?
1523                 (struct rte_pipeline_table_entry *) &entry_arp1 :
1524                 (struct rte_pipeline_table_entry *) &entry_arp0;
1525
1526         if ((req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) ||
1527                 ((p_rt->params.n_arp_entries == 0) &&
1528                         (req->data.flags & PIPELINE_ROUTING_ROUTE_ARP)) ||
1529                 (p_rt->params.n_arp_entries &&
1530                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_ARP) == 0)) ||
1531                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1532                         (req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ)) ||
1533                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1534                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ) == 0)) ||
1535                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1536                         (req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS)) ||
1537                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1538                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS) == 0))) {
1539                 rsp->status = -1;
1540                 return rsp;
1541         }
1542
1543         /* Ether - ARP off */
1544         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1545                 (p_rt->params.n_arp_entries == 0)) {
1546                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1547                 uint64_t macaddr_dst;
1548                 uint64_t ethertype = ETHER_TYPE_IPv4;
1549
1550                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1551                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1552
1553                 entry_arp0.slab[0] =
1554                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype);
1555                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1556
1557                 entry_arp0.slab[1] = rte_bswap64(macaddr_dst);
1558                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1559
1560                 entry_arp0.data_offset = entry_arp0.slab_offset[1] + 2
1561                         - sizeof(struct rte_mbuf);
1562                 entry_arp0.ether_l2_length = 14;
1563         }
1564
1565         /* Ether - ARP on */
1566         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1567                 p_rt->params.n_arp_entries) {
1568                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1569                 uint64_t ethertype = ETHER_TYPE_IPv4;
1570
1571                 entry_arp1.slab[0] =
1572                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype);
1573                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1574
1575                 entry_arp1.data_offset = entry_arp1.slab_offset[0] - 6
1576                         - sizeof(struct rte_mbuf);
1577                 entry_arp1.ether_l2_length = 14;
1578         }
1579
1580         /* Ether QinQ - ARP off */
1581         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1582                 (p_rt->params.n_arp_entries == 0)) {
1583                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1584                 uint64_t macaddr_dst;
1585                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1586                 uint64_t ethertype_vlan = 0x8100;
1587                 uint64_t ethertype_qinq = 0x9100;
1588                 uint64_t svlan = req->data.l2.qinq.svlan;
1589                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1590
1591                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1592                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1593
1594                 entry_arp0.slab[0] = rte_bswap64((svlan << 48) |
1595                         (ethertype_vlan << 32) |
1596                         (cvlan << 16) |
1597                         ethertype_ipv4);
1598                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1599
1600                 entry_arp0.slab[1] =
1601                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_qinq);
1602                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1603
1604                 entry_arp0.slab[2] = rte_bswap64(macaddr_dst);
1605                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset - 3 * 8;
1606
1607                 entry_arp0.data_offset = entry_arp0.slab_offset[2] + 2
1608                         - sizeof(struct rte_mbuf);
1609                 entry_arp0.ether_l2_length = 22;
1610         }
1611
1612         /* Ether QinQ - ARP on */
1613         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1614                 p_rt->params.n_arp_entries) {
1615                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1616                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1617                 uint64_t ethertype_vlan = 0x8100;
1618                 uint64_t ethertype_qinq = 0x9100;
1619                 uint64_t svlan = req->data.l2.qinq.svlan;
1620                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1621
1622                 entry_arp1.slab[0] = rte_bswap64((svlan << 48) |
1623                         (ethertype_vlan << 32) |
1624                         (cvlan << 16) |
1625                         ethertype_ipv4);
1626                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1627
1628                 entry_arp1.slab[1] =
1629                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_qinq);
1630                 entry_arp1.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1631
1632                 entry_arp1.data_offset = entry_arp1.slab_offset[1] - 6
1633                         - sizeof(struct rte_mbuf);
1634                 entry_arp1.ether_l2_length = 22;
1635         }
1636
1637         /* Ether MPLS - ARP off */
1638         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1639                 (p_rt->params.n_arp_entries == 0)) {
1640                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1641                 uint64_t macaddr_dst;
1642                 uint64_t ethertype_mpls = 0x8847;
1643
1644                 uint64_t label0 = req->data.l2.mpls.labels[0];
1645                 uint64_t label1 = req->data.l2.mpls.labels[1];
1646                 uint64_t label2 = req->data.l2.mpls.labels[2];
1647                 uint64_t label3 = req->data.l2.mpls.labels[3];
1648                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1649
1650                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1651                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1652
1653                 switch (n_labels) {
1654                 case 1:
1655                         entry_arp0.slab[0] = 0;
1656                         entry_arp0.slab_offset[0] =
1657                                 p_rt->params.ip_hdr_offset - 8;
1658
1659                         entry_arp0.slab[1] = rte_bswap64(
1660                                 MPLS_LABEL(label0, 0, 1, 0));
1661                         entry_arp0.slab_offset[1] =
1662                                 p_rt->params.ip_hdr_offset - 8;
1663                         break;
1664
1665                 case 2:
1666                         entry_arp0.slab[0] = 0;
1667                         entry_arp0.slab_offset[0] =
1668                                 p_rt->params.ip_hdr_offset - 8;
1669
1670                         entry_arp0.slab[1] = rte_bswap64(
1671                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1672                                 MPLS_LABEL(label1, 0, 1, 0));
1673                         entry_arp0.slab_offset[1] =
1674                                 p_rt->params.ip_hdr_offset - 8;
1675                         break;
1676
1677                 case 3:
1678                         entry_arp0.slab[0] = rte_bswap64(
1679                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1680                                 MPLS_LABEL(label2, 0, 1, 0));
1681                         entry_arp0.slab_offset[0] =
1682                                 p_rt->params.ip_hdr_offset - 8;
1683
1684                         entry_arp0.slab[1] = rte_bswap64(
1685                                 MPLS_LABEL(label0, 0, 0, 0));
1686                         entry_arp0.slab_offset[1] =
1687                                 p_rt->params.ip_hdr_offset - 2 * 8;
1688                         break;
1689
1690                 case 4:
1691                         entry_arp0.slab[0] = rte_bswap64(
1692                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1693                                 MPLS_LABEL(label3, 0, 1, 0));
1694                         entry_arp0.slab_offset[0] =
1695                                 p_rt->params.ip_hdr_offset - 8;
1696
1697                         entry_arp0.slab[1] = rte_bswap64(
1698                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1699                                 MPLS_LABEL(label1, 0, 0, 0));
1700                         entry_arp0.slab_offset[1] =
1701                                 p_rt->params.ip_hdr_offset - 2 * 8;
1702                         break;
1703
1704                 default:
1705                         rsp->status = -1;
1706                         return rsp;
1707                 }
1708
1709                 entry_arp0.slab[2] =
1710                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_mpls);
1711                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset -
1712                         (n_labels * 4 + 8);
1713
1714                 entry_arp0.slab[3] = rte_bswap64(macaddr_dst);
1715                 entry_arp0.slab_offset[3] = p_rt->params.ip_hdr_offset -
1716                         (n_labels * 4 + 2 * 8);
1717
1718                 entry_arp0.data_offset = entry_arp0.slab_offset[3] + 2
1719                         - sizeof(struct rte_mbuf);
1720                 entry_arp0.ether_l2_length = n_labels * 4 + 14;
1721         }
1722
1723         /* Ether MPLS - ARP on */
1724         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1725                 p_rt->params.n_arp_entries) {
1726                 uint64_t macaddr_src = p_rt->macaddr[req->data.port_id];
1727                 uint64_t ethertype_mpls = 0x8847;
1728
1729                 uint64_t label0 = req->data.l2.mpls.labels[0];
1730                 uint64_t label1 = req->data.l2.mpls.labels[1];
1731                 uint64_t label2 = req->data.l2.mpls.labels[2];
1732                 uint64_t label3 = req->data.l2.mpls.labels[3];
1733                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1734
1735                 switch (n_labels) {
1736                 case 1:
1737                         entry_arp1.slab[0] = 0;
1738                         entry_arp1.slab_offset[0] =
1739                                 p_rt->params.ip_hdr_offset - 8;
1740
1741                         entry_arp1.slab[1] = rte_bswap64(
1742                                 MPLS_LABEL(label0, 0, 1, 0));
1743                         entry_arp1.slab_offset[1] =
1744                                 p_rt->params.ip_hdr_offset - 8;
1745                         break;
1746
1747                 case 2:
1748                         entry_arp1.slab[0] = 0;
1749                         entry_arp1.slab_offset[0] =
1750                                 p_rt->params.ip_hdr_offset - 8;
1751
1752                         entry_arp1.slab[1] = rte_bswap64(
1753                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1754                                 MPLS_LABEL(label1, 0, 1, 0));
1755                         entry_arp1.slab_offset[1] =
1756                                 p_rt->params.ip_hdr_offset - 8;
1757                         break;
1758
1759                 case 3:
1760                         entry_arp1.slab[0] = rte_bswap64(
1761                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1762                                 MPLS_LABEL(label2, 0, 1, 0));
1763                         entry_arp1.slab_offset[0] =
1764                                 p_rt->params.ip_hdr_offset - 8;
1765
1766                         entry_arp1.slab[1] = rte_bswap64(
1767                                 MPLS_LABEL(label0, 0, 0, 0));
1768                         entry_arp1.slab_offset[1] =
1769                                 p_rt->params.ip_hdr_offset - 2 * 8;
1770                         break;
1771
1772                 case 4:
1773                         entry_arp1.slab[0] = rte_bswap64(
1774                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1775                                 MPLS_LABEL(label3, 0, 1, 0));
1776                         entry_arp1.slab_offset[0] =
1777                                 p_rt->params.ip_hdr_offset - 8;
1778
1779                         entry_arp1.slab[1] = rte_bswap64(
1780                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1781                                 MPLS_LABEL(label1, 0, 0, 0));
1782                         entry_arp1.slab_offset[1] =
1783                                 p_rt->params.ip_hdr_offset - 2 * 8;
1784                         break;
1785
1786                 default:
1787                         rsp->status = -1;
1788                         return rsp;
1789                 }
1790
1791                 entry_arp1.slab[2] =
1792                         SLAB_NBO_MACADDRSRC_ETHERTYPE(macaddr_src, ethertype_mpls);
1793                 entry_arp1.slab_offset[2] = p_rt->params.ip_hdr_offset -
1794                         (n_labels * 4 + 8);
1795
1796                 entry_arp1.data_offset = entry_arp1.slab_offset[2] - 6
1797                         - sizeof(struct rte_mbuf);
1798                 entry_arp1.ether_l2_length = n_labels * 4 + 14;
1799         }
1800
1801         rsp->status = rte_pipeline_table_entry_add(p->p,
1802                 p->table_id[0],
1803                 &key,
1804                 entry,
1805                 &rsp->key_found,
1806                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1807
1808         return rsp;
1809 }
1810
1811 void *
1812 pipeline_routing_msg_req_route_del_handler(struct pipeline *p, void *msg)
1813 {
1814         struct pipeline_routing_route_delete_msg_req *req = msg;
1815         struct pipeline_routing_route_delete_msg_rsp *rsp = msg;
1816
1817         struct rte_table_lpm_key key = {
1818                 .ip = req->key.key.ipv4.ip,
1819                 .depth = req->key.key.ipv4.depth,
1820         };
1821
1822         if (req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) {
1823                 rsp->status = -1;
1824                 return rsp;
1825         }
1826
1827         rsp->status = rte_pipeline_table_entry_delete(p->p,
1828                 p->table_id[0],
1829                 &key,
1830                 &rsp->key_found,
1831                 NULL);
1832
1833         return rsp;
1834 }
1835
1836 void *
1837 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
1838         void *msg)
1839 {
1840         struct pipeline_routing_route_add_default_msg_req *req = msg;
1841         struct pipeline_routing_route_add_default_msg_rsp *rsp = msg;
1842
1843         struct routing_table_entry default_entry = {
1844                 .head = {
1845                         .action = RTE_PIPELINE_ACTION_PORT,
1846                         {.port_id = p->port_out_id[req->port_id]},
1847                 },
1848
1849                 .flags = 0,
1850                 .port_id = 0,
1851                 .ip = 0,
1852         };
1853
1854         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1855                 p->table_id[0],
1856                 (struct rte_pipeline_table_entry *) &default_entry,
1857                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1858
1859         return rsp;
1860 }
1861
1862 void *
1863 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
1864         void *msg)
1865 {
1866         struct pipeline_routing_route_delete_default_msg_rsp *rsp = msg;
1867
1868         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1869                 p->table_id[0],
1870                 NULL);
1871
1872         return rsp;
1873 }
1874
1875 void *
1876 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p, void *msg)
1877 {
1878         struct pipeline_routing_arp_add_msg_req *req = msg;
1879         struct pipeline_routing_arp_add_msg_rsp *rsp = msg;
1880
1881         struct pipeline_routing_arp_key_ipv4 key = {
1882                 .port_id = req->key.key.ipv4.port_id,
1883                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1884         };
1885
1886         struct arp_table_entry entry = {
1887                 .head = {
1888                         .action = RTE_PIPELINE_ACTION_PORT,
1889                         {.port_id = p->port_out_id[req->key.key.ipv4.port_id]},
1890                 },
1891
1892                 .macaddr = 0, /* set below */
1893         };
1894
1895         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1896                 rsp->status = -1;
1897                 return rsp;
1898         }
1899
1900         entry.macaddr = *((uint64_t *)&(req->macaddr));
1901         entry.macaddr = entry.macaddr << 16;
1902
1903         rsp->status = rte_pipeline_table_entry_add(p->p,
1904                 p->table_id[1],
1905                 &key,
1906                 (struct rte_pipeline_table_entry *) &entry,
1907                 &rsp->key_found,
1908                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1909
1910         return rsp;
1911 }
1912
1913 void *
1914 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p, void *msg)
1915 {
1916         struct pipeline_routing_arp_delete_msg_req *req = msg;
1917         struct pipeline_routing_arp_delete_msg_rsp *rsp = msg;
1918
1919         struct pipeline_routing_arp_key_ipv4 key = {
1920                 .port_id = req->key.key.ipv4.port_id,
1921                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1922         };
1923
1924         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1925                 rsp->status = -1;
1926                 return rsp;
1927         }
1928
1929         rsp->status = rte_pipeline_table_entry_delete(p->p,
1930                 p->table_id[1],
1931                 &key,
1932                 &rsp->key_found,
1933                 NULL);
1934
1935         return rsp;
1936 }
1937
1938 void *
1939 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p, void *msg)
1940 {
1941         struct pipeline_routing_arp_add_default_msg_req *req = msg;
1942         struct pipeline_routing_arp_add_default_msg_rsp *rsp = msg;
1943
1944         struct arp_table_entry default_entry = {
1945                 .head = {
1946                         .action = RTE_PIPELINE_ACTION_PORT,
1947                         {.port_id = p->port_out_id[req->port_id]},
1948                 },
1949
1950                 .macaddr = 0,
1951         };
1952
1953         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1954                 p->table_id[1],
1955                 (struct rte_pipeline_table_entry *) &default_entry,
1956                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1957
1958         return rsp;
1959 }
1960
1961 void *
1962 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p, void *msg)
1963 {
1964         struct pipeline_routing_arp_delete_default_msg_rsp *rsp = msg;
1965
1966         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1967                 p->table_id[1],
1968                 NULL);
1969
1970         return rsp;
1971 }
1972
1973 void *
1974 pipeline_routing_msg_req_set_macaddr_handler(struct pipeline *p, void *msg)
1975 {
1976         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1977         struct pipeline_routing_set_macaddr_msg_req *req = msg;
1978         struct pipeline_routing_set_macaddr_msg_rsp *rsp = msg;
1979         uint32_t port_id;
1980
1981         for (port_id = 0; port_id < p->n_ports_out; port_id++)
1982                 p_rt->macaddr[port_id] = req->macaddr[port_id];
1983
1984         rsp->status = 0;
1985
1986         return rsp;
1987 }
1988
1989 struct pipeline_be_ops pipeline_routing_be_ops = {
1990         .f_init = pipeline_routing_init,
1991         .f_free = pipeline_routing_free,
1992         .f_run = NULL,
1993         .f_timer = pipeline_routing_timer,
1994 };