examples/ip_pipeline/pipeline/pipeline_routing_be.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_malloc.h>
  42 #include <rte_ip.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_table_lpm.h>
  45 #include <rte_table_hash.h>
  46 #include <rte_pipeline.h>
  47
  48 #include "pipeline_routing_be.h"
  49 #include "pipeline_actions_common.h"
  50 #include "hash_func.h"
  51
  52 #define MPLS_LABEL(label, exp, s, ttl)                                  \
  53         (((((uint64_t) (label)) & 0xFFFFFLLU) << 12) |          \
  54         ((((uint64_t) (exp)) & 0x7LLU) << 9) |                          \
  55         ((((uint64_t) (s)) & 0x1LLU) << 8) |                            \
  56         (((uint64_t) (ttl)) & 0xFFLU))
  57
  58 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe,         \
  59         traffic_class, queue, color)                            \
  60         ((((uint64_t) (queue)) & 0x3) |                \
  61         ((((uint64_t) (traffic_class)) & 0x3) << 2) |  \
  62         ((((uint64_t) (color)) & 0x3) << 4) |          \
  63         ((((uint64_t) (subport)) & 0xFFFF) << 16) |    \
  64         ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
  65
  66
  67 #define MAC_SRC_DEFAULT 0x112233445566
  68
  69 struct pipeline_routing {
  70         struct pipeline p;
  71         struct pipeline_routing_params params;
  72         pipeline_msg_req_handler custom_handlers[PIPELINE_ROUTING_MSG_REQS];
  73 } __rte_cache_aligned;
  74
  75 /*
  76  * Message handlers
  77  */
  78 static void *
  79 pipeline_routing_msg_req_custom_handler(struct pipeline *p, void *msg);
  80
  81 static pipeline_msg_req_handler handlers[] = {
  82         [PIPELINE_MSG_REQ_PING] =
  83                 pipeline_msg_req_ping_handler,
  84         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  85                 pipeline_msg_req_stats_port_in_handler,
  86         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  87                 pipeline_msg_req_stats_port_out_handler,
  88         [PIPELINE_MSG_REQ_STATS_TABLE] =
  89                 pipeline_msg_req_stats_table_handler,
  90         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  91                 pipeline_msg_req_port_in_enable_handler,
  92         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
  93                 pipeline_msg_req_port_in_disable_handler,
  94         [PIPELINE_MSG_REQ_CUSTOM] =
  95                 pipeline_routing_msg_req_custom_handler,
  96 };
  97
  98 static void *
  99 pipeline_routing_msg_req_route_add_handler(struct pipeline *p,
 100         void *msg);
 101
 102 static void *
 103 pipeline_routing_msg_req_route_del_handler(struct pipeline *p,
 104         void *msg);
 105
 106 static void *
 107 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
 108         void *msg);
 109
 110 static void *
 111 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
 112         void *msg);
 113
 114 static void *
 115 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p,
 116         void *msg);
 117
 118 static void *
 119 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p,
 120         void *msg);
 121
 122 static void *
 123 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p,
 124         void *msg);
 125
 126 static void *
 127 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p,
 128         void *msg);
 129
 130 static pipeline_msg_req_handler custom_handlers[] = {
 131         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD] =
 132                 pipeline_routing_msg_req_route_add_handler,
 133         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL] =
 134                 pipeline_routing_msg_req_route_del_handler,
 135         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD_DEFAULT] =
 136                 pipeline_routing_msg_req_route_add_default_handler,
 137         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL_DEFAULT] =
 138                 pipeline_routing_msg_req_route_del_default_handler,
 139         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD] =
 140                 pipeline_routing_msg_req_arp_add_handler,
 141         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL] =
 142                 pipeline_routing_msg_req_arp_del_handler,
 143         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD_DEFAULT] =
 144                 pipeline_routing_msg_req_arp_add_default_handler,
 145         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL_DEFAULT] =
 146                 pipeline_routing_msg_req_arp_del_default_handler,
 147 };
 148
 149 /*
 150  * Routing table
 151  */
 152 struct routing_table_entry {
 153         struct rte_pipeline_table_entry head;
 154         uint32_t flags;
 155         uint32_t port_id; /* Output port ID */
 156         uint32_t ip; /* Next hop IP address (only valid for remote routes) */
 157
 158         /* ether_l2 */
 159         uint16_t data_offset;
 160         uint16_t ether_l2_length;
 161         uint64_t slab[4];
 162         uint16_t slab_offset[4];
 163 };
 164
 165 struct layout {
 166         uint16_t a;
 167         uint32_t b;
 168         uint16_t c;
 169 } __attribute__((__packed__));
 170
 171 #define MACADDR_DST_WRITE(slab_ptr, slab)                       \
 172 {                                                               \
 173         struct layout *dst = (struct layout *) (slab_ptr);      \
 174         struct layout *src = (struct layout *) &(slab);         \
 175                                                                 \
 176         dst->b = src->b;                                        \
 177         dst->c = src->c;                                        \
 178 }
 179
 180 static inline __attribute__((always_inline)) void
 181 pkt_work_routing(
 182         struct rte_mbuf *pkt,
 183         struct rte_pipeline_table_entry *table_entry,
 184         void *arg,
 185         int arp,
 186         int qinq,
 187         int qinq_sched,
 188         int mpls,
 189         int mpls_color_mark)
 190 {
 191         struct pipeline_routing *p_rt = arg;
 192
 193         struct routing_table_entry *entry =
 194                 (struct routing_table_entry *) table_entry;
 195
 196         struct ipv4_hdr *ip = (struct ipv4_hdr *)
 197                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.ip_hdr_offset);
 198
 199         enum rte_meter_color pkt_color = (enum rte_meter_color)
 200                 RTE_MBUF_METADATA_UINT32(pkt, p_rt->params.color_offset);
 201
 202         struct pipeline_routing_arp_key_ipv4 *arp_key =
 203                 (struct pipeline_routing_arp_key_ipv4 *)
 204                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.arp_key_offset);
 205
 206         uint64_t *slab0_ptr, *slab1_ptr, *slab2_ptr, *slab3_ptr, sched;
 207         uint32_t ip_da, nh_ip, port_id;
 208         uint16_t total_length, data_offset, ether_l2_length;
 209
 210         /* Read */
 211         total_length = rte_bswap16(ip->total_length);
 212         ip_da = ip->dst_addr;
 213         data_offset = entry->data_offset;
 214         ether_l2_length = entry->ether_l2_length;
 215         slab0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[0]);
 216         slab1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[1]);
 217         slab2_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[2]);
 218         slab3_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[3]);
 219
 220         if (arp) {
 221                 port_id = entry->port_id;
 222                 nh_ip = entry->ip;
 223                 if (entry->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 224                         nh_ip = ip_da;
 225         }
 226
 227         /* Compute */
 228         total_length += ether_l2_length;
 229
 230         if (qinq && qinq_sched) {
 231                 uint32_t dscp = ip->type_of_service >> 2;
 232                 uint32_t svlan, cvlan, tc, tc_q;
 233
 234                 if (qinq_sched == 1) {
 235                         uint64_t slab_qinq = rte_bswap64(entry->slab[0]);
 236
 237                         svlan = (slab_qinq >> 48) & 0xFFF;
 238                         cvlan = (slab_qinq >> 16) & 0xFFF;
 239                         tc = (dscp >> 2) & 0x3;
 240                         tc_q = dscp & 0x3;
 241                 } else {
 242                         uint32_t ip_src = rte_bswap32(ip->src_addr);
 243
 244                         svlan = 0;
 245                         cvlan = (ip_src >> 16) & 0xFFF;
 246                         tc = (ip_src >> 2) & 0x3;
 247                         tc_q = ip_src & 0x3;
 248                 }
 249                 sched = RTE_SCHED_PORT_HIERARCHY(svlan,
 250                         cvlan,
 251                         tc,
 252                         tc_q,
 253                         e_RTE_METER_GREEN);
 254         }
 255
 256         /* Write */
 257         pkt->data_off = data_offset;
 258         pkt->data_len = total_length;
 259         pkt->pkt_len = total_length;
 260
 261         if ((qinq == 0) && (mpls == 0)) {
 262                 *slab0_ptr = entry->slab[0];
 263
 264                 if (arp == 0)
 265                         MACADDR_DST_WRITE(slab1_ptr, entry->slab[1]);
 266         }
 267
 268         if (qinq) {
 269                 *slab0_ptr = entry->slab[0];
 270                 *slab1_ptr = entry->slab[1];
 271
 272                 if (arp == 0)
 273                         MACADDR_DST_WRITE(slab2_ptr, entry->slab[2]);
 274
 275                 if (qinq_sched) {
 276                         pkt->hash.sched.lo = sched & 0xFFFFFFFF;
 277                         pkt->hash.sched.hi = sched >> 32;
 278                 }
 279         }
 280
 281         if (mpls) {
 282                 if (mpls_color_mark) {
 283                         uint64_t mpls_exp = rte_bswap64(
 284                                 (MPLS_LABEL(0, pkt_color, 0, 0) << 32) |
 285                                 MPLS_LABEL(0, pkt_color, 0, 0));
 286
 287                         *slab0_ptr = entry->slab[0] | mpls_exp;
 288                         *slab1_ptr = entry->slab[1] | mpls_exp;
 289                         *slab2_ptr = entry->slab[2];
 290                 } else {
 291                         *slab0_ptr = entry->slab[0];
 292                         *slab1_ptr = entry->slab[1];
 293                         *slab2_ptr = entry->slab[2];
 294                 }
 295
 296                 if (arp == 0)
 297                         MACADDR_DST_WRITE(slab3_ptr, entry->slab[3]);
 298         }
 299
 300         if (arp) {
 301                 arp_key->port_id = port_id;
 302                 arp_key->ip = nh_ip;
 303         }
 304 }
 305
 306 static inline __attribute__((always_inline)) void
 307 pkt4_work_routing(
 308         struct rte_mbuf **pkts,
 309         struct rte_pipeline_table_entry **table_entries,
 310         void *arg,
 311         int arp,
 312         int qinq,
 313         int qinq_sched,
 314         int mpls,
 315         int mpls_color_mark)
 316 {
 317         struct pipeline_routing *p_rt = arg;
 318
 319         struct routing_table_entry *entry0 =
 320                 (struct routing_table_entry *) table_entries[0];
 321         struct routing_table_entry *entry1 =
 322                 (struct routing_table_entry *) table_entries[1];
 323         struct routing_table_entry *entry2 =
 324                 (struct routing_table_entry *) table_entries[2];
 325         struct routing_table_entry *entry3 =
 326                 (struct routing_table_entry *) table_entries[3];
 327
 328         struct ipv4_hdr *ip0 = (struct ipv4_hdr *)
 329                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 330                         p_rt->params.ip_hdr_offset);
 331         struct ipv4_hdr *ip1 = (struct ipv4_hdr *)
 332                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 333                         p_rt->params.ip_hdr_offset);
 334         struct ipv4_hdr *ip2 = (struct ipv4_hdr *)
 335                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 336                         p_rt->params.ip_hdr_offset);
 337         struct ipv4_hdr *ip3 = (struct ipv4_hdr *)
 338                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 339                         p_rt->params.ip_hdr_offset);
 340
 341         enum rte_meter_color pkt0_color = (enum rte_meter_color)
 342                 RTE_MBUF_METADATA_UINT32(pkts[0], p_rt->params.color_offset);
 343         enum rte_meter_color pkt1_color = (enum rte_meter_color)
 344                 RTE_MBUF_METADATA_UINT32(pkts[1], p_rt->params.color_offset);
 345         enum rte_meter_color pkt2_color = (enum rte_meter_color)
 346                 RTE_MBUF_METADATA_UINT32(pkts[2], p_rt->params.color_offset);
 347         enum rte_meter_color pkt3_color = (enum rte_meter_color)
 348                 RTE_MBUF_METADATA_UINT32(pkts[3], p_rt->params.color_offset);
 349
 350         struct pipeline_routing_arp_key_ipv4 *arp_key0 =
 351                 (struct pipeline_routing_arp_key_ipv4 *)
 352                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 353                         p_rt->params.arp_key_offset);
 354         struct pipeline_routing_arp_key_ipv4 *arp_key1 =
 355                 (struct pipeline_routing_arp_key_ipv4 *)
 356                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 357                         p_rt->params.arp_key_offset);
 358         struct pipeline_routing_arp_key_ipv4 *arp_key2 =
 359                 (struct pipeline_routing_arp_key_ipv4 *)
 360                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 361                         p_rt->params.arp_key_offset);
 362         struct pipeline_routing_arp_key_ipv4 *arp_key3 =
 363                 (struct pipeline_routing_arp_key_ipv4 *)
 364                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 365                         p_rt->params.arp_key_offset);
 366
 367         uint64_t *slab0_ptr0, *slab1_ptr0, *slab2_ptr0, *slab3_ptr0;
 368         uint64_t *slab0_ptr1, *slab1_ptr1, *slab2_ptr1, *slab3_ptr1;
 369         uint64_t *slab0_ptr2, *slab1_ptr2, *slab2_ptr2, *slab3_ptr2;
 370         uint64_t *slab0_ptr3, *slab1_ptr3, *slab2_ptr3, *slab3_ptr3;
 371         uint64_t sched0, sched1, sched2, sched3;
 372
 373         uint32_t ip_da0, nh_ip0, port_id0;
 374         uint32_t ip_da1, nh_ip1, port_id1;
 375         uint32_t ip_da2, nh_ip2, port_id2;
 376         uint32_t ip_da3, nh_ip3, port_id3;
 377
 378         uint16_t total_length0, data_offset0, ether_l2_length0;
 379         uint16_t total_length1, data_offset1, ether_l2_length1;
 380         uint16_t total_length2, data_offset2, ether_l2_length2;
 381         uint16_t total_length3, data_offset3, ether_l2_length3;
 382
 383         /* Read */
 384         total_length0 = rte_bswap16(ip0->total_length);
 385         total_length1 = rte_bswap16(ip1->total_length);
 386         total_length2 = rte_bswap16(ip2->total_length);
 387         total_length3 = rte_bswap16(ip3->total_length);
 388
 389         ip_da0 = ip0->dst_addr;
 390         ip_da1 = ip1->dst_addr;
 391         ip_da2 = ip2->dst_addr;
 392         ip_da3 = ip3->dst_addr;
 393
 394         data_offset0 = entry0->data_offset;
 395         data_offset1 = entry1->data_offset;
 396         data_offset2 = entry2->data_offset;
 397         data_offset3 = entry3->data_offset;
 398
 399         ether_l2_length0 = entry0->ether_l2_length;
 400         ether_l2_length1 = entry1->ether_l2_length;
 401         ether_l2_length2 = entry2->ether_l2_length;
 402         ether_l2_length3 = entry3->ether_l2_length;
 403
 404         slab0_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 405                 entry0->slab_offset[0]);
 406         slab1_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 407                 entry0->slab_offset[1]);
 408         slab2_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 409                 entry0->slab_offset[2]);
 410         slab3_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 411                 entry0->slab_offset[3]);
 412
 413         slab0_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 414                 entry1->slab_offset[0]);
 415         slab1_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 416                 entry1->slab_offset[1]);
 417         slab2_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 418                 entry1->slab_offset[2]);
 419         slab3_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 420                 entry1->slab_offset[3]);
 421
 422         slab0_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 423                 entry2->slab_offset[0]);
 424         slab1_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 425                 entry2->slab_offset[1]);
 426         slab2_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 427                 entry2->slab_offset[2]);
 428         slab3_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 429                 entry2->slab_offset[3]);
 430
 431         slab0_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 432                 entry3->slab_offset[0]);
 433         slab1_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 434                 entry3->slab_offset[1]);
 435         slab2_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 436                 entry3->slab_offset[2]);
 437         slab3_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 438                 entry3->slab_offset[3]);
 439
 440         if (arp) {
 441                 port_id0 = entry0->port_id;
 442                 nh_ip0 = entry0->ip;
 443                 if (entry0->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 444                         nh_ip0 = ip_da0;
 445
 446                 port_id1 = entry1->port_id;
 447                 nh_ip1 = entry1->ip;
 448                 if (entry1->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 449                         nh_ip1 = ip_da1;
 450
 451                 port_id2 = entry2->port_id;
 452                 nh_ip2 = entry2->ip;
 453                 if (entry2->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 454                         nh_ip2 = ip_da2;
 455
 456                 port_id3 = entry3->port_id;
 457                 nh_ip3 = entry3->ip;
 458                 if (entry3->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 459                         nh_ip3 = ip_da3;
 460         }
 461
 462         /* Compute */
 463         total_length0 += ether_l2_length0;
 464         total_length1 += ether_l2_length1;
 465         total_length2 += ether_l2_length2;
 466         total_length3 += ether_l2_length3;
 467
 468         if (qinq && qinq_sched) {
 469                 uint32_t dscp0 = ip0->type_of_service >> 2;
 470                 uint32_t dscp1 = ip1->type_of_service >> 2;
 471                 uint32_t dscp2 = ip2->type_of_service >> 2;
 472                 uint32_t dscp3 = ip3->type_of_service >> 2;
 473                 uint32_t svlan0, cvlan0, tc0, tc_q0;
 474                 uint32_t svlan1, cvlan1, tc1, tc_q1;
 475                 uint32_t svlan2, cvlan2, tc2, tc_q2;
 476                 uint32_t svlan3, cvlan3, tc3, tc_q3;
 477
 478                 if (qinq_sched == 1) {
 479                         uint64_t slab_qinq0 = rte_bswap64(entry0->slab[0]);
 480                         uint64_t slab_qinq1 = rte_bswap64(entry1->slab[0]);
 481                         uint64_t slab_qinq2 = rte_bswap64(entry2->slab[0]);
 482                         uint64_t slab_qinq3 = rte_bswap64(entry3->slab[0]);
 483
 484                         svlan0 = (slab_qinq0 >> 48) & 0xFFF;
 485                         svlan1 = (slab_qinq1 >> 48) & 0xFFF;
 486                         svlan2 = (slab_qinq2 >> 48) & 0xFFF;
 487                         svlan3 = (slab_qinq3 >> 48) & 0xFFF;
 488
 489                         cvlan0 = (slab_qinq0 >> 16) & 0xFFF;
 490                         cvlan1 = (slab_qinq1 >> 16) & 0xFFF;
 491                         cvlan2 = (slab_qinq2 >> 16) & 0xFFF;
 492                         cvlan3 = (slab_qinq3 >> 16) & 0xFFF;
 493
 494                         tc0 = (dscp0 >> 2) & 0x3;
 495                         tc1 = (dscp1 >> 2) & 0x3;
 496                         tc2 = (dscp2 >> 2) & 0x3;
 497                         tc3 = (dscp3 >> 2) & 0x3;
 498
 499                         tc_q0 = dscp0 & 0x3;
 500                         tc_q1 = dscp1 & 0x3;
 501                         tc_q2 = dscp2 & 0x3;
 502                         tc_q3 = dscp3 & 0x3;
 503                 } else {
 504                         uint32_t ip_src0 = rte_bswap32(ip0->src_addr);
 505                         uint32_t ip_src1 = rte_bswap32(ip1->src_addr);
 506                         uint32_t ip_src2 = rte_bswap32(ip2->src_addr);
 507                         uint32_t ip_src3 = rte_bswap32(ip3->src_addr);
 508
 509                         svlan0 = 0;
 510                         svlan1 = 0;
 511                         svlan2 = 0;
 512                         svlan3 = 0;
 513
 514                         cvlan0 = (ip_src0 >> 16) & 0xFFF;
 515                         cvlan1 = (ip_src1 >> 16) & 0xFFF;
 516                         cvlan2 = (ip_src2 >> 16) & 0xFFF;
 517                         cvlan3 = (ip_src3 >> 16) & 0xFFF;
 518
 519                         tc0 = (ip_src0 >> 2) & 0x3;
 520                         tc1 = (ip_src1 >> 2) & 0x3;
 521                         tc2 = (ip_src2 >> 2) & 0x3;
 522                         tc3 = (ip_src3 >> 2) & 0x3;
 523
 524                         tc_q0 = ip_src0 & 0x3;
 525                         tc_q1 = ip_src1 & 0x3;
 526                         tc_q2 = ip_src2 & 0x3;
 527                         tc_q3 = ip_src3 & 0x3;
 528                 }
 529
 530                 sched0 = RTE_SCHED_PORT_HIERARCHY(svlan0,
 531                         cvlan0,
 532                         tc0,
 533                         tc_q0,
 534                         e_RTE_METER_GREEN);
 535                 sched1 = RTE_SCHED_PORT_HIERARCHY(svlan1,
 536                         cvlan1,
 537                         tc1,
 538                         tc_q1,
 539                         e_RTE_METER_GREEN);
 540                 sched2 = RTE_SCHED_PORT_HIERARCHY(svlan2,
 541                         cvlan2,
 542                         tc2,
 543                         tc_q2,
 544                         e_RTE_METER_GREEN);
 545                 sched3 = RTE_SCHED_PORT_HIERARCHY(svlan3,
 546                         cvlan3,
 547                         tc3,
 548                         tc_q3,
 549                         e_RTE_METER_GREEN);
 550
 551         }
 552
 553         /* Write */
 554         pkts[0]->data_off = data_offset0;
 555         pkts[1]->data_off = data_offset1;
 556         pkts[2]->data_off = data_offset2;
 557         pkts[3]->data_off = data_offset3;
 558
 559         pkts[0]->data_len = total_length0;
 560         pkts[1]->data_len = total_length1;
 561         pkts[2]->data_len = total_length2;
 562         pkts[3]->data_len = total_length3;
 563
 564         pkts[0]->pkt_len = total_length0;
 565         pkts[1]->pkt_len = total_length1;
 566         pkts[2]->pkt_len = total_length2;
 567         pkts[3]->pkt_len = total_length3;
 568
 569         if ((qinq == 0) && (mpls == 0)) {
 570                 *slab0_ptr0 = entry0->slab[0];
 571                 *slab0_ptr1 = entry1->slab[0];
 572                 *slab0_ptr2 = entry2->slab[0];
 573                 *slab0_ptr3 = entry3->slab[0];
 574
 575                 if (arp == 0) {
 576                         MACADDR_DST_WRITE(slab1_ptr0, entry0->slab[1]);
 577                         MACADDR_DST_WRITE(slab1_ptr1, entry1->slab[1]);
 578                         MACADDR_DST_WRITE(slab1_ptr2, entry2->slab[1]);
 579                         MACADDR_DST_WRITE(slab1_ptr3, entry3->slab[1]);
 580                 }
 581         }
 582
 583         if (qinq) {
 584                 *slab0_ptr0 = entry0->slab[0];
 585                 *slab0_ptr1 = entry1->slab[0];
 586                 *slab0_ptr2 = entry2->slab[0];
 587                 *slab0_ptr3 = entry3->slab[0];
 588
 589                 *slab1_ptr0 = entry0->slab[1];
 590                 *slab1_ptr1 = entry1->slab[1];
 591                 *slab1_ptr2 = entry2->slab[1];
 592                 *slab1_ptr3 = entry3->slab[1];
 593
 594                 if (arp == 0) {
 595                         MACADDR_DST_WRITE(slab2_ptr0, entry0->slab[2]);
 596                         MACADDR_DST_WRITE(slab2_ptr1, entry1->slab[2]);
 597                         MACADDR_DST_WRITE(slab2_ptr2, entry2->slab[2]);
 598                         MACADDR_DST_WRITE(slab2_ptr3, entry3->slab[2]);
 599                 }
 600
 601                 if (qinq_sched) {
 602                         pkts[0]->hash.sched.lo = sched0 & 0xFFFFFFFF;
 603                         pkts[0]->hash.sched.hi = sched0 >> 32;
 604                         pkts[1]->hash.sched.lo = sched1 & 0xFFFFFFFF;
 605                         pkts[1]->hash.sched.hi = sched1 >> 32;
 606                         pkts[2]->hash.sched.lo = sched2 & 0xFFFFFFFF;
 607                         pkts[2]->hash.sched.hi = sched2 >> 32;
 608                         pkts[3]->hash.sched.lo = sched3 & 0xFFFFFFFF;
 609                         pkts[3]->hash.sched.hi = sched3 >> 32;
 610                 }
 611         }
 612
 613         if (mpls) {
 614                 if (mpls_color_mark) {
 615                         uint64_t mpls_exp0 = rte_bswap64(
 616                                 (MPLS_LABEL(0, pkt0_color, 0, 0) << 32) |
 617                                 MPLS_LABEL(0, pkt0_color, 0, 0));
 618                         uint64_t mpls_exp1 = rte_bswap64(
 619                                 (MPLS_LABEL(0, pkt1_color, 0, 0) << 32) |
 620                                 MPLS_LABEL(0, pkt1_color, 0, 0));
 621                         uint64_t mpls_exp2 = rte_bswap64(
 622                                 (MPLS_LABEL(0, pkt2_color, 0, 0) << 32) |
 623                                 MPLS_LABEL(0, pkt2_color, 0, 0));
 624                         uint64_t mpls_exp3 = rte_bswap64(
 625                                 (MPLS_LABEL(0, pkt3_color, 0, 0) << 32) |
 626                                 MPLS_LABEL(0, pkt3_color, 0, 0));
 627
 628                         *slab0_ptr0 = entry0->slab[0] | mpls_exp0;
 629                         *slab0_ptr1 = entry1->slab[0] | mpls_exp1;
 630                         *slab0_ptr2 = entry2->slab[0] | mpls_exp2;
 631                         *slab0_ptr3 = entry3->slab[0] | mpls_exp3;
 632
 633                         *slab1_ptr0 = entry0->slab[1] | mpls_exp0;
 634                         *slab1_ptr1 = entry1->slab[1] | mpls_exp1;
 635                         *slab1_ptr2 = entry2->slab[1] | mpls_exp2;
 636                         *slab1_ptr3 = entry3->slab[1] | mpls_exp3;
 637
 638                         *slab2_ptr0 = entry0->slab[2];
 639                         *slab2_ptr1 = entry1->slab[2];
 640                         *slab2_ptr2 = entry2->slab[2];
 641                         *slab2_ptr3 = entry3->slab[2];
 642                 } else {
 643                         *slab0_ptr0 = entry0->slab[0];
 644                         *slab0_ptr1 = entry1->slab[0];
 645                         *slab0_ptr2 = entry2->slab[0];
 646                         *slab0_ptr3 = entry3->slab[0];
 647
 648                         *slab1_ptr0 = entry0->slab[1];
 649                         *slab1_ptr1 = entry1->slab[1];
 650                         *slab1_ptr2 = entry2->slab[1];
 651                         *slab1_ptr3 = entry3->slab[1];
 652
 653                         *slab2_ptr0 = entry0->slab[2];
 654                         *slab2_ptr1 = entry1->slab[2];
 655                         *slab2_ptr2 = entry2->slab[2];
 656                         *slab2_ptr3 = entry3->slab[2];
 657                 }
 658
 659                 if (arp == 0) {
 660                         MACADDR_DST_WRITE(slab3_ptr0, entry0->slab[3]);
 661                         MACADDR_DST_WRITE(slab3_ptr1, entry1->slab[3]);
 662                         MACADDR_DST_WRITE(slab3_ptr2, entry2->slab[3]);
 663                         MACADDR_DST_WRITE(slab3_ptr3, entry3->slab[3]);
 664                 }
 665         }
 666
 667         if (arp) {
 668                 arp_key0->port_id = port_id0;
 669                 arp_key1->port_id = port_id1;
 670                 arp_key2->port_id = port_id2;
 671                 arp_key3->port_id = port_id3;
 672
 673                 arp_key0->ip = nh_ip0;
 674                 arp_key1->ip = nh_ip1;
 675                 arp_key2->ip = nh_ip2;
 676                 arp_key3->ip = nh_ip3;
 677         }
 678 }
 679
 680 #define PKT_WORK_ROUTING_ETHERNET(arp)                          \
 681 static inline void                                              \
 682 pkt_work_routing_ether_arp##arp(                                \
 683         struct rte_mbuf *pkt,                                   \
 684         struct rte_pipeline_table_entry *table_entry,           \
 685         void *arg)                                              \
 686 {                                                               \
 687         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 0, 0);\
 688 }
 689
 690 #define PKT4_WORK_ROUTING_ETHERNET(arp)                         \
 691 static inline void                                              \
 692 pkt4_work_routing_ether_arp##arp(                               \
 693         struct rte_mbuf **pkts,                                 \
 694         struct rte_pipeline_table_entry **table_entries,        \
 695         void *arg)                                              \
 696 {                                                               \
 697         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 0, 0);\
 698 }
 699
 700 #define routing_table_ah_hit_ether(arp)                         \
 701 PKT_WORK_ROUTING_ETHERNET(arp)                                  \
 702 PKT4_WORK_ROUTING_ETHERNET(arp)                                 \
 703 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_arp##arp,      \
 704         pkt_work_routing_ether_arp##arp,                        \
 705         pkt4_work_routing_ether_arp##arp)
 706
 707 routing_table_ah_hit_ether(0)
 708 routing_table_ah_hit_ether(1)
 709
 710 #define PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)              \
 711 static inline void                                              \
 712 pkt_work_routing_ether_qinq_sched##sched##_arp##arp(            \
 713         struct rte_mbuf *pkt,                                   \
 714         struct rte_pipeline_table_entry *table_entry,           \
 715         void *arg)                                              \
 716 {                                                               \
 717         pkt_work_routing(pkt, table_entry, arg, arp, 1, sched, 0, 0);\
 718 }
 719
 720 #define PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)             \
 721 static inline void                                              \
 722 pkt4_work_routing_ether_qinq_sched##sched##_arp##arp(           \
 723         struct rte_mbuf **pkts,                                 \
 724         struct rte_pipeline_table_entry **table_entries,        \
 725         void *arg)                                              \
 726 {                                                               \
 727         pkt4_work_routing(pkts, table_entries, arg, arp, 1, sched, 0, 0);\
 728 }
 729
 730 #define routing_table_ah_hit_ether_qinq(sched, arp)             \
 731 PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                      \
 732 PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                     \
 733 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_qinq_sched##sched##_arp##arp,\
 734         pkt_work_routing_ether_qinq_sched##sched##_arp##arp,    \
 735         pkt4_work_routing_ether_qinq_sched##sched##_arp##arp)
 736
 737 routing_table_ah_hit_ether_qinq(0, 0)
 738 routing_table_ah_hit_ether_qinq(1, 0)
 739 routing_table_ah_hit_ether_qinq(2, 0)
 740 routing_table_ah_hit_ether_qinq(0, 1)
 741 routing_table_ah_hit_ether_qinq(1, 1)
 742 routing_table_ah_hit_ether_qinq(2, 1)
 743
 744 #define PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)              \
 745 static inline void                                              \
 746 pkt_work_routing_ether_mpls_color##color##_arp##arp(            \
 747         struct rte_mbuf *pkt,                                   \
 748         struct rte_pipeline_table_entry *table_entry,           \
 749         void *arg)                                              \
 750 {                                                               \
 751         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 1, color);\
 752 }
 753
 754 #define PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)             \
 755 static inline void                                              \
 756 pkt4_work_routing_ether_mpls_color##color##_arp##arp(           \
 757         struct rte_mbuf **pkts,                                 \
 758         struct rte_pipeline_table_entry **table_entries,        \
 759         void *arg)                                              \
 760 {                                                               \
 761         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 1, color);\
 762 }
 763
 764 #define routing_table_ah_hit_ether_mpls(color, arp)             \
 765 PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)                      \
 766 PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)                     \
 767 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_mpls_color##color##_arp##arp,\
 768         pkt_work_routing_ether_mpls_color##color##_arp##arp,    \
 769         pkt4_work_routing_ether_mpls_color##color##_arp##arp)
 770
 771 routing_table_ah_hit_ether_mpls(0, 0)
 772 routing_table_ah_hit_ether_mpls(1, 0)
 773 routing_table_ah_hit_ether_mpls(0, 1)
 774 routing_table_ah_hit_ether_mpls(1, 1)
 775
 776 static rte_pipeline_table_action_handler_hit
 777 get_routing_table_ah_hit(struct pipeline_routing *p)
 778 {
 779         if (p->params.dbg_ah_disable)
 780                 return NULL;
 781
 782         switch (p->params.encap) {
 783         case PIPELINE_ROUTING_ENCAP_ETHERNET:
 784                 return (p->params.n_arp_entries) ?
 785                         routing_table_ah_hit_ether_arp1 :
 786                         routing_table_ah_hit_ether_arp0;
 787
 788         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
 789                 if (p->params.n_arp_entries)
 790                         switch (p->params.qinq_sched) {
 791                         case 0:
 792                                 return routing_table_ah_hit_ether_qinq_sched0_arp1;
 793                         case 1:
 794                                 return routing_table_ah_hit_ether_qinq_sched1_arp1;
 795                         case 2:
 796                                 return routing_table_ah_hit_ether_qinq_sched2_arp1;
 797                         default:
 798                                 return NULL;
 799                         }
 800                  else
 801                         switch (p->params.qinq_sched) {
 802                         case 0:
 803                                 return routing_table_ah_hit_ether_qinq_sched0_arp0;
 804                         case 1:
 805                                 return routing_table_ah_hit_ether_qinq_sched1_arp0;
 806                         case 2:
 807                                 return routing_table_ah_hit_ether_qinq_sched2_arp0;
 808                         default:
 809                                 return NULL;
 810                         }
 811
 812         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
 813                 if (p->params.n_arp_entries)
 814                         if (p->params.mpls_color_mark)
 815                                 return routing_table_ah_hit_ether_mpls_color1_arp1;
 816                         else
 817                                 return routing_table_ah_hit_ether_mpls_color0_arp1;
 818                 else
 819                         if (p->params.mpls_color_mark)
 820                                 return routing_table_ah_hit_ether_mpls_color1_arp0;
 821                         else
 822                                 return routing_table_ah_hit_ether_mpls_color0_arp0;
 823
 824         default:
 825                 return NULL;
 826         }
 827 }
 828
 829 /*
 830  * ARP table
 831  */
 832 struct arp_table_entry {
 833         struct rte_pipeline_table_entry head;
 834         uint64_t macaddr;
 835 };
 836
 837 /**
 838  * ARP table AH
 839  */
 840 static inline void
 841 pkt_work_arp(
 842         struct rte_mbuf *pkt,
 843         struct rte_pipeline_table_entry *table_entry,
 844         __rte_unused void *arg)
 845 {
 846         struct arp_table_entry *entry = (struct arp_table_entry *) table_entry;
 847
 848         /* Read */
 849         uint64_t macaddr_dst = entry->macaddr;
 850         uint64_t *slab_ptr = (uint64_t *) ((char *) pkt->buf_addr +
 851                 (pkt->data_off - 2));
 852
 853         /* Compute */
 854
 855         /* Write */
 856         MACADDR_DST_WRITE(slab_ptr, macaddr_dst);
 857 }
 858
 859 static inline void
 860 pkt4_work_arp(
 861         struct rte_mbuf **pkts,
 862         struct rte_pipeline_table_entry **table_entries,
 863         __rte_unused void *arg)
 864 {
 865         struct arp_table_entry *entry0 =
 866                 (struct arp_table_entry *) table_entries[0];
 867         struct arp_table_entry *entry1 =
 868                 (struct arp_table_entry *) table_entries[1];
 869         struct arp_table_entry *entry2 =
 870                 (struct arp_table_entry *) table_entries[2];
 871         struct arp_table_entry *entry3 =
 872                 (struct arp_table_entry *) table_entries[3];
 873
 874         /* Read */
 875         uint64_t macaddr_dst0 = entry0->macaddr;
 876         uint64_t macaddr_dst1 = entry1->macaddr;
 877         uint64_t macaddr_dst2 = entry2->macaddr;
 878         uint64_t macaddr_dst3 = entry3->macaddr;
 879
 880         uint64_t *slab_ptr0 = (uint64_t *) ((char *) pkts[0]->buf_addr +
 881                 (pkts[0]->data_off - 2));
 882         uint64_t *slab_ptr1 = (uint64_t *) ((char *) pkts[1]->buf_addr +
 883                 (pkts[1]->data_off - 2));
 884         uint64_t *slab_ptr2 = (uint64_t *) ((char *) pkts[2]->buf_addr +
 885                 (pkts[2]->data_off - 2));
 886         uint64_t *slab_ptr3 = (uint64_t *) ((char *) pkts[3]->buf_addr +
 887                 (pkts[3]->data_off - 2));
 888
 889         /* Compute */
 890
 891         /* Write */
 892         MACADDR_DST_WRITE(slab_ptr0, macaddr_dst0);
 893         MACADDR_DST_WRITE(slab_ptr1, macaddr_dst1);
 894         MACADDR_DST_WRITE(slab_ptr2, macaddr_dst2);
 895         MACADDR_DST_WRITE(slab_ptr3, macaddr_dst3);
 896 }
 897
 898 PIPELINE_TABLE_AH_HIT(arp_table_ah_hit,
 899         pkt_work_arp,
 900         pkt4_work_arp);
 901
 902 static rte_pipeline_table_action_handler_hit
 903 get_arp_table_ah_hit(struct pipeline_routing *p)
 904 {
 905         if (p->params.dbg_ah_disable)
 906                 return NULL;
 907
 908         return arp_table_ah_hit;
 909 }
 910
 911 /*
 912  * Argument parsing
 913  */
 914 int
 915 pipeline_routing_parse_args(struct pipeline_routing_params *p,
 916         struct pipeline_params *params)
 917 {
 918         uint32_t n_routes_present = 0;
 919         uint32_t encap_present = 0;
 920         uint32_t qinq_sched_present = 0;
 921         uint32_t mpls_color_mark_present = 0;
 922         uint32_t n_arp_entries_present = 0;
 923         uint32_t ip_hdr_offset_present = 0;
 924         uint32_t arp_key_offset_present = 0;
 925         uint32_t color_offset_present = 0;
 926         uint32_t dbg_ah_disable_present = 0;
 927         uint32_t i;
 928
 929         /* default values */
 930         p->n_routes = PIPELINE_ROUTING_N_ROUTES_DEFAULT;
 931         p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 932         p->qinq_sched = 0;
 933         p->mpls_color_mark = 0;
 934         p->n_arp_entries = 0;
 935         p->dbg_ah_disable = 0;
 936
 937         for (i = 0; i < params->n_args; i++) {
 938                 char *arg_name = params->args_name[i];
 939                 char *arg_value = params->args_value[i];
 940
 941                 /* n_routes */
 942                 if (strcmp(arg_name, "n_routes") == 0) {
 943                         if (n_routes_present)
 944                                 return -1;
 945                         n_routes_present = 1;
 946
 947                         p->n_routes = atoi(arg_value);
 948                         if (p->n_routes == 0)
 949                                 return -1;
 950
 951                         continue;
 952                 }
 953
 954                 /* encap */
 955                 if (strcmp(arg_name, "encap") == 0) {
 956                         if (encap_present)
 957                                 return -1;
 958                         encap_present = 1;
 959
 960                         /* ethernet */
 961                         if (strcmp(arg_value, "ethernet") == 0) {
 962                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 963                                 continue;
 964                         }
 965
 966                         /* ethernet_qinq */
 967                         if (strcmp(arg_value, "ethernet_qinq") == 0) {
 968                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ;
 969                                 continue;
 970                         }
 971
 972                         /* ethernet_mpls */
 973                         if (strcmp(arg_value, "ethernet_mpls") == 0) {
 974                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS;
 975                                 continue;
 976                         }
 977
 978                         /* any other */
 979                         return -1;
 980                 }
 981
 982                 /* qinq_sched */
 983                 if (strcmp(arg_name, "qinq_sched") == 0) {
 984                         if (qinq_sched_present)
 985                                 return -1;
 986
 987                         qinq_sched_present = 1;
 988
 989                         if (strcmp(arg_value, "no") == 0)
 990                                 p->qinq_sched = 0;
 991                         else if (strcmp(arg_value, "yes") == 0)
 992                                 p->qinq_sched = 1;
 993                         else if (strcmp(arg_value, "test") == 0)
 994                                 p->qinq_sched = 2;
 995                         else
 996                                 return -1;
 997
 998                         continue;
 999                 }
1000
1001                 /* mpls_color_mark */
1002                 if (strcmp(arg_name, "mpls_color_mark") == 0) {
1003                         if (mpls_color_mark_present)
1004                                 return -1;
1005
1006                         mpls_color_mark_present = 1;
1007
1008                         if (strcmp(arg_value, "no") == 0)
1009                                 p->mpls_color_mark = 0;
1010                         else if (strcmp(arg_value, "yes") == 0)
1011                                 p->mpls_color_mark = 1;
1012                         else
1013                                 return -1;
1014
1015                         continue;
1016                 }
1017
1018                 /* n_arp_entries */
1019                 if (strcmp(arg_name, "n_arp_entries") == 0) {
1020                         if (n_arp_entries_present)
1021                                 return -1;
1022                         n_arp_entries_present = 1;
1023
1024                         p->n_arp_entries = atoi(arg_value);
1025
1026                         continue;
1027                 }
1028
1029                 /* ip_hdr_offset */
1030                 if (strcmp(arg_name, "ip_hdr_offset") == 0) {
1031                         if (ip_hdr_offset_present)
1032                                 return -1;
1033                         ip_hdr_offset_present = 1;
1034
1035                         p->ip_hdr_offset = atoi(arg_value);
1036
1037                         continue;
1038                 }
1039
1040                 /* arp_key_offset */
1041                 if (strcmp(arg_name, "arp_key_offset") == 0) {
1042                         if (arp_key_offset_present)
1043                                 return -1;
1044                         arp_key_offset_present = 1;
1045
1046                         p->arp_key_offset = atoi(arg_value);
1047
1048                         continue;
1049                 }
1050
1051                 /* color_offset */
1052                 if (strcmp(arg_name, "color_offset") == 0) {
1053                         if (color_offset_present)
1054                                 return -1;
1055                         color_offset_present = 1;
1056
1057                         p->color_offset = atoi(arg_value);
1058
1059                         continue;
1060                 }
1061
1062                 /* debug */
1063                 if (strcmp(arg_name, "dbg_ah_disable") == 0) {
1064                         if (dbg_ah_disable_present)
1065                                 return -1;
1066                         dbg_ah_disable_present = 1;
1067
1068                         if (strcmp(arg_value, "no") == 0)
1069                                 p->dbg_ah_disable = 0;
1070                         else if (strcmp(arg_value, "yes") == 0)
1071                                 p->dbg_ah_disable = 1;
1072                         else
1073                                 return -1;
1074
1075                         continue;
1076                 }
1077
1078                 /* any other */
1079                 return -1;
1080         }
1081
1082         /* Check that mandatory arguments are present */
1083         if (ip_hdr_offset_present == 0)
1084                 return -1;
1085
1086         /* Check relations between arguments */
1087         switch (p->encap) {
1088         case PIPELINE_ROUTING_ENCAP_ETHERNET:
1089                 if (p->qinq_sched ||
1090                         p->mpls_color_mark ||
1091                         color_offset_present)
1092                         return -1;
1093                 break;
1094
1095         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
1096                 if (p->mpls_color_mark ||
1097                         color_offset_present)
1098                         return -1;
1099                 break;
1100
1101         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
1102                 if (p->qinq_sched)
1103                         return -1;
1104                 break;
1105
1106         default:
1107                 return -1;
1108         }
1109
1110         if ((p->n_arp_entries && (arp_key_offset_present == 0)) ||
1111                 ((p->n_arp_entries == 0) && arp_key_offset_present))
1112                 return -1;
1113
1114         return 0;
1115 }
1116
1117 static void *
1118 pipeline_routing_init(struct pipeline_params *params,
1119         __rte_unused void *arg)
1120 {
1121         struct pipeline *p;
1122         struct pipeline_routing *p_rt;
1123         uint32_t size, i;
1124
1125         /* Check input arguments */
1126         if ((params == NULL) ||
1127                 (params->n_ports_in == 0) ||
1128                 (params->n_ports_out == 0))
1129                 return NULL;
1130
1131         /* Memory allocation */
1132         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_routing));
1133         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
1134         p_rt = (struct pipeline_routing *) p;
1135         if (p == NULL)
1136                 return NULL;
1137
1138         strcpy(p->name, params->name);
1139         p->log_level = params->log_level;
1140
1141         PLOG(p, HIGH, "Routing");
1142
1143         /* Parse arguments */
1144         if (pipeline_routing_parse_args(&p_rt->params, params))
1145                 return NULL;
1146
1147         /* Pipeline */
1148         {
1149                 struct rte_pipeline_params pipeline_params = {
1150                         .name = params->name,
1151                         .socket_id = params->socket_id,
1152                         .offset_port_id = 0,
1153                 };
1154
1155                 p->p = rte_pipeline_create(&pipeline_params);
1156                 if (p->p == NULL) {
1157                         rte_free(p);
1158                         return NULL;
1159                 }
1160         }
1161
1162         /* Input ports */
1163         p->n_ports_in = params->n_ports_in;
1164         for (i = 0; i < p->n_ports_in; i++) {
1165                 struct rte_pipeline_port_in_params port_params = {
1166                         .ops = pipeline_port_in_params_get_ops(
1167                                 &params->port_in[i]),
1168                         .arg_create = pipeline_port_in_params_convert(
1169                                 &params->port_in[i]),
1170                         .f_action = NULL,
1171                         .arg_ah = NULL,
1172                         .burst_size = params->port_in[i].burst_size,
1173                 };
1174
1175                 int status = rte_pipeline_port_in_create(p->p,
1176                         &port_params,
1177                         &p->port_in_id[i]);
1178
1179                 if (status) {
1180                         rte_pipeline_free(p->p);
1181                         rte_free(p);
1182                         return NULL;
1183                 }
1184         }
1185
1186         /* Output ports */
1187         p->n_ports_out = params->n_ports_out;
1188         for (i = 0; i < p->n_ports_out; i++) {
1189                 struct rte_pipeline_port_out_params port_params = {
1190                         .ops = pipeline_port_out_params_get_ops(
1191                                 &params->port_out[i]),
1192                         .arg_create = pipeline_port_out_params_convert(
1193                                 &params->port_out[i]),
1194                         .f_action = NULL,
1195                         .f_action_bulk = NULL,
1196                         .arg_ah = NULL,
1197                 };
1198
1199                 int status = rte_pipeline_port_out_create(p->p,
1200                         &port_params,
1201                         &p->port_out_id[i]);
1202
1203                 if (status) {
1204                         rte_pipeline_free(p->p);
1205                         rte_free(p);
1206                         return NULL;
1207                 }
1208         }
1209
1210         /* Routing table */
1211         p->n_tables = 1;
1212         {
1213                 struct rte_table_lpm_params table_lpm_params = {
1214                         .name = p->name,
1215                         .n_rules = p_rt->params.n_routes,
1216                         .entry_unique_size = sizeof(struct routing_table_entry),
1217                         .offset = p_rt->params.ip_hdr_offset +
1218                                 __builtin_offsetof(struct ipv4_hdr, dst_addr),
1219                 };
1220
1221                 struct rte_pipeline_table_params table_params = {
1222                                 .ops = &rte_table_lpm_ops,
1223                                 .arg_create = &table_lpm_params,
1224                                 .f_action_hit = get_routing_table_ah_hit(p_rt),
1225                                 .f_action_miss = NULL,
1226                                 .arg_ah = p_rt,
1227                                 .action_data_size =
1228                                         sizeof(struct routing_table_entry) -
1229                                         sizeof(struct rte_pipeline_table_entry),
1230                         };
1231
1232                 int status;
1233
1234                 status = rte_pipeline_table_create(p->p,
1235                         &table_params,
1236                         &p->table_id[0]);
1237
1238                 if (status) {
1239                         rte_pipeline_free(p->p);
1240                         rte_free(p);
1241                         return NULL;
1242                 }
1243         }
1244
1245         /* ARP table configuration */
1246         if (p_rt->params.n_arp_entries) {
1247                 struct rte_table_hash_key8_ext_params table_arp_params = {
1248                         .n_entries = p_rt->params.n_arp_entries,
1249                         .n_entries_ext = p_rt->params.n_arp_entries,
1250                         .f_hash = hash_default_key8,
1251                         .seed = 0,
1252                         .signature_offset = 0, /* Unused */
1253                         .key_offset = p_rt->params.arp_key_offset,
1254                 };
1255
1256                 struct rte_pipeline_table_params table_params = {
1257                         .ops = &rte_table_hash_key8_ext_dosig_ops,
1258                         .arg_create = &table_arp_params,
1259                         .f_action_hit = get_arp_table_ah_hit(p_rt),
1260                         .f_action_miss = NULL,
1261                         .arg_ah = p_rt,
1262                         .action_data_size = sizeof(struct arp_table_entry) -
1263                                 sizeof(struct rte_pipeline_table_entry),
1264                 };
1265
1266                 int status;
1267
1268                 status = rte_pipeline_table_create(p->p,
1269                         &table_params,
1270                         &p->table_id[1]);
1271
1272                 if (status) {
1273                         rte_pipeline_free(p->p);
1274                         rte_free(p);
1275                         return NULL;
1276                 }
1277
1278                 p->n_tables++;
1279         }
1280
1281         /* Connecting input ports to tables */
1282         for (i = 0; i < p->n_ports_in; i++) {
1283                 int status = rte_pipeline_port_in_connect_to_table(p->p,
1284                         p->port_in_id[i],
1285                         p->table_id[0]);
1286
1287                 if (status) {
1288                         rte_pipeline_free(p->p);
1289                         rte_free(p);
1290                         return NULL;
1291                 }
1292         }
1293
1294         /* Enable input ports */
1295         for (i = 0; i < p->n_ports_in; i++) {
1296                 int status = rte_pipeline_port_in_enable(p->p,
1297                         p->port_in_id[i]);
1298
1299                 if (status) {
1300                         rte_pipeline_free(p->p);
1301                         rte_free(p);
1302                         return NULL;
1303                 }
1304         }
1305
1306         /* Check pipeline consistency */
1307         if (rte_pipeline_check(p->p) < 0) {
1308                 rte_pipeline_free(p->p);
1309                 rte_free(p);
1310                 return NULL;
1311         }
1312
1313         /* Message queues */
1314         p->n_msgq = params->n_msgq;
1315         for (i = 0; i < p->n_msgq; i++)
1316                 p->msgq_in[i] = params->msgq_in[i];
1317         for (i = 0; i < p->n_msgq; i++)
1318                 p->msgq_out[i] = params->msgq_out[i];
1319
1320         /* Message handlers */
1321         memcpy(p->handlers, handlers, sizeof(p->handlers));
1322         memcpy(p_rt->custom_handlers,
1323                 custom_handlers,
1324                 sizeof(p_rt->custom_handlers));
1325
1326         return p;
1327 }
1328
1329 static int
1330 pipeline_routing_free(void *pipeline)
1331 {
1332         struct pipeline *p = (struct pipeline *) pipeline;
1333
1334         /* Check input arguments */
1335         if (p == NULL)
1336                 return -1;
1337
1338         /* Free resources */
1339         rte_pipeline_free(p->p);
1340         rte_free(p);
1341         return 0;
1342 }
1343
1344 static int
1345 pipeline_routing_track(void *pipeline,
1346         __rte_unused uint32_t port_in,
1347         uint32_t *port_out)
1348 {
1349         struct pipeline *p = (struct pipeline *) pipeline;
1350
1351         /* Check input arguments */
1352         if ((p == NULL) ||
1353                 (port_in >= p->n_ports_in) ||
1354                 (port_out == NULL))
1355                 return -1;
1356
1357         if (p->n_ports_in == 1) {
1358                 *port_out = 0;
1359                 return 0;
1360         }
1361
1362         return -1;
1363 }
1364
1365 static int
1366 pipeline_routing_timer(void *pipeline)
1367 {
1368         struct pipeline *p = (struct pipeline *) pipeline;
1369
1370         pipeline_msg_req_handle(p);
1371         rte_pipeline_flush(p->p);
1372
1373         return 0;
1374 }
1375
1376 void *
1377 pipeline_routing_msg_req_custom_handler(struct pipeline *p,
1378         void *msg)
1379 {
1380         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1381         struct pipeline_custom_msg_req *req = msg;
1382         pipeline_msg_req_handler f_handle;
1383
1384         f_handle = (req->subtype < PIPELINE_ROUTING_MSG_REQS) ?
1385                 p_rt->custom_handlers[req->subtype] :
1386                 pipeline_msg_req_invalid_handler;
1387
1388         if (f_handle == NULL)
1389                 f_handle = pipeline_msg_req_invalid_handler;
1390
1391         return f_handle(p, req);
1392 }
1393
1394 void *
1395 pipeline_routing_msg_req_route_add_handler(struct pipeline *p, void *msg)
1396 {
1397         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1398         struct pipeline_routing_route_add_msg_req *req = msg;
1399         struct pipeline_routing_route_add_msg_rsp *rsp = msg;
1400
1401         struct rte_table_lpm_key key = {
1402                 .ip = req->key.key.ipv4.ip,
1403                 .depth = req->key.key.ipv4.depth,
1404         };
1405
1406         struct routing_table_entry entry_arp0 = {
1407                 .head = {
1408                         .action = RTE_PIPELINE_ACTION_PORT,
1409                         {.port_id = p->port_out_id[req->data.port_id]},
1410                 },
1411
1412                 .flags = req->data.flags,
1413                 .port_id = req->data.port_id,
1414                 .ip = 0,
1415                 .data_offset = 0,
1416                 .ether_l2_length = 0,
1417                 .slab = {0},
1418                 .slab_offset = {0},
1419         };
1420
1421         struct routing_table_entry entry_arp1 = {
1422                 .head = {
1423                         .action = RTE_PIPELINE_ACTION_TABLE,
1424                         {.table_id = p->table_id[1]},
1425                 },
1426
1427                 .flags = req->data.flags,
1428                 .port_id = req->data.port_id,
1429                 .ip = rte_bswap32(req->data.ethernet.ip),
1430                 .data_offset = 0,
1431                 .ether_l2_length = 0,
1432                 .slab = {0},
1433                 .slab_offset = {0},
1434         };
1435
1436         struct rte_pipeline_table_entry *entry = (p_rt->params.n_arp_entries) ?
1437                 (struct rte_pipeline_table_entry *) &entry_arp1 :
1438                 (struct rte_pipeline_table_entry *) &entry_arp0;
1439
1440         if ((req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) ||
1441                 ((p_rt->params.n_arp_entries == 0) &&
1442                         (req->data.flags & PIPELINE_ROUTING_ROUTE_ARP)) ||
1443                 (p_rt->params.n_arp_entries &&
1444                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_ARP) == 0)) ||
1445                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1446                         (req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ)) ||
1447                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1448                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ) == 0)) ||
1449                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1450                         (req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS)) ||
1451                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1452                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS) == 0))) {
1453                 rsp->status = -1;
1454                 return rsp;
1455         }
1456
1457         /* Ether - ARP off */
1458         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1459                 (p_rt->params.n_arp_entries == 0)) {
1460                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1461                 uint64_t macaddr_dst;
1462                 uint64_t ethertype = ETHER_TYPE_IPv4;
1463
1464                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1465                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1466
1467                 entry_arp0.slab[0] =
1468                         rte_bswap64((macaddr_src << 16) | ethertype);
1469                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1470
1471                 entry_arp0.slab[1] = rte_bswap64(macaddr_dst);
1472                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1473
1474                 entry_arp0.data_offset = entry_arp0.slab_offset[1] + 2
1475                         - sizeof(struct rte_mbuf);
1476                 entry_arp0.ether_l2_length = 14;
1477         }
1478
1479         /* Ether - ARP on */
1480         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1481                 p_rt->params.n_arp_entries) {
1482                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1483                 uint64_t ethertype = ETHER_TYPE_IPv4;
1484
1485                 entry_arp1.slab[0] = rte_bswap64((macaddr_src << 16) |
1486                         ethertype);
1487                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1488
1489                 entry_arp1.data_offset = entry_arp1.slab_offset[0] - 6
1490                         - sizeof(struct rte_mbuf);
1491                 entry_arp1.ether_l2_length = 14;
1492         }
1493
1494         /* Ether QinQ - ARP off */
1495         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1496                 (p_rt->params.n_arp_entries == 0)) {
1497                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1498                 uint64_t macaddr_dst;
1499                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1500                 uint64_t ethertype_vlan = 0x8100;
1501                 uint64_t ethertype_qinq = 0x9100;
1502                 uint64_t svlan = req->data.l2.qinq.svlan;
1503                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1504
1505                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1506                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1507
1508                 entry_arp0.slab[0] = rte_bswap64((svlan << 48) |
1509                         (ethertype_vlan << 32) |
1510                         (cvlan << 16) |
1511                         ethertype_ipv4);
1512                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1513
1514                 entry_arp0.slab[1] = rte_bswap64((macaddr_src << 16) |
1515                         ethertype_qinq);
1516                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1517
1518                 entry_arp0.slab[2] = rte_bswap64(macaddr_dst);
1519                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset - 3 * 8;
1520
1521                 entry_arp0.data_offset = entry_arp0.slab_offset[2] + 2
1522                         - sizeof(struct rte_mbuf);
1523                 entry_arp0.ether_l2_length = 22;
1524         }
1525
1526         /* Ether QinQ - ARP on */
1527         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1528                 p_rt->params.n_arp_entries) {
1529                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1530                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1531                 uint64_t ethertype_vlan = 0x8100;
1532                 uint64_t ethertype_qinq = 0x9100;
1533                 uint64_t svlan = req->data.l2.qinq.svlan;
1534                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1535
1536                 entry_arp1.slab[0] = rte_bswap64((svlan << 48) |
1537                         (ethertype_vlan << 32) |
1538                         (cvlan << 16) |
1539                         ethertype_ipv4);
1540                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1541
1542                 entry_arp1.slab[1] = rte_bswap64((macaddr_src << 16) |
1543                         ethertype_qinq);
1544                 entry_arp1.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1545
1546                 entry_arp1.data_offset = entry_arp1.slab_offset[1] - 6
1547                         - sizeof(struct rte_mbuf);
1548                 entry_arp1.ether_l2_length = 22;
1549         }
1550
1551         /* Ether MPLS - ARP off */
1552         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1553                 (p_rt->params.n_arp_entries == 0)) {
1554                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1555                 uint64_t macaddr_dst;
1556                 uint64_t ethertype_mpls = 0x8847;
1557
1558                 uint64_t label0 = req->data.l2.mpls.labels[0];
1559                 uint64_t label1 = req->data.l2.mpls.labels[1];
1560                 uint64_t label2 = req->data.l2.mpls.labels[2];
1561                 uint64_t label3 = req->data.l2.mpls.labels[3];
1562                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1563
1564                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1565                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1566
1567                 switch (n_labels) {
1568                 case 1:
1569                         entry_arp0.slab[0] = 0;
1570                         entry_arp0.slab_offset[0] =
1571                                 p_rt->params.ip_hdr_offset - 8;
1572
1573                         entry_arp0.slab[1] = rte_bswap64(
1574                                 MPLS_LABEL(label0, 0, 1, 0));
1575                         entry_arp0.slab_offset[1] =
1576                                 p_rt->params.ip_hdr_offset - 8;
1577                         break;
1578
1579                 case 2:
1580                         entry_arp0.slab[0] = 0;
1581                         entry_arp0.slab_offset[0] =
1582                                 p_rt->params.ip_hdr_offset - 8;
1583
1584                         entry_arp0.slab[1] = rte_bswap64(
1585                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1586                                 MPLS_LABEL(label1, 0, 1, 0));
1587                         entry_arp0.slab_offset[1] =
1588                                 p_rt->params.ip_hdr_offset - 8;
1589                         break;
1590
1591                 case 3:
1592                         entry_arp0.slab[0] = rte_bswap64(
1593                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1594                                 MPLS_LABEL(label2, 0, 1, 0));
1595                         entry_arp0.slab_offset[0] =
1596                                 p_rt->params.ip_hdr_offset - 8;
1597
1598                         entry_arp0.slab[1] = rte_bswap64(
1599                                 MPLS_LABEL(label0, 0, 0, 0));
1600                         entry_arp0.slab_offset[1] =
1601                                 p_rt->params.ip_hdr_offset - 2 * 8;
1602                         break;
1603
1604                 case 4:
1605                         entry_arp0.slab[0] = rte_bswap64(
1606                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1607                                 MPLS_LABEL(label3, 0, 1, 0));
1608                         entry_arp0.slab_offset[0] =
1609                                 p_rt->params.ip_hdr_offset - 8;
1610
1611                         entry_arp0.slab[1] = rte_bswap64(
1612                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1613                                 MPLS_LABEL(label1, 0, 0, 0));
1614                         entry_arp0.slab_offset[1] =
1615                                 p_rt->params.ip_hdr_offset - 2 * 8;
1616                         break;
1617
1618                 default:
1619                         rsp->status = -1;
1620                         return rsp;
1621                 }
1622
1623                 entry_arp0.slab[2] = rte_bswap64((macaddr_src << 16) |
1624                         ethertype_mpls);
1625                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset -
1626                         (n_labels * 4 + 8);
1627
1628                 entry_arp0.slab[3] = rte_bswap64(macaddr_dst);
1629                 entry_arp0.slab_offset[3] = p_rt->params.ip_hdr_offset -
1630                         (n_labels * 4 + 2 * 8);
1631
1632                 entry_arp0.data_offset = entry_arp0.slab_offset[3] + 2
1633                         - sizeof(struct rte_mbuf);
1634                 entry_arp0.ether_l2_length = n_labels * 4 + 14;
1635         }
1636
1637         /* Ether MPLS - ARP on */
1638         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1639                 p_rt->params.n_arp_entries) {
1640                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1641                 uint64_t ethertype_mpls = 0x8847;
1642
1643                 uint64_t label0 = req->data.l2.mpls.labels[0];
1644                 uint64_t label1 = req->data.l2.mpls.labels[1];
1645                 uint64_t label2 = req->data.l2.mpls.labels[2];
1646                 uint64_t label3 = req->data.l2.mpls.labels[3];
1647                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1648
1649                 switch (n_labels) {
1650                 case 1:
1651                         entry_arp1.slab[0] = 0;
1652                         entry_arp1.slab_offset[0] =
1653                                 p_rt->params.ip_hdr_offset - 8;
1654
1655                         entry_arp1.slab[1] = rte_bswap64(
1656                                 MPLS_LABEL(label0, 0, 1, 0));
1657                         entry_arp1.slab_offset[1] =
1658                                 p_rt->params.ip_hdr_offset - 8;
1659                         break;
1660
1661                 case 2:
1662                         entry_arp1.slab[0] = 0;
1663                         entry_arp1.slab_offset[0] =
1664                                 p_rt->params.ip_hdr_offset - 8;
1665
1666                         entry_arp1.slab[1] = rte_bswap64(
1667                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1668                                 MPLS_LABEL(label1, 0, 1, 0));
1669                         entry_arp1.slab_offset[1] =
1670                                 p_rt->params.ip_hdr_offset - 8;
1671                         break;
1672
1673                 case 3:
1674                         entry_arp1.slab[0] = rte_bswap64(
1675                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1676                                 MPLS_LABEL(label2, 0, 1, 0));
1677                         entry_arp1.slab_offset[0] =
1678                                 p_rt->params.ip_hdr_offset - 8;
1679
1680                         entry_arp1.slab[1] = rte_bswap64(
1681                                 MPLS_LABEL(label0, 0, 0, 0));
1682                         entry_arp1.slab_offset[1] =
1683                                 p_rt->params.ip_hdr_offset - 2 * 8;
1684                         break;
1685
1686                 case 4:
1687                         entry_arp1.slab[0] = rte_bswap64(
1688                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1689                                 MPLS_LABEL(label3, 0, 1, 0));
1690                         entry_arp1.slab_offset[0] =
1691                                 p_rt->params.ip_hdr_offset - 8;
1692
1693                         entry_arp1.slab[1] = rte_bswap64(
1694                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1695                                 MPLS_LABEL(label1, 0, 0, 0));
1696                         entry_arp1.slab_offset[1] =
1697                                 p_rt->params.ip_hdr_offset - 2 * 8;
1698                         break;
1699
1700                 default:
1701                         rsp->status = -1;
1702                         return rsp;
1703                 }
1704
1705                 entry_arp1.slab[2] = rte_bswap64((macaddr_src << 16) |
1706                         ethertype_mpls);
1707                 entry_arp1.slab_offset[2] = p_rt->params.ip_hdr_offset -
1708                         (n_labels * 4 + 8);
1709
1710                 entry_arp1.data_offset = entry_arp1.slab_offset[2] - 6
1711                         - sizeof(struct rte_mbuf);
1712                 entry_arp1.ether_l2_length = n_labels * 4 + 14;
1713         }
1714
1715         rsp->status = rte_pipeline_table_entry_add(p->p,
1716                 p->table_id[0],
1717                 &key,
1718                 entry,
1719                 &rsp->key_found,
1720                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1721
1722         return rsp;
1723 }
1724
1725 void *
1726 pipeline_routing_msg_req_route_del_handler(struct pipeline *p, void *msg)
1727 {
1728         struct pipeline_routing_route_delete_msg_req *req = msg;
1729         struct pipeline_routing_route_delete_msg_rsp *rsp = msg;
1730
1731         struct rte_table_lpm_key key = {
1732                 .ip = req->key.key.ipv4.ip,
1733                 .depth = req->key.key.ipv4.depth,
1734         };
1735
1736         if (req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) {
1737                 rsp->status = -1;
1738                 return rsp;
1739         }
1740
1741         rsp->status = rte_pipeline_table_entry_delete(p->p,
1742                 p->table_id[0],
1743                 &key,
1744                 &rsp->key_found,
1745                 NULL);
1746
1747         return rsp;
1748 }
1749
1750 void *
1751 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
1752         void *msg)
1753 {
1754         struct pipeline_routing_route_add_default_msg_req *req = msg;
1755         struct pipeline_routing_route_add_default_msg_rsp *rsp = msg;
1756
1757         struct routing_table_entry default_entry = {
1758                 .head = {
1759                         .action = RTE_PIPELINE_ACTION_PORT,
1760                         {.port_id = p->port_out_id[req->port_id]},
1761                 },
1762
1763                 .flags = 0,
1764                 .port_id = 0,
1765                 .ip = 0,
1766         };
1767
1768         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1769                 p->table_id[0],
1770                 (struct rte_pipeline_table_entry *) &default_entry,
1771                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1772
1773         return rsp;
1774 }
1775
1776 void *
1777 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
1778         void *msg)
1779 {
1780         struct pipeline_routing_route_delete_default_msg_rsp *rsp = msg;
1781
1782         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1783                 p->table_id[0],
1784                 NULL);
1785
1786         return rsp;
1787 }
1788
1789 void *
1790 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p, void *msg)
1791 {
1792         struct pipeline_routing_arp_add_msg_req *req = msg;
1793         struct pipeline_routing_arp_add_msg_rsp *rsp = msg;
1794
1795         struct pipeline_routing_arp_key_ipv4 key = {
1796                 .port_id = req->key.key.ipv4.port_id,
1797                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1798         };
1799
1800         struct arp_table_entry entry = {
1801                 .head = {
1802                         .action = RTE_PIPELINE_ACTION_PORT,
1803                         {.port_id = p->port_out_id[req->key.key.ipv4.port_id]},
1804                 },
1805
1806                 .macaddr = 0, /* set below */
1807         };
1808
1809         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1810                 rsp->status = -1;
1811                 return rsp;
1812         }
1813
1814         entry.macaddr = *((uint64_t *)&(req->macaddr));
1815         entry.macaddr = entry.macaddr << 16;
1816
1817         rsp->status = rte_pipeline_table_entry_add(p->p,
1818                 p->table_id[1],
1819                 &key,
1820                 (struct rte_pipeline_table_entry *) &entry,
1821                 &rsp->key_found,
1822                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1823
1824         return rsp;
1825 }
1826
1827 void *
1828 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p, void *msg)
1829 {
1830         struct pipeline_routing_arp_delete_msg_req *req = msg;
1831         struct pipeline_routing_arp_delete_msg_rsp *rsp = msg;
1832
1833         struct pipeline_routing_arp_key_ipv4 key = {
1834                 .port_id = req->key.key.ipv4.port_id,
1835                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1836         };
1837
1838         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1839                 rsp->status = -1;
1840                 return rsp;
1841         }
1842
1843         rsp->status = rte_pipeline_table_entry_delete(p->p,
1844                 p->table_id[1],
1845                 &key,
1846                 &rsp->key_found,
1847                 NULL);
1848
1849         return rsp;
1850 }
1851
1852 void *
1853 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p, void *msg)
1854 {
1855         struct pipeline_routing_arp_add_default_msg_req *req = msg;
1856         struct pipeline_routing_arp_add_default_msg_rsp *rsp = msg;
1857
1858         struct arp_table_entry default_entry = {
1859                 .head = {
1860                         .action = RTE_PIPELINE_ACTION_PORT,
1861                         {.port_id = p->port_out_id[req->port_id]},
1862                 },
1863
1864                 .macaddr = 0,
1865         };
1866
1867         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1868                 p->table_id[1],
1869                 (struct rte_pipeline_table_entry *) &default_entry,
1870                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1871
1872         return rsp;
1873 }
1874
1875 void *
1876 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p, void *msg)
1877 {
1878         struct pipeline_routing_arp_delete_default_msg_rsp *rsp = msg;
1879
1880         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1881                 p->table_id[1],
1882                 NULL);
1883
1884         return rsp;
1885 }
1886
1887 struct pipeline_be_ops pipeline_routing_be_ops = {
1888         .f_init = pipeline_routing_init,
1889         .f_free = pipeline_routing_free,
1890         .f_run = NULL,
1891         .f_timer = pipeline_routing_timer,
1892         .f_track = pipeline_routing_track,
1893 };