examples/ip_pipeline/pipeline/pipeline_routing_be.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <stdint.h>
  37 #include <string.h>
  38 #include <unistd.h>
  39
  40 #include <rte_common.h>
  41 #include <rte_malloc.h>
  42 #include <rte_ip.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_table_lpm.h>
  45 #include <rte_table_hash.h>
  46 #include <rte_pipeline.h>
  47
  48 #include "pipeline_routing_be.h"
  49 #include "pipeline_actions_common.h"
  50 #include "parser.h"
  51 #include "hash_func.h"
  52
  53 #define MPLS_LABEL(label, exp, s, ttl)                                  \
  54         (((((uint64_t) (label)) & 0xFFFFFLLU) << 12) |          \
  55         ((((uint64_t) (exp)) & 0x7LLU) << 9) |                          \
  56         ((((uint64_t) (s)) & 0x1LLU) << 8) |                            \
  57         (((uint64_t) (ttl)) & 0xFFLU))
  58
  59 #define RTE_SCHED_PORT_HIERARCHY(subport, pipe,         \
  60         traffic_class, queue, color)                            \
  61         ((((uint64_t) (queue)) & 0x3) |                \
  62         ((((uint64_t) (traffic_class)) & 0x3) << 2) |  \
  63         ((((uint64_t) (color)) & 0x3) << 4) |          \
  64         ((((uint64_t) (subport)) & 0xFFFF) << 16) |    \
  65         ((((uint64_t) (pipe)) & 0xFFFFFFFF) << 32))
  66
  67
  68 #define MAC_SRC_DEFAULT 0x112233445566
  69
  70 struct pipeline_routing {
  71         struct pipeline p;
  72         struct pipeline_routing_params params;
  73         pipeline_msg_req_handler custom_handlers[PIPELINE_ROUTING_MSG_REQS];
  74 } __rte_cache_aligned;
  75
  76 /*
  77  * Message handlers
  78  */
  79 static void *
  80 pipeline_routing_msg_req_custom_handler(struct pipeline *p, void *msg);
  81
  82 static pipeline_msg_req_handler handlers[] = {
  83         [PIPELINE_MSG_REQ_PING] =
  84                 pipeline_msg_req_ping_handler,
  85         [PIPELINE_MSG_REQ_STATS_PORT_IN] =
  86                 pipeline_msg_req_stats_port_in_handler,
  87         [PIPELINE_MSG_REQ_STATS_PORT_OUT] =
  88                 pipeline_msg_req_stats_port_out_handler,
  89         [PIPELINE_MSG_REQ_STATS_TABLE] =
  90                 pipeline_msg_req_stats_table_handler,
  91         [PIPELINE_MSG_REQ_PORT_IN_ENABLE] =
  92                 pipeline_msg_req_port_in_enable_handler,
  93         [PIPELINE_MSG_REQ_PORT_IN_DISABLE] =
  94                 pipeline_msg_req_port_in_disable_handler,
  95         [PIPELINE_MSG_REQ_CUSTOM] =
  96                 pipeline_routing_msg_req_custom_handler,
  97 };
  98
  99 static void *
 100 pipeline_routing_msg_req_route_add_handler(struct pipeline *p,
 101         void *msg);
 102
 103 static void *
 104 pipeline_routing_msg_req_route_del_handler(struct pipeline *p,
 105         void *msg);
 106
 107 static void *
 108 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
 109         void *msg);
 110
 111 static void *
 112 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
 113         void *msg);
 114
 115 static void *
 116 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p,
 117         void *msg);
 118
 119 static void *
 120 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p,
 121         void *msg);
 122
 123 static void *
 124 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p,
 125         void *msg);
 126
 127 static void *
 128 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p,
 129         void *msg);
 130
 131 static pipeline_msg_req_handler custom_handlers[] = {
 132         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD] =
 133                 pipeline_routing_msg_req_route_add_handler,
 134         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL] =
 135                 pipeline_routing_msg_req_route_del_handler,
 136         [PIPELINE_ROUTING_MSG_REQ_ROUTE_ADD_DEFAULT] =
 137                 pipeline_routing_msg_req_route_add_default_handler,
 138         [PIPELINE_ROUTING_MSG_REQ_ROUTE_DEL_DEFAULT] =
 139                 pipeline_routing_msg_req_route_del_default_handler,
 140         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD] =
 141                 pipeline_routing_msg_req_arp_add_handler,
 142         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL] =
 143                 pipeline_routing_msg_req_arp_del_handler,
 144         [PIPELINE_ROUTING_MSG_REQ_ARP_ADD_DEFAULT] =
 145                 pipeline_routing_msg_req_arp_add_default_handler,
 146         [PIPELINE_ROUTING_MSG_REQ_ARP_DEL_DEFAULT] =
 147                 pipeline_routing_msg_req_arp_del_default_handler,
 148 };
 149
 150 /*
 151  * Routing table
 152  */
 153 struct routing_table_entry {
 154         struct rte_pipeline_table_entry head;
 155         uint32_t flags;
 156         uint32_t port_id; /* Output port ID */
 157         uint32_t ip; /* Next hop IP address (only valid for remote routes) */
 158
 159         /* ether_l2 */
 160         uint16_t data_offset;
 161         uint16_t ether_l2_length;
 162         uint64_t slab[4];
 163         uint16_t slab_offset[4];
 164 };
 165
 166 struct layout {
 167         uint16_t a;
 168         uint32_t b;
 169         uint16_t c;
 170 } __attribute__((__packed__));
 171
 172 #define MACADDR_DST_WRITE(slab_ptr, slab)                       \
 173 {                                                               \
 174         struct layout *dst = (struct layout *) (slab_ptr);      \
 175         struct layout *src = (struct layout *) &(slab);         \
 176                                                                 \
 177         dst->b = src->b;                                        \
 178         dst->c = src->c;                                        \
 179 }
 180
 181 static inline __attribute__((always_inline)) void
 182 pkt_work_routing(
 183         struct rte_mbuf *pkt,
 184         struct rte_pipeline_table_entry *table_entry,
 185         void *arg,
 186         int arp,
 187         int qinq,
 188         int qinq_sched,
 189         int mpls,
 190         int mpls_color_mark)
 191 {
 192         struct pipeline_routing *p_rt = arg;
 193
 194         struct routing_table_entry *entry =
 195                 (struct routing_table_entry *) table_entry;
 196
 197         struct ipv4_hdr *ip = (struct ipv4_hdr *)
 198                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.ip_hdr_offset);
 199
 200         enum rte_meter_color pkt_color = (enum rte_meter_color)
 201                 RTE_MBUF_METADATA_UINT32(pkt, p_rt->params.color_offset);
 202
 203         struct pipeline_routing_arp_key_ipv4 *arp_key =
 204                 (struct pipeline_routing_arp_key_ipv4 *)
 205                 RTE_MBUF_METADATA_UINT8_PTR(pkt, p_rt->params.arp_key_offset);
 206
 207         uint64_t *slab0_ptr, *slab1_ptr, *slab2_ptr, *slab3_ptr, sched;
 208         uint32_t ip_da, nh_ip, port_id;
 209         uint16_t total_length, data_offset, ether_l2_length;
 210
 211         /* Read */
 212         total_length = rte_bswap16(ip->total_length);
 213         ip_da = ip->dst_addr;
 214         data_offset = entry->data_offset;
 215         ether_l2_length = entry->ether_l2_length;
 216         slab0_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[0]);
 217         slab1_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[1]);
 218         slab2_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[2]);
 219         slab3_ptr = RTE_MBUF_METADATA_UINT64_PTR(pkt, entry->slab_offset[3]);
 220
 221         if (arp) {
 222                 port_id = entry->port_id;
 223                 nh_ip = entry->ip;
 224                 if (entry->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 225                         nh_ip = ip_da;
 226         }
 227
 228         /* Compute */
 229         total_length += ether_l2_length;
 230
 231         if (qinq && qinq_sched) {
 232                 uint32_t dscp = ip->type_of_service >> 2;
 233                 uint32_t svlan, cvlan, tc, tc_q;
 234
 235                 if (qinq_sched == 1) {
 236                         uint64_t slab_qinq = rte_bswap64(entry->slab[0]);
 237
 238                         svlan = (slab_qinq >> 48) & 0xFFF;
 239                         cvlan = (slab_qinq >> 16) & 0xFFF;
 240                         tc = (dscp >> 2) & 0x3;
 241                         tc_q = dscp & 0x3;
 242                 } else {
 243                         uint32_t ip_src = rte_bswap32(ip->src_addr);
 244
 245                         svlan = 0;
 246                         cvlan = (ip_src >> 16) & 0xFFF;
 247                         tc = (ip_src >> 2) & 0x3;
 248                         tc_q = ip_src & 0x3;
 249                 }
 250                 sched = RTE_SCHED_PORT_HIERARCHY(svlan,
 251                         cvlan,
 252                         tc,
 253                         tc_q,
 254                         e_RTE_METER_GREEN);
 255         }
 256
 257         /* Write */
 258         pkt->data_off = data_offset;
 259         pkt->data_len = total_length;
 260         pkt->pkt_len = total_length;
 261
 262         if ((qinq == 0) && (mpls == 0)) {
 263                 *slab0_ptr = entry->slab[0];
 264
 265                 if (arp == 0)
 266                         MACADDR_DST_WRITE(slab1_ptr, entry->slab[1]);
 267         }
 268
 269         if (qinq) {
 270                 *slab0_ptr = entry->slab[0];
 271                 *slab1_ptr = entry->slab[1];
 272
 273                 if (arp == 0)
 274                         MACADDR_DST_WRITE(slab2_ptr, entry->slab[2]);
 275
 276                 if (qinq_sched) {
 277                         pkt->hash.sched.lo = sched & 0xFFFFFFFF;
 278                         pkt->hash.sched.hi = sched >> 32;
 279                 }
 280         }
 281
 282         if (mpls) {
 283                 if (mpls_color_mark) {
 284                         uint64_t mpls_exp = rte_bswap64(
 285                                 (MPLS_LABEL(0, pkt_color, 0, 0) << 32) |
 286                                 MPLS_LABEL(0, pkt_color, 0, 0));
 287
 288                         *slab0_ptr = entry->slab[0] | mpls_exp;
 289                         *slab1_ptr = entry->slab[1] | mpls_exp;
 290                         *slab2_ptr = entry->slab[2];
 291                 } else {
 292                         *slab0_ptr = entry->slab[0];
 293                         *slab1_ptr = entry->slab[1];
 294                         *slab2_ptr = entry->slab[2];
 295                 }
 296
 297                 if (arp == 0)
 298                         MACADDR_DST_WRITE(slab3_ptr, entry->slab[3]);
 299         }
 300
 301         if (arp) {
 302                 arp_key->port_id = port_id;
 303                 arp_key->ip = nh_ip;
 304         }
 305 }
 306
 307 static inline __attribute__((always_inline)) void
 308 pkt4_work_routing(
 309         struct rte_mbuf **pkts,
 310         struct rte_pipeline_table_entry **table_entries,
 311         void *arg,
 312         int arp,
 313         int qinq,
 314         int qinq_sched,
 315         int mpls,
 316         int mpls_color_mark)
 317 {
 318         struct pipeline_routing *p_rt = arg;
 319
 320         struct routing_table_entry *entry0 =
 321                 (struct routing_table_entry *) table_entries[0];
 322         struct routing_table_entry *entry1 =
 323                 (struct routing_table_entry *) table_entries[1];
 324         struct routing_table_entry *entry2 =
 325                 (struct routing_table_entry *) table_entries[2];
 326         struct routing_table_entry *entry3 =
 327                 (struct routing_table_entry *) table_entries[3];
 328
 329         struct ipv4_hdr *ip0 = (struct ipv4_hdr *)
 330                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 331                         p_rt->params.ip_hdr_offset);
 332         struct ipv4_hdr *ip1 = (struct ipv4_hdr *)
 333                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 334                         p_rt->params.ip_hdr_offset);
 335         struct ipv4_hdr *ip2 = (struct ipv4_hdr *)
 336                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 337                         p_rt->params.ip_hdr_offset);
 338         struct ipv4_hdr *ip3 = (struct ipv4_hdr *)
 339                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 340                         p_rt->params.ip_hdr_offset);
 341
 342         enum rte_meter_color pkt0_color = (enum rte_meter_color)
 343                 RTE_MBUF_METADATA_UINT32(pkts[0], p_rt->params.color_offset);
 344         enum rte_meter_color pkt1_color = (enum rte_meter_color)
 345                 RTE_MBUF_METADATA_UINT32(pkts[1], p_rt->params.color_offset);
 346         enum rte_meter_color pkt2_color = (enum rte_meter_color)
 347                 RTE_MBUF_METADATA_UINT32(pkts[2], p_rt->params.color_offset);
 348         enum rte_meter_color pkt3_color = (enum rte_meter_color)
 349                 RTE_MBUF_METADATA_UINT32(pkts[3], p_rt->params.color_offset);
 350
 351         struct pipeline_routing_arp_key_ipv4 *arp_key0 =
 352                 (struct pipeline_routing_arp_key_ipv4 *)
 353                 RTE_MBUF_METADATA_UINT8_PTR(pkts[0],
 354                         p_rt->params.arp_key_offset);
 355         struct pipeline_routing_arp_key_ipv4 *arp_key1 =
 356                 (struct pipeline_routing_arp_key_ipv4 *)
 357                 RTE_MBUF_METADATA_UINT8_PTR(pkts[1],
 358                         p_rt->params.arp_key_offset);
 359         struct pipeline_routing_arp_key_ipv4 *arp_key2 =
 360                 (struct pipeline_routing_arp_key_ipv4 *)
 361                 RTE_MBUF_METADATA_UINT8_PTR(pkts[2],
 362                         p_rt->params.arp_key_offset);
 363         struct pipeline_routing_arp_key_ipv4 *arp_key3 =
 364                 (struct pipeline_routing_arp_key_ipv4 *)
 365                 RTE_MBUF_METADATA_UINT8_PTR(pkts[3],
 366                         p_rt->params.arp_key_offset);
 367
 368         uint64_t *slab0_ptr0, *slab1_ptr0, *slab2_ptr0, *slab3_ptr0;
 369         uint64_t *slab0_ptr1, *slab1_ptr1, *slab2_ptr1, *slab3_ptr1;
 370         uint64_t *slab0_ptr2, *slab1_ptr2, *slab2_ptr2, *slab3_ptr2;
 371         uint64_t *slab0_ptr3, *slab1_ptr3, *slab2_ptr3, *slab3_ptr3;
 372         uint64_t sched0, sched1, sched2, sched3;
 373
 374         uint32_t ip_da0, nh_ip0, port_id0;
 375         uint32_t ip_da1, nh_ip1, port_id1;
 376         uint32_t ip_da2, nh_ip2, port_id2;
 377         uint32_t ip_da3, nh_ip3, port_id3;
 378
 379         uint16_t total_length0, data_offset0, ether_l2_length0;
 380         uint16_t total_length1, data_offset1, ether_l2_length1;
 381         uint16_t total_length2, data_offset2, ether_l2_length2;
 382         uint16_t total_length3, data_offset3, ether_l2_length3;
 383
 384         /* Read */
 385         total_length0 = rte_bswap16(ip0->total_length);
 386         total_length1 = rte_bswap16(ip1->total_length);
 387         total_length2 = rte_bswap16(ip2->total_length);
 388         total_length3 = rte_bswap16(ip3->total_length);
 389
 390         ip_da0 = ip0->dst_addr;
 391         ip_da1 = ip1->dst_addr;
 392         ip_da2 = ip2->dst_addr;
 393         ip_da3 = ip3->dst_addr;
 394
 395         data_offset0 = entry0->data_offset;
 396         data_offset1 = entry1->data_offset;
 397         data_offset2 = entry2->data_offset;
 398         data_offset3 = entry3->data_offset;
 399
 400         ether_l2_length0 = entry0->ether_l2_length;
 401         ether_l2_length1 = entry1->ether_l2_length;
 402         ether_l2_length2 = entry2->ether_l2_length;
 403         ether_l2_length3 = entry3->ether_l2_length;
 404
 405         slab0_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 406                 entry0->slab_offset[0]);
 407         slab1_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 408                 entry0->slab_offset[1]);
 409         slab2_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 410                 entry0->slab_offset[2]);
 411         slab3_ptr0 = RTE_MBUF_METADATA_UINT64_PTR(pkts[0],
 412                 entry0->slab_offset[3]);
 413
 414         slab0_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 415                 entry1->slab_offset[0]);
 416         slab1_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 417                 entry1->slab_offset[1]);
 418         slab2_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 419                 entry1->slab_offset[2]);
 420         slab3_ptr1 = RTE_MBUF_METADATA_UINT64_PTR(pkts[1],
 421                 entry1->slab_offset[3]);
 422
 423         slab0_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 424                 entry2->slab_offset[0]);
 425         slab1_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 426                 entry2->slab_offset[1]);
 427         slab2_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 428                 entry2->slab_offset[2]);
 429         slab3_ptr2 = RTE_MBUF_METADATA_UINT64_PTR(pkts[2],
 430                 entry2->slab_offset[3]);
 431
 432         slab0_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 433                 entry3->slab_offset[0]);
 434         slab1_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 435                 entry3->slab_offset[1]);
 436         slab2_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 437                 entry3->slab_offset[2]);
 438         slab3_ptr3 = RTE_MBUF_METADATA_UINT64_PTR(pkts[3],
 439                 entry3->slab_offset[3]);
 440
 441         if (arp) {
 442                 port_id0 = entry0->port_id;
 443                 nh_ip0 = entry0->ip;
 444                 if (entry0->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 445                         nh_ip0 = ip_da0;
 446
 447                 port_id1 = entry1->port_id;
 448                 nh_ip1 = entry1->ip;
 449                 if (entry1->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 450                         nh_ip1 = ip_da1;
 451
 452                 port_id2 = entry2->port_id;
 453                 nh_ip2 = entry2->ip;
 454                 if (entry2->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 455                         nh_ip2 = ip_da2;
 456
 457                 port_id3 = entry3->port_id;
 458                 nh_ip3 = entry3->ip;
 459                 if (entry3->flags & PIPELINE_ROUTING_ROUTE_LOCAL)
 460                         nh_ip3 = ip_da3;
 461         }
 462
 463         /* Compute */
 464         total_length0 += ether_l2_length0;
 465         total_length1 += ether_l2_length1;
 466         total_length2 += ether_l2_length2;
 467         total_length3 += ether_l2_length3;
 468
 469         if (qinq && qinq_sched) {
 470                 uint32_t dscp0 = ip0->type_of_service >> 2;
 471                 uint32_t dscp1 = ip1->type_of_service >> 2;
 472                 uint32_t dscp2 = ip2->type_of_service >> 2;
 473                 uint32_t dscp3 = ip3->type_of_service >> 2;
 474                 uint32_t svlan0, cvlan0, tc0, tc_q0;
 475                 uint32_t svlan1, cvlan1, tc1, tc_q1;
 476                 uint32_t svlan2, cvlan2, tc2, tc_q2;
 477                 uint32_t svlan3, cvlan3, tc3, tc_q3;
 478
 479                 if (qinq_sched == 1) {
 480                         uint64_t slab_qinq0 = rte_bswap64(entry0->slab[0]);
 481                         uint64_t slab_qinq1 = rte_bswap64(entry1->slab[0]);
 482                         uint64_t slab_qinq2 = rte_bswap64(entry2->slab[0]);
 483                         uint64_t slab_qinq3 = rte_bswap64(entry3->slab[0]);
 484
 485                         svlan0 = (slab_qinq0 >> 48) & 0xFFF;
 486                         svlan1 = (slab_qinq1 >> 48) & 0xFFF;
 487                         svlan2 = (slab_qinq2 >> 48) & 0xFFF;
 488                         svlan3 = (slab_qinq3 >> 48) & 0xFFF;
 489
 490                         cvlan0 = (slab_qinq0 >> 16) & 0xFFF;
 491                         cvlan1 = (slab_qinq1 >> 16) & 0xFFF;
 492                         cvlan2 = (slab_qinq2 >> 16) & 0xFFF;
 493                         cvlan3 = (slab_qinq3 >> 16) & 0xFFF;
 494
 495                         tc0 = (dscp0 >> 2) & 0x3;
 496                         tc1 = (dscp1 >> 2) & 0x3;
 497                         tc2 = (dscp2 >> 2) & 0x3;
 498                         tc3 = (dscp3 >> 2) & 0x3;
 499
 500                         tc_q0 = dscp0 & 0x3;
 501                         tc_q1 = dscp1 & 0x3;
 502                         tc_q2 = dscp2 & 0x3;
 503                         tc_q3 = dscp3 & 0x3;
 504                 } else {
 505                         uint32_t ip_src0 = rte_bswap32(ip0->src_addr);
 506                         uint32_t ip_src1 = rte_bswap32(ip1->src_addr);
 507                         uint32_t ip_src2 = rte_bswap32(ip2->src_addr);
 508                         uint32_t ip_src3 = rte_bswap32(ip3->src_addr);
 509
 510                         svlan0 = 0;
 511                         svlan1 = 0;
 512                         svlan2 = 0;
 513                         svlan3 = 0;
 514
 515                         cvlan0 = (ip_src0 >> 16) & 0xFFF;
 516                         cvlan1 = (ip_src1 >> 16) & 0xFFF;
 517                         cvlan2 = (ip_src2 >> 16) & 0xFFF;
 518                         cvlan3 = (ip_src3 >> 16) & 0xFFF;
 519
 520                         tc0 = (ip_src0 >> 2) & 0x3;
 521                         tc1 = (ip_src1 >> 2) & 0x3;
 522                         tc2 = (ip_src2 >> 2) & 0x3;
 523                         tc3 = (ip_src3 >> 2) & 0x3;
 524
 525                         tc_q0 = ip_src0 & 0x3;
 526                         tc_q1 = ip_src1 & 0x3;
 527                         tc_q2 = ip_src2 & 0x3;
 528                         tc_q3 = ip_src3 & 0x3;
 529                 }
 530
 531                 sched0 = RTE_SCHED_PORT_HIERARCHY(svlan0,
 532                         cvlan0,
 533                         tc0,
 534                         tc_q0,
 535                         e_RTE_METER_GREEN);
 536                 sched1 = RTE_SCHED_PORT_HIERARCHY(svlan1,
 537                         cvlan1,
 538                         tc1,
 539                         tc_q1,
 540                         e_RTE_METER_GREEN);
 541                 sched2 = RTE_SCHED_PORT_HIERARCHY(svlan2,
 542                         cvlan2,
 543                         tc2,
 544                         tc_q2,
 545                         e_RTE_METER_GREEN);
 546                 sched3 = RTE_SCHED_PORT_HIERARCHY(svlan3,
 547                         cvlan3,
 548                         tc3,
 549                         tc_q3,
 550                         e_RTE_METER_GREEN);
 551
 552         }
 553
 554         /* Write */
 555         pkts[0]->data_off = data_offset0;
 556         pkts[1]->data_off = data_offset1;
 557         pkts[2]->data_off = data_offset2;
 558         pkts[3]->data_off = data_offset3;
 559
 560         pkts[0]->data_len = total_length0;
 561         pkts[1]->data_len = total_length1;
 562         pkts[2]->data_len = total_length2;
 563         pkts[3]->data_len = total_length3;
 564
 565         pkts[0]->pkt_len = total_length0;
 566         pkts[1]->pkt_len = total_length1;
 567         pkts[2]->pkt_len = total_length2;
 568         pkts[3]->pkt_len = total_length3;
 569
 570         if ((qinq == 0) && (mpls == 0)) {
 571                 *slab0_ptr0 = entry0->slab[0];
 572                 *slab0_ptr1 = entry1->slab[0];
 573                 *slab0_ptr2 = entry2->slab[0];
 574                 *slab0_ptr3 = entry3->slab[0];
 575
 576                 if (arp == 0) {
 577                         MACADDR_DST_WRITE(slab1_ptr0, entry0->slab[1]);
 578                         MACADDR_DST_WRITE(slab1_ptr1, entry1->slab[1]);
 579                         MACADDR_DST_WRITE(slab1_ptr2, entry2->slab[1]);
 580                         MACADDR_DST_WRITE(slab1_ptr3, entry3->slab[1]);
 581                 }
 582         }
 583
 584         if (qinq) {
 585                 *slab0_ptr0 = entry0->slab[0];
 586                 *slab0_ptr1 = entry1->slab[0];
 587                 *slab0_ptr2 = entry2->slab[0];
 588                 *slab0_ptr3 = entry3->slab[0];
 589
 590                 *slab1_ptr0 = entry0->slab[1];
 591                 *slab1_ptr1 = entry1->slab[1];
 592                 *slab1_ptr2 = entry2->slab[1];
 593                 *slab1_ptr3 = entry3->slab[1];
 594
 595                 if (arp == 0) {
 596                         MACADDR_DST_WRITE(slab2_ptr0, entry0->slab[2]);
 597                         MACADDR_DST_WRITE(slab2_ptr1, entry1->slab[2]);
 598                         MACADDR_DST_WRITE(slab2_ptr2, entry2->slab[2]);
 599                         MACADDR_DST_WRITE(slab2_ptr3, entry3->slab[2]);
 600                 }
 601
 602                 if (qinq_sched) {
 603                         pkts[0]->hash.sched.lo = sched0 & 0xFFFFFFFF;
 604                         pkts[0]->hash.sched.hi = sched0 >> 32;
 605                         pkts[1]->hash.sched.lo = sched1 & 0xFFFFFFFF;
 606                         pkts[1]->hash.sched.hi = sched1 >> 32;
 607                         pkts[2]->hash.sched.lo = sched2 & 0xFFFFFFFF;
 608                         pkts[2]->hash.sched.hi = sched2 >> 32;
 609                         pkts[3]->hash.sched.lo = sched3 & 0xFFFFFFFF;
 610                         pkts[3]->hash.sched.hi = sched3 >> 32;
 611                 }
 612         }
 613
 614         if (mpls) {
 615                 if (mpls_color_mark) {
 616                         uint64_t mpls_exp0 = rte_bswap64(
 617                                 (MPLS_LABEL(0, pkt0_color, 0, 0) << 32) |
 618                                 MPLS_LABEL(0, pkt0_color, 0, 0));
 619                         uint64_t mpls_exp1 = rte_bswap64(
 620                                 (MPLS_LABEL(0, pkt1_color, 0, 0) << 32) |
 621                                 MPLS_LABEL(0, pkt1_color, 0, 0));
 622                         uint64_t mpls_exp2 = rte_bswap64(
 623                                 (MPLS_LABEL(0, pkt2_color, 0, 0) << 32) |
 624                                 MPLS_LABEL(0, pkt2_color, 0, 0));
 625                         uint64_t mpls_exp3 = rte_bswap64(
 626                                 (MPLS_LABEL(0, pkt3_color, 0, 0) << 32) |
 627                                 MPLS_LABEL(0, pkt3_color, 0, 0));
 628
 629                         *slab0_ptr0 = entry0->slab[0] | mpls_exp0;
 630                         *slab0_ptr1 = entry1->slab[0] | mpls_exp1;
 631                         *slab0_ptr2 = entry2->slab[0] | mpls_exp2;
 632                         *slab0_ptr3 = entry3->slab[0] | mpls_exp3;
 633
 634                         *slab1_ptr0 = entry0->slab[1] | mpls_exp0;
 635                         *slab1_ptr1 = entry1->slab[1] | mpls_exp1;
 636                         *slab1_ptr2 = entry2->slab[1] | mpls_exp2;
 637                         *slab1_ptr3 = entry3->slab[1] | mpls_exp3;
 638
 639                         *slab2_ptr0 = entry0->slab[2];
 640                         *slab2_ptr1 = entry1->slab[2];
 641                         *slab2_ptr2 = entry2->slab[2];
 642                         *slab2_ptr3 = entry3->slab[2];
 643                 } else {
 644                         *slab0_ptr0 = entry0->slab[0];
 645                         *slab0_ptr1 = entry1->slab[0];
 646                         *slab0_ptr2 = entry2->slab[0];
 647                         *slab0_ptr3 = entry3->slab[0];
 648
 649                         *slab1_ptr0 = entry0->slab[1];
 650                         *slab1_ptr1 = entry1->slab[1];
 651                         *slab1_ptr2 = entry2->slab[1];
 652                         *slab1_ptr3 = entry3->slab[1];
 653
 654                         *slab2_ptr0 = entry0->slab[2];
 655                         *slab2_ptr1 = entry1->slab[2];
 656                         *slab2_ptr2 = entry2->slab[2];
 657                         *slab2_ptr3 = entry3->slab[2];
 658                 }
 659
 660                 if (arp == 0) {
 661                         MACADDR_DST_WRITE(slab3_ptr0, entry0->slab[3]);
 662                         MACADDR_DST_WRITE(slab3_ptr1, entry1->slab[3]);
 663                         MACADDR_DST_WRITE(slab3_ptr2, entry2->slab[3]);
 664                         MACADDR_DST_WRITE(slab3_ptr3, entry3->slab[3]);
 665                 }
 666         }
 667
 668         if (arp) {
 669                 arp_key0->port_id = port_id0;
 670                 arp_key1->port_id = port_id1;
 671                 arp_key2->port_id = port_id2;
 672                 arp_key3->port_id = port_id3;
 673
 674                 arp_key0->ip = nh_ip0;
 675                 arp_key1->ip = nh_ip1;
 676                 arp_key2->ip = nh_ip2;
 677                 arp_key3->ip = nh_ip3;
 678         }
 679 }
 680
 681 #define PKT_WORK_ROUTING_ETHERNET(arp)                          \
 682 static inline void                                              \
 683 pkt_work_routing_ether_arp##arp(                                \
 684         struct rte_mbuf *pkt,                                   \
 685         struct rte_pipeline_table_entry *table_entry,           \
 686         void *arg)                                              \
 687 {                                                               \
 688         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 0, 0);\
 689 }
 690
 691 #define PKT4_WORK_ROUTING_ETHERNET(arp)                         \
 692 static inline void                                              \
 693 pkt4_work_routing_ether_arp##arp(                               \
 694         struct rte_mbuf **pkts,                                 \
 695         struct rte_pipeline_table_entry **table_entries,        \
 696         void *arg)                                              \
 697 {                                                               \
 698         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 0, 0);\
 699 }
 700
 701 #define routing_table_ah_hit_ether(arp)                         \
 702 PKT_WORK_ROUTING_ETHERNET(arp)                                  \
 703 PKT4_WORK_ROUTING_ETHERNET(arp)                                 \
 704 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_arp##arp,      \
 705         pkt_work_routing_ether_arp##arp,                        \
 706         pkt4_work_routing_ether_arp##arp)
 707
 708 routing_table_ah_hit_ether(0)
 709 routing_table_ah_hit_ether(1)
 710
 711 #define PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)              \
 712 static inline void                                              \
 713 pkt_work_routing_ether_qinq_sched##sched##_arp##arp(            \
 714         struct rte_mbuf *pkt,                                   \
 715         struct rte_pipeline_table_entry *table_entry,           \
 716         void *arg)                                              \
 717 {                                                               \
 718         pkt_work_routing(pkt, table_entry, arg, arp, 1, sched, 0, 0);\
 719 }
 720
 721 #define PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)             \
 722 static inline void                                              \
 723 pkt4_work_routing_ether_qinq_sched##sched##_arp##arp(           \
 724         struct rte_mbuf **pkts,                                 \
 725         struct rte_pipeline_table_entry **table_entries,        \
 726         void *arg)                                              \
 727 {                                                               \
 728         pkt4_work_routing(pkts, table_entries, arg, arp, 1, sched, 0, 0);\
 729 }
 730
 731 #define routing_table_ah_hit_ether_qinq(sched, arp)             \
 732 PKT_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                      \
 733 PKT4_WORK_ROUTING_ETHERNET_QINQ(sched, arp)                     \
 734 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_qinq_sched##sched##_arp##arp,\
 735         pkt_work_routing_ether_qinq_sched##sched##_arp##arp,    \
 736         pkt4_work_routing_ether_qinq_sched##sched##_arp##arp)
 737
 738 routing_table_ah_hit_ether_qinq(0, 0)
 739 routing_table_ah_hit_ether_qinq(1, 0)
 740 routing_table_ah_hit_ether_qinq(2, 0)
 741 routing_table_ah_hit_ether_qinq(0, 1)
 742 routing_table_ah_hit_ether_qinq(1, 1)
 743 routing_table_ah_hit_ether_qinq(2, 1)
 744
 745 #define PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)              \
 746 static inline void                                              \
 747 pkt_work_routing_ether_mpls_color##color##_arp##arp(            \
 748         struct rte_mbuf *pkt,                                   \
 749         struct rte_pipeline_table_entry *table_entry,           \
 750         void *arg)                                              \
 751 {                                                               \
 752         pkt_work_routing(pkt, table_entry, arg, arp, 0, 0, 1, color);\
 753 }
 754
 755 #define PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)             \
 756 static inline void                                              \
 757 pkt4_work_routing_ether_mpls_color##color##_arp##arp(           \
 758         struct rte_mbuf **pkts,                                 \
 759         struct rte_pipeline_table_entry **table_entries,        \
 760         void *arg)                                              \
 761 {                                                               \
 762         pkt4_work_routing(pkts, table_entries, arg, arp, 0, 0, 1, color);\
 763 }
 764
 765 #define routing_table_ah_hit_ether_mpls(color, arp)             \
 766 PKT_WORK_ROUTING_ETHERNET_MPLS(color, arp)                      \
 767 PKT4_WORK_ROUTING_ETHERNET_MPLS(color, arp)                     \
 768 PIPELINE_TABLE_AH_HIT(routing_table_ah_hit_ether_mpls_color##color##_arp##arp,\
 769         pkt_work_routing_ether_mpls_color##color##_arp##arp,    \
 770         pkt4_work_routing_ether_mpls_color##color##_arp##arp)
 771
 772 routing_table_ah_hit_ether_mpls(0, 0)
 773 routing_table_ah_hit_ether_mpls(1, 0)
 774 routing_table_ah_hit_ether_mpls(0, 1)
 775 routing_table_ah_hit_ether_mpls(1, 1)
 776
 777 static rte_pipeline_table_action_handler_hit
 778 get_routing_table_ah_hit(struct pipeline_routing *p)
 779 {
 780         if (p->params.dbg_ah_disable)
 781                 return NULL;
 782
 783         switch (p->params.encap) {
 784         case PIPELINE_ROUTING_ENCAP_ETHERNET:
 785                 return (p->params.n_arp_entries) ?
 786                         routing_table_ah_hit_ether_arp1 :
 787                         routing_table_ah_hit_ether_arp0;
 788
 789         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
 790                 if (p->params.n_arp_entries)
 791                         switch (p->params.qinq_sched) {
 792                         case 0:
 793                                 return routing_table_ah_hit_ether_qinq_sched0_arp1;
 794                         case 1:
 795                                 return routing_table_ah_hit_ether_qinq_sched1_arp1;
 796                         case 2:
 797                                 return routing_table_ah_hit_ether_qinq_sched2_arp1;
 798                         default:
 799                                 return NULL;
 800                         }
 801                  else
 802                         switch (p->params.qinq_sched) {
 803                         case 0:
 804                                 return routing_table_ah_hit_ether_qinq_sched0_arp0;
 805                         case 1:
 806                                 return routing_table_ah_hit_ether_qinq_sched1_arp0;
 807                         case 2:
 808                                 return routing_table_ah_hit_ether_qinq_sched2_arp0;
 809                         default:
 810                                 return NULL;
 811                         }
 812
 813         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
 814                 if (p->params.n_arp_entries)
 815                         if (p->params.mpls_color_mark)
 816                                 return routing_table_ah_hit_ether_mpls_color1_arp1;
 817                         else
 818                                 return routing_table_ah_hit_ether_mpls_color0_arp1;
 819                 else
 820                         if (p->params.mpls_color_mark)
 821                                 return routing_table_ah_hit_ether_mpls_color1_arp0;
 822                         else
 823                                 return routing_table_ah_hit_ether_mpls_color0_arp0;
 824
 825         default:
 826                 return NULL;
 827         }
 828 }
 829
 830 /*
 831  * ARP table
 832  */
 833 struct arp_table_entry {
 834         struct rte_pipeline_table_entry head;
 835         uint64_t macaddr;
 836 };
 837
 838 /**
 839  * ARP table AH
 840  */
 841 static inline void
 842 pkt_work_arp(
 843         struct rte_mbuf *pkt,
 844         struct rte_pipeline_table_entry *table_entry,
 845         __rte_unused void *arg)
 846 {
 847         struct arp_table_entry *entry = (struct arp_table_entry *) table_entry;
 848
 849         /* Read */
 850         uint64_t macaddr_dst = entry->macaddr;
 851         uint64_t *slab_ptr = (uint64_t *) ((char *) pkt->buf_addr +
 852                 (pkt->data_off - 2));
 853
 854         /* Compute */
 855
 856         /* Write */
 857         MACADDR_DST_WRITE(slab_ptr, macaddr_dst);
 858 }
 859
 860 static inline void
 861 pkt4_work_arp(
 862         struct rte_mbuf **pkts,
 863         struct rte_pipeline_table_entry **table_entries,
 864         __rte_unused void *arg)
 865 {
 866         struct arp_table_entry *entry0 =
 867                 (struct arp_table_entry *) table_entries[0];
 868         struct arp_table_entry *entry1 =
 869                 (struct arp_table_entry *) table_entries[1];
 870         struct arp_table_entry *entry2 =
 871                 (struct arp_table_entry *) table_entries[2];
 872         struct arp_table_entry *entry3 =
 873                 (struct arp_table_entry *) table_entries[3];
 874
 875         /* Read */
 876         uint64_t macaddr_dst0 = entry0->macaddr;
 877         uint64_t macaddr_dst1 = entry1->macaddr;
 878         uint64_t macaddr_dst2 = entry2->macaddr;
 879         uint64_t macaddr_dst3 = entry3->macaddr;
 880
 881         uint64_t *slab_ptr0 = (uint64_t *) ((char *) pkts[0]->buf_addr +
 882                 (pkts[0]->data_off - 2));
 883         uint64_t *slab_ptr1 = (uint64_t *) ((char *) pkts[1]->buf_addr +
 884                 (pkts[1]->data_off - 2));
 885         uint64_t *slab_ptr2 = (uint64_t *) ((char *) pkts[2]->buf_addr +
 886                 (pkts[2]->data_off - 2));
 887         uint64_t *slab_ptr3 = (uint64_t *) ((char *) pkts[3]->buf_addr +
 888                 (pkts[3]->data_off - 2));
 889
 890         /* Compute */
 891
 892         /* Write */
 893         MACADDR_DST_WRITE(slab_ptr0, macaddr_dst0);
 894         MACADDR_DST_WRITE(slab_ptr1, macaddr_dst1);
 895         MACADDR_DST_WRITE(slab_ptr2, macaddr_dst2);
 896         MACADDR_DST_WRITE(slab_ptr3, macaddr_dst3);
 897 }
 898
 899 PIPELINE_TABLE_AH_HIT(arp_table_ah_hit,
 900         pkt_work_arp,
 901         pkt4_work_arp);
 902
 903 static rte_pipeline_table_action_handler_hit
 904 get_arp_table_ah_hit(struct pipeline_routing *p)
 905 {
 906         if (p->params.dbg_ah_disable)
 907                 return NULL;
 908
 909         return arp_table_ah_hit;
 910 }
 911
 912 /*
 913  * Argument parsing
 914  */
 915 int
 916 pipeline_routing_parse_args(struct pipeline_routing_params *p,
 917         struct pipeline_params *params)
 918 {
 919         uint32_t n_routes_present = 0;
 920         uint32_t encap_present = 0;
 921         uint32_t qinq_sched_present = 0;
 922         uint32_t mpls_color_mark_present = 0;
 923         uint32_t n_arp_entries_present = 0;
 924         uint32_t ip_hdr_offset_present = 0;
 925         uint32_t arp_key_offset_present = 0;
 926         uint32_t color_offset_present = 0;
 927         uint32_t dbg_ah_disable_present = 0;
 928         uint32_t i;
 929
 930         /* default values */
 931         p->n_routes = PIPELINE_ROUTING_N_ROUTES_DEFAULT;
 932         p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 933         p->qinq_sched = 0;
 934         p->mpls_color_mark = 0;
 935         p->n_arp_entries = 0;
 936         p->dbg_ah_disable = 0;
 937
 938         for (i = 0; i < params->n_args; i++) {
 939                 char *arg_name = params->args_name[i];
 940                 char *arg_value = params->args_value[i];
 941
 942                 /* n_routes */
 943                 if (strcmp(arg_name, "n_routes") == 0) {
 944                         int status;
 945
 946                         PIPELINE_PARSE_ERR_DUPLICATE(
 947                                 n_routes_present == 0, params->name,
 948                                 arg_name);
 949                         n_routes_present = 1;
 950
 951                         status = parser_read_uint32(&p->n_routes,
 952                                 arg_value);
 953                         PIPELINE_PARSE_ERR_INV_VAL(((status != -EINVAL) &&
 954                                 (p->n_routes != 0)), params->name,
 955                                 arg_name, arg_value);
 956                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
 957                                 params->name, arg_name, arg_value);
 958
 959                         continue;
 960                 }
 961
 962                 /* encap */
 963                 if (strcmp(arg_name, "encap") == 0) {
 964                         PIPELINE_PARSE_ERR_DUPLICATE(encap_present == 0,
 965                                 params->name, arg_name);
 966                         encap_present = 1;
 967
 968                         /* ethernet */
 969                         if (strcmp(arg_value, "ethernet") == 0) {
 970                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET;
 971                                 continue;
 972                         }
 973
 974                         /* ethernet_qinq */
 975                         if (strcmp(arg_value, "ethernet_qinq") == 0) {
 976                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ;
 977                                 continue;
 978                         }
 979
 980                         /* ethernet_mpls */
 981                         if (strcmp(arg_value, "ethernet_mpls") == 0) {
 982                                 p->encap = PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS;
 983                                 continue;
 984                         }
 985
 986                         /* any other */
 987                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
 988                                 arg_name, arg_value);
 989                 }
 990
 991                 /* qinq_sched */
 992                 if (strcmp(arg_name, "qinq_sched") == 0) {
 993                         int status;
 994
 995                         PIPELINE_PARSE_ERR_DUPLICATE(
 996                                 qinq_sched_present == 0, params->name,
 997                                 arg_name);
 998                         qinq_sched_present = 1;
 999
1000                         status = parser_read_arg_bool(arg_value);
1001                         if (status == -EINVAL) {
1002                                 if (strcmp(arg_value, "test") == 0) {
1003                                         p->qinq_sched = 2;
1004                                         continue;
1005                                 }
1006                         } else {
1007                                 p->qinq_sched = status;
1008                                 continue;
1009                         }
1010
1011                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1012                                 arg_name, arg_value);
1013                 }
1014
1015                 /* mpls_color_mark */
1016                 if (strcmp(arg_name, "mpls_color_mark") == 0) {
1017                         int status;
1018
1019                         PIPELINE_PARSE_ERR_DUPLICATE(
1020                                 mpls_color_mark_present == 0,
1021                                 params->name, arg_name);
1022                         mpls_color_mark_present = 1;
1023
1024
1025                         status = parser_read_arg_bool(arg_value);
1026                         if (status >= 0) {
1027                                 p->mpls_color_mark = status;
1028                                 continue;
1029                         }
1030
1031                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1032                                 arg_name, arg_value);
1033                 }
1034
1035                 /* n_arp_entries */
1036                 if (strcmp(arg_name, "n_arp_entries") == 0) {
1037                         int status;
1038
1039                         PIPELINE_PARSE_ERR_DUPLICATE(
1040                                 n_arp_entries_present == 0, params->name,
1041                                 arg_name);
1042                         n_arp_entries_present = 1;
1043
1044                         status = parser_read_uint32(&p->n_arp_entries,
1045                                 arg_value);
1046                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1047                                 params->name, arg_name, arg_value);
1048                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1049                                 params->name, arg_name, arg_value);
1050
1051                         continue;
1052                 }
1053
1054                 /* ip_hdr_offset */
1055                 if (strcmp(arg_name, "ip_hdr_offset") == 0) {
1056                         int status;
1057
1058                         PIPELINE_PARSE_ERR_DUPLICATE(
1059                                 ip_hdr_offset_present == 0, params->name,
1060                                 arg_name);
1061                         ip_hdr_offset_present = 1;
1062
1063                         status = parser_read_uint32(&p->ip_hdr_offset,
1064                                 arg_value);
1065                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1066                                 params->name, arg_name, arg_value);
1067                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1068                                 params->name, arg_name, arg_value);
1069
1070                         continue;
1071                 }
1072
1073                 /* arp_key_offset */
1074                 if (strcmp(arg_name, "arp_key_offset") == 0) {
1075                         int status;
1076
1077                         PIPELINE_PARSE_ERR_DUPLICATE(
1078                                 arp_key_offset_present == 0, params->name,
1079                                 arg_name);
1080                         arp_key_offset_present = 1;
1081
1082                         status = parser_read_uint32(&p->arp_key_offset,
1083                                 arg_value);
1084                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1085                                 params->name, arg_name, arg_value);
1086                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1087                                 params->name, arg_name, arg_value);
1088
1089                         continue;
1090                 }
1091
1092                 /* color_offset */
1093                 if (strcmp(arg_name, "color_offset") == 0) {
1094                         int status;
1095
1096                         PIPELINE_PARSE_ERR_DUPLICATE(
1097                                 color_offset_present == 0, params->name,
1098                                 arg_name);
1099                         color_offset_present = 1;
1100
1101                         status = parser_read_uint32(&p->color_offset,
1102                                 arg_value);
1103                         PIPELINE_PARSE_ERR_INV_VAL((status != -EINVAL),
1104                                 params->name, arg_name, arg_value);
1105                         PIPELINE_PARSE_ERR_OUT_RNG((status != -ERANGE),
1106                                 params->name, arg_name, arg_value);
1107
1108                         continue;
1109                 }
1110
1111                 /* debug */
1112                 if (strcmp(arg_name, "dbg_ah_disable") == 0) {
1113                         int status;
1114
1115                         PIPELINE_PARSE_ERR_DUPLICATE(
1116                                 dbg_ah_disable_present == 0, params->name,
1117                                 arg_name);
1118                         dbg_ah_disable_present = 1;
1119
1120                         status = parser_read_arg_bool(arg_value);
1121                         if (status >= 0) {
1122                                 p->dbg_ah_disable = status;
1123                                 continue;
1124                         }
1125
1126                         PIPELINE_PARSE_ERR_INV_VAL(0, params->name,
1127                                 arg_name, arg_value);
1128
1129                         continue;
1130                 }
1131
1132                 /* any other */
1133                 PIPELINE_PARSE_ERR_INV_ENT(0, params->name, arg_name);
1134         }
1135
1136         /* Check that mandatory arguments are present */
1137         PIPELINE_PARSE_ERR_MANDATORY(ip_hdr_offset_present, params->name,
1138                 "ip_hdr_offset");
1139
1140         /* Check relations between arguments */
1141         switch (p->encap) {
1142         case PIPELINE_ROUTING_ENCAP_ETHERNET:
1143                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1144                         "section \"%s\": encap = ethernet, therefore "
1145                         "qinq_sched = yes/test is not allowed",
1146                         params->name);
1147                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1148                         "in section \"%s\": encap = ethernet, therefore "
1149                         "mpls_color_mark = yes is not allowed",
1150                         params->name);
1151                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1152                         "in section \"%s\": encap = ethernet, therefore "
1153                         "color_offset is not allowed",
1154                         params->name);
1155                 break;
1156
1157         case PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ:
1158                 PIPELINE_ARG_CHECK((!p->mpls_color_mark), "Parse error "
1159                         "in section \"%s\": encap = ethernet_qinq, "
1160                         "therefore mpls_color_mark = yes is not allowed",
1161                         params->name);
1162                 PIPELINE_ARG_CHECK((!color_offset_present), "Parse error "
1163                         "in section \"%s\": encap = ethernet_qinq, "
1164                         "therefore color_offset is not allowed",
1165                         params->name);
1166                 break;
1167
1168         case PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS:
1169                 PIPELINE_ARG_CHECK((!p->qinq_sched), "Parse error in "
1170                         "section \"%s\": encap = ethernet_mpls, therefore "
1171                         "qinq_sched  = yes/test is not allowed",
1172                         params->name);
1173                 break;
1174         }
1175
1176         PIPELINE_ARG_CHECK((!(p->n_arp_entries &&
1177                 (!arp_key_offset_present))), "Parse error in section "
1178                         "\"%s\": n_arp_entries is set while "
1179                         "arp_key_offset is not set", params->name);
1180
1181         PIPELINE_ARG_CHECK((!((p->n_arp_entries == 0) &&
1182                 arp_key_offset_present)), "Parse error in section "
1183                         "\"%s\": arp_key_offset present while "
1184                         "n_arp_entries is not set", params->name);
1185
1186         return 0;
1187 }
1188
1189 static void *
1190 pipeline_routing_init(struct pipeline_params *params,
1191         __rte_unused void *arg)
1192 {
1193         struct pipeline *p;
1194         struct pipeline_routing *p_rt;
1195         uint32_t size, i;
1196
1197         /* Check input arguments */
1198         if ((params == NULL) ||
1199                 (params->n_ports_in == 0) ||
1200                 (params->n_ports_out == 0))
1201                 return NULL;
1202
1203         /* Memory allocation */
1204         size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_routing));
1205         p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
1206         p_rt = (struct pipeline_routing *) p;
1207         if (p == NULL)
1208                 return NULL;
1209
1210         strcpy(p->name, params->name);
1211         p->log_level = params->log_level;
1212
1213         PLOG(p, HIGH, "Routing");
1214
1215         /* Parse arguments */
1216         if (pipeline_routing_parse_args(&p_rt->params, params))
1217                 return NULL;
1218
1219         /* Pipeline */
1220         {
1221                 struct rte_pipeline_params pipeline_params = {
1222                         .name = params->name,
1223                         .socket_id = params->socket_id,
1224                         .offset_port_id = 0,
1225                 };
1226
1227                 p->p = rte_pipeline_create(&pipeline_params);
1228                 if (p->p == NULL) {
1229                         rte_free(p);
1230                         return NULL;
1231                 }
1232         }
1233
1234         /* Input ports */
1235         p->n_ports_in = params->n_ports_in;
1236         for (i = 0; i < p->n_ports_in; i++) {
1237                 struct rte_pipeline_port_in_params port_params = {
1238                         .ops = pipeline_port_in_params_get_ops(
1239                                 &params->port_in[i]),
1240                         .arg_create = pipeline_port_in_params_convert(
1241                                 &params->port_in[i]),
1242                         .f_action = NULL,
1243                         .arg_ah = NULL,
1244                         .burst_size = params->port_in[i].burst_size,
1245                 };
1246
1247                 int status = rte_pipeline_port_in_create(p->p,
1248                         &port_params,
1249                         &p->port_in_id[i]);
1250
1251                 if (status) {
1252                         rte_pipeline_free(p->p);
1253                         rte_free(p);
1254                         return NULL;
1255                 }
1256         }
1257
1258         /* Output ports */
1259         p->n_ports_out = params->n_ports_out;
1260         for (i = 0; i < p->n_ports_out; i++) {
1261                 struct rte_pipeline_port_out_params port_params = {
1262                         .ops = pipeline_port_out_params_get_ops(
1263                                 &params->port_out[i]),
1264                         .arg_create = pipeline_port_out_params_convert(
1265                                 &params->port_out[i]),
1266                         .f_action = NULL,
1267                         .f_action_bulk = NULL,
1268                         .arg_ah = NULL,
1269                 };
1270
1271                 int status = rte_pipeline_port_out_create(p->p,
1272                         &port_params,
1273                         &p->port_out_id[i]);
1274
1275                 if (status) {
1276                         rte_pipeline_free(p->p);
1277                         rte_free(p);
1278                         return NULL;
1279                 }
1280         }
1281
1282         /* Routing table */
1283         p->n_tables = 1;
1284         {
1285                 struct rte_table_lpm_params table_lpm_params = {
1286                         .name = p->name,
1287                         .n_rules = p_rt->params.n_routes,
1288                         .entry_unique_size = sizeof(struct routing_table_entry),
1289                         .offset = p_rt->params.ip_hdr_offset +
1290                                 __builtin_offsetof(struct ipv4_hdr, dst_addr),
1291                 };
1292
1293                 struct rte_pipeline_table_params table_params = {
1294                                 .ops = &rte_table_lpm_ops,
1295                                 .arg_create = &table_lpm_params,
1296                                 .f_action_hit = get_routing_table_ah_hit(p_rt),
1297                                 .f_action_miss = NULL,
1298                                 .arg_ah = p_rt,
1299                                 .action_data_size =
1300                                         sizeof(struct routing_table_entry) -
1301                                         sizeof(struct rte_pipeline_table_entry),
1302                         };
1303
1304                 int status;
1305
1306                 status = rte_pipeline_table_create(p->p,
1307                         &table_params,
1308                         &p->table_id[0]);
1309
1310                 if (status) {
1311                         rte_pipeline_free(p->p);
1312                         rte_free(p);
1313                         return NULL;
1314                 }
1315         }
1316
1317         /* ARP table configuration */
1318         if (p_rt->params.n_arp_entries) {
1319                 struct rte_table_hash_key8_ext_params table_arp_params = {
1320                         .n_entries = p_rt->params.n_arp_entries,
1321                         .n_entries_ext = p_rt->params.n_arp_entries,
1322                         .f_hash = hash_default_key8,
1323                         .seed = 0,
1324                         .signature_offset = 0, /* Unused */
1325                         .key_offset = p_rt->params.arp_key_offset,
1326                 };
1327
1328                 struct rte_pipeline_table_params table_params = {
1329                         .ops = &rte_table_hash_key8_ext_dosig_ops,
1330                         .arg_create = &table_arp_params,
1331                         .f_action_hit = get_arp_table_ah_hit(p_rt),
1332                         .f_action_miss = NULL,
1333                         .arg_ah = p_rt,
1334                         .action_data_size = sizeof(struct arp_table_entry) -
1335                                 sizeof(struct rte_pipeline_table_entry),
1336                 };
1337
1338                 int status;
1339
1340                 status = rte_pipeline_table_create(p->p,
1341                         &table_params,
1342                         &p->table_id[1]);
1343
1344                 if (status) {
1345                         rte_pipeline_free(p->p);
1346                         rte_free(p);
1347                         return NULL;
1348                 }
1349
1350                 p->n_tables++;
1351         }
1352
1353         /* Connecting input ports to tables */
1354         for (i = 0; i < p->n_ports_in; i++) {
1355                 int status = rte_pipeline_port_in_connect_to_table(p->p,
1356                         p->port_in_id[i],
1357                         p->table_id[0]);
1358
1359                 if (status) {
1360                         rte_pipeline_free(p->p);
1361                         rte_free(p);
1362                         return NULL;
1363                 }
1364         }
1365
1366         /* Enable input ports */
1367         for (i = 0; i < p->n_ports_in; i++) {
1368                 int status = rte_pipeline_port_in_enable(p->p,
1369                         p->port_in_id[i]);
1370
1371                 if (status) {
1372                         rte_pipeline_free(p->p);
1373                         rte_free(p);
1374                         return NULL;
1375                 }
1376         }
1377
1378         /* Check pipeline consistency */
1379         if (rte_pipeline_check(p->p) < 0) {
1380                 rte_pipeline_free(p->p);
1381                 rte_free(p);
1382                 return NULL;
1383         }
1384
1385         /* Message queues */
1386         p->n_msgq = params->n_msgq;
1387         for (i = 0; i < p->n_msgq; i++)
1388                 p->msgq_in[i] = params->msgq_in[i];
1389         for (i = 0; i < p->n_msgq; i++)
1390                 p->msgq_out[i] = params->msgq_out[i];
1391
1392         /* Message handlers */
1393         memcpy(p->handlers, handlers, sizeof(p->handlers));
1394         memcpy(p_rt->custom_handlers,
1395                 custom_handlers,
1396                 sizeof(p_rt->custom_handlers));
1397
1398         return p;
1399 }
1400
1401 static int
1402 pipeline_routing_free(void *pipeline)
1403 {
1404         struct pipeline *p = (struct pipeline *) pipeline;
1405
1406         /* Check input arguments */
1407         if (p == NULL)
1408                 return -1;
1409
1410         /* Free resources */
1411         rte_pipeline_free(p->p);
1412         rte_free(p);
1413         return 0;
1414 }
1415
1416 static int
1417 pipeline_routing_track(void *pipeline,
1418         __rte_unused uint32_t port_in,
1419         uint32_t *port_out)
1420 {
1421         struct pipeline *p = (struct pipeline *) pipeline;
1422
1423         /* Check input arguments */
1424         if ((p == NULL) ||
1425                 (port_in >= p->n_ports_in) ||
1426                 (port_out == NULL))
1427                 return -1;
1428
1429         if (p->n_ports_in == 1) {
1430                 *port_out = 0;
1431                 return 0;
1432         }
1433
1434         return -1;
1435 }
1436
1437 static int
1438 pipeline_routing_timer(void *pipeline)
1439 {
1440         struct pipeline *p = (struct pipeline *) pipeline;
1441
1442         pipeline_msg_req_handle(p);
1443         rte_pipeline_flush(p->p);
1444
1445         return 0;
1446 }
1447
1448 void *
1449 pipeline_routing_msg_req_custom_handler(struct pipeline *p,
1450         void *msg)
1451 {
1452         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1453         struct pipeline_custom_msg_req *req = msg;
1454         pipeline_msg_req_handler f_handle;
1455
1456         f_handle = (req->subtype < PIPELINE_ROUTING_MSG_REQS) ?
1457                 p_rt->custom_handlers[req->subtype] :
1458                 pipeline_msg_req_invalid_handler;
1459
1460         if (f_handle == NULL)
1461                 f_handle = pipeline_msg_req_invalid_handler;
1462
1463         return f_handle(p, req);
1464 }
1465
1466 void *
1467 pipeline_routing_msg_req_route_add_handler(struct pipeline *p, void *msg)
1468 {
1469         struct pipeline_routing *p_rt = (struct pipeline_routing *) p;
1470         struct pipeline_routing_route_add_msg_req *req = msg;
1471         struct pipeline_routing_route_add_msg_rsp *rsp = msg;
1472
1473         struct rte_table_lpm_key key = {
1474                 .ip = req->key.key.ipv4.ip,
1475                 .depth = req->key.key.ipv4.depth,
1476         };
1477
1478         struct routing_table_entry entry_arp0 = {
1479                 .head = {
1480                         .action = RTE_PIPELINE_ACTION_PORT,
1481                         {.port_id = p->port_out_id[req->data.port_id]},
1482                 },
1483
1484                 .flags = req->data.flags,
1485                 .port_id = req->data.port_id,
1486                 .ip = 0,
1487                 .data_offset = 0,
1488                 .ether_l2_length = 0,
1489                 .slab = {0},
1490                 .slab_offset = {0},
1491         };
1492
1493         struct routing_table_entry entry_arp1 = {
1494                 .head = {
1495                         .action = RTE_PIPELINE_ACTION_TABLE,
1496                         {.table_id = p->table_id[1]},
1497                 },
1498
1499                 .flags = req->data.flags,
1500                 .port_id = req->data.port_id,
1501                 .ip = rte_bswap32(req->data.ethernet.ip),
1502                 .data_offset = 0,
1503                 .ether_l2_length = 0,
1504                 .slab = {0},
1505                 .slab_offset = {0},
1506         };
1507
1508         struct rte_pipeline_table_entry *entry = (p_rt->params.n_arp_entries) ?
1509                 (struct rte_pipeline_table_entry *) &entry_arp1 :
1510                 (struct rte_pipeline_table_entry *) &entry_arp0;
1511
1512         if ((req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) ||
1513                 ((p_rt->params.n_arp_entries == 0) &&
1514                         (req->data.flags & PIPELINE_ROUTING_ROUTE_ARP)) ||
1515                 (p_rt->params.n_arp_entries &&
1516                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_ARP) == 0)) ||
1517                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1518                         (req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ)) ||
1519                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1520                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_QINQ) == 0)) ||
1521                 ((p_rt->params.encap != PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1522                         (req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS)) ||
1523                 ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1524                         ((req->data.flags & PIPELINE_ROUTING_ROUTE_MPLS) == 0))) {
1525                 rsp->status = -1;
1526                 return rsp;
1527         }
1528
1529         /* Ether - ARP off */
1530         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1531                 (p_rt->params.n_arp_entries == 0)) {
1532                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1533                 uint64_t macaddr_dst;
1534                 uint64_t ethertype = ETHER_TYPE_IPv4;
1535
1536                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1537                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1538
1539                 entry_arp0.slab[0] =
1540                         rte_bswap64((macaddr_src << 16) | ethertype);
1541                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1542
1543                 entry_arp0.slab[1] = rte_bswap64(macaddr_dst);
1544                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1545
1546                 entry_arp0.data_offset = entry_arp0.slab_offset[1] + 2
1547                         - sizeof(struct rte_mbuf);
1548                 entry_arp0.ether_l2_length = 14;
1549         }
1550
1551         /* Ether - ARP on */
1552         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET) &&
1553                 p_rt->params.n_arp_entries) {
1554                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1555                 uint64_t ethertype = ETHER_TYPE_IPv4;
1556
1557                 entry_arp1.slab[0] = rte_bswap64((macaddr_src << 16) |
1558                         ethertype);
1559                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1560
1561                 entry_arp1.data_offset = entry_arp1.slab_offset[0] - 6
1562                         - sizeof(struct rte_mbuf);
1563                 entry_arp1.ether_l2_length = 14;
1564         }
1565
1566         /* Ether QinQ - ARP off */
1567         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1568                 (p_rt->params.n_arp_entries == 0)) {
1569                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1570                 uint64_t macaddr_dst;
1571                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1572                 uint64_t ethertype_vlan = 0x8100;
1573                 uint64_t ethertype_qinq = 0x9100;
1574                 uint64_t svlan = req->data.l2.qinq.svlan;
1575                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1576
1577                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1578                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1579
1580                 entry_arp0.slab[0] = rte_bswap64((svlan << 48) |
1581                         (ethertype_vlan << 32) |
1582                         (cvlan << 16) |
1583                         ethertype_ipv4);
1584                 entry_arp0.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1585
1586                 entry_arp0.slab[1] = rte_bswap64((macaddr_src << 16) |
1587                         ethertype_qinq);
1588                 entry_arp0.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1589
1590                 entry_arp0.slab[2] = rte_bswap64(macaddr_dst);
1591                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset - 3 * 8;
1592
1593                 entry_arp0.data_offset = entry_arp0.slab_offset[2] + 2
1594                         - sizeof(struct rte_mbuf);
1595                 entry_arp0.ether_l2_length = 22;
1596         }
1597
1598         /* Ether QinQ - ARP on */
1599         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_QINQ) &&
1600                 p_rt->params.n_arp_entries) {
1601                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1602                 uint64_t ethertype_ipv4 = ETHER_TYPE_IPv4;
1603                 uint64_t ethertype_vlan = 0x8100;
1604                 uint64_t ethertype_qinq = 0x9100;
1605                 uint64_t svlan = req->data.l2.qinq.svlan;
1606                 uint64_t cvlan = req->data.l2.qinq.cvlan;
1607
1608                 entry_arp1.slab[0] = rte_bswap64((svlan << 48) |
1609                         (ethertype_vlan << 32) |
1610                         (cvlan << 16) |
1611                         ethertype_ipv4);
1612                 entry_arp1.slab_offset[0] = p_rt->params.ip_hdr_offset - 8;
1613
1614                 entry_arp1.slab[1] = rte_bswap64((macaddr_src << 16) |
1615                         ethertype_qinq);
1616                 entry_arp1.slab_offset[1] = p_rt->params.ip_hdr_offset - 2 * 8;
1617
1618                 entry_arp1.data_offset = entry_arp1.slab_offset[1] - 6
1619                         - sizeof(struct rte_mbuf);
1620                 entry_arp1.ether_l2_length = 22;
1621         }
1622
1623         /* Ether MPLS - ARP off */
1624         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1625                 (p_rt->params.n_arp_entries == 0)) {
1626                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1627                 uint64_t macaddr_dst;
1628                 uint64_t ethertype_mpls = 0x8847;
1629
1630                 uint64_t label0 = req->data.l2.mpls.labels[0];
1631                 uint64_t label1 = req->data.l2.mpls.labels[1];
1632                 uint64_t label2 = req->data.l2.mpls.labels[2];
1633                 uint64_t label3 = req->data.l2.mpls.labels[3];
1634                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1635
1636                 macaddr_dst = *((uint64_t *)&(req->data.ethernet.macaddr));
1637                 macaddr_dst = rte_bswap64(macaddr_dst << 16);
1638
1639                 switch (n_labels) {
1640                 case 1:
1641                         entry_arp0.slab[0] = 0;
1642                         entry_arp0.slab_offset[0] =
1643                                 p_rt->params.ip_hdr_offset - 8;
1644
1645                         entry_arp0.slab[1] = rte_bswap64(
1646                                 MPLS_LABEL(label0, 0, 1, 0));
1647                         entry_arp0.slab_offset[1] =
1648                                 p_rt->params.ip_hdr_offset - 8;
1649                         break;
1650
1651                 case 2:
1652                         entry_arp0.slab[0] = 0;
1653                         entry_arp0.slab_offset[0] =
1654                                 p_rt->params.ip_hdr_offset - 8;
1655
1656                         entry_arp0.slab[1] = rte_bswap64(
1657                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1658                                 MPLS_LABEL(label1, 0, 1, 0));
1659                         entry_arp0.slab_offset[1] =
1660                                 p_rt->params.ip_hdr_offset - 8;
1661                         break;
1662
1663                 case 3:
1664                         entry_arp0.slab[0] = rte_bswap64(
1665                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1666                                 MPLS_LABEL(label2, 0, 1, 0));
1667                         entry_arp0.slab_offset[0] =
1668                                 p_rt->params.ip_hdr_offset - 8;
1669
1670                         entry_arp0.slab[1] = rte_bswap64(
1671                                 MPLS_LABEL(label0, 0, 0, 0));
1672                         entry_arp0.slab_offset[1] =
1673                                 p_rt->params.ip_hdr_offset - 2 * 8;
1674                         break;
1675
1676                 case 4:
1677                         entry_arp0.slab[0] = rte_bswap64(
1678                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1679                                 MPLS_LABEL(label3, 0, 1, 0));
1680                         entry_arp0.slab_offset[0] =
1681                                 p_rt->params.ip_hdr_offset - 8;
1682
1683                         entry_arp0.slab[1] = rte_bswap64(
1684                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1685                                 MPLS_LABEL(label1, 0, 0, 0));
1686                         entry_arp0.slab_offset[1] =
1687                                 p_rt->params.ip_hdr_offset - 2 * 8;
1688                         break;
1689
1690                 default:
1691                         rsp->status = -1;
1692                         return rsp;
1693                 }
1694
1695                 entry_arp0.slab[2] = rte_bswap64((macaddr_src << 16) |
1696                         ethertype_mpls);
1697                 entry_arp0.slab_offset[2] = p_rt->params.ip_hdr_offset -
1698                         (n_labels * 4 + 8);
1699
1700                 entry_arp0.slab[3] = rte_bswap64(macaddr_dst);
1701                 entry_arp0.slab_offset[3] = p_rt->params.ip_hdr_offset -
1702                         (n_labels * 4 + 2 * 8);
1703
1704                 entry_arp0.data_offset = entry_arp0.slab_offset[3] + 2
1705                         - sizeof(struct rte_mbuf);
1706                 entry_arp0.ether_l2_length = n_labels * 4 + 14;
1707         }
1708
1709         /* Ether MPLS - ARP on */
1710         if ((p_rt->params.encap == PIPELINE_ROUTING_ENCAP_ETHERNET_MPLS) &&
1711                 p_rt->params.n_arp_entries) {
1712                 uint64_t macaddr_src = MAC_SRC_DEFAULT;
1713                 uint64_t ethertype_mpls = 0x8847;
1714
1715                 uint64_t label0 = req->data.l2.mpls.labels[0];
1716                 uint64_t label1 = req->data.l2.mpls.labels[1];
1717                 uint64_t label2 = req->data.l2.mpls.labels[2];
1718                 uint64_t label3 = req->data.l2.mpls.labels[3];
1719                 uint32_t n_labels = req->data.l2.mpls.n_labels;
1720
1721                 switch (n_labels) {
1722                 case 1:
1723                         entry_arp1.slab[0] = 0;
1724                         entry_arp1.slab_offset[0] =
1725                                 p_rt->params.ip_hdr_offset - 8;
1726
1727                         entry_arp1.slab[1] = rte_bswap64(
1728                                 MPLS_LABEL(label0, 0, 1, 0));
1729                         entry_arp1.slab_offset[1] =
1730                                 p_rt->params.ip_hdr_offset - 8;
1731                         break;
1732
1733                 case 2:
1734                         entry_arp1.slab[0] = 0;
1735                         entry_arp1.slab_offset[0] =
1736                                 p_rt->params.ip_hdr_offset - 8;
1737
1738                         entry_arp1.slab[1] = rte_bswap64(
1739                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1740                                 MPLS_LABEL(label1, 0, 1, 0));
1741                         entry_arp1.slab_offset[1] =
1742                                 p_rt->params.ip_hdr_offset - 8;
1743                         break;
1744
1745                 case 3:
1746                         entry_arp1.slab[0] = rte_bswap64(
1747                                 (MPLS_LABEL(label1, 0, 0, 0) << 32) |
1748                                 MPLS_LABEL(label2, 0, 1, 0));
1749                         entry_arp1.slab_offset[0] =
1750                                 p_rt->params.ip_hdr_offset - 8;
1751
1752                         entry_arp1.slab[1] = rte_bswap64(
1753                                 MPLS_LABEL(label0, 0, 0, 0));
1754                         entry_arp1.slab_offset[1] =
1755                                 p_rt->params.ip_hdr_offset - 2 * 8;
1756                         break;
1757
1758                 case 4:
1759                         entry_arp1.slab[0] = rte_bswap64(
1760                                 (MPLS_LABEL(label2, 0, 0, 0) << 32) |
1761                                 MPLS_LABEL(label3, 0, 1, 0));
1762                         entry_arp1.slab_offset[0] =
1763                                 p_rt->params.ip_hdr_offset - 8;
1764
1765                         entry_arp1.slab[1] = rte_bswap64(
1766                                 (MPLS_LABEL(label0, 0, 0, 0) << 32) |
1767                                 MPLS_LABEL(label1, 0, 0, 0));
1768                         entry_arp1.slab_offset[1] =
1769                                 p_rt->params.ip_hdr_offset - 2 * 8;
1770                         break;
1771
1772                 default:
1773                         rsp->status = -1;
1774                         return rsp;
1775                 }
1776
1777                 entry_arp1.slab[2] = rte_bswap64((macaddr_src << 16) |
1778                         ethertype_mpls);
1779                 entry_arp1.slab_offset[2] = p_rt->params.ip_hdr_offset -
1780                         (n_labels * 4 + 8);
1781
1782                 entry_arp1.data_offset = entry_arp1.slab_offset[2] - 6
1783                         - sizeof(struct rte_mbuf);
1784                 entry_arp1.ether_l2_length = n_labels * 4 + 14;
1785         }
1786
1787         rsp->status = rte_pipeline_table_entry_add(p->p,
1788                 p->table_id[0],
1789                 &key,
1790                 entry,
1791                 &rsp->key_found,
1792                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1793
1794         return rsp;
1795 }
1796
1797 void *
1798 pipeline_routing_msg_req_route_del_handler(struct pipeline *p, void *msg)
1799 {
1800         struct pipeline_routing_route_delete_msg_req *req = msg;
1801         struct pipeline_routing_route_delete_msg_rsp *rsp = msg;
1802
1803         struct rte_table_lpm_key key = {
1804                 .ip = req->key.key.ipv4.ip,
1805                 .depth = req->key.key.ipv4.depth,
1806         };
1807
1808         if (req->key.type != PIPELINE_ROUTING_ROUTE_IPV4) {
1809                 rsp->status = -1;
1810                 return rsp;
1811         }
1812
1813         rsp->status = rte_pipeline_table_entry_delete(p->p,
1814                 p->table_id[0],
1815                 &key,
1816                 &rsp->key_found,
1817                 NULL);
1818
1819         return rsp;
1820 }
1821
1822 void *
1823 pipeline_routing_msg_req_route_add_default_handler(struct pipeline *p,
1824         void *msg)
1825 {
1826         struct pipeline_routing_route_add_default_msg_req *req = msg;
1827         struct pipeline_routing_route_add_default_msg_rsp *rsp = msg;
1828
1829         struct routing_table_entry default_entry = {
1830                 .head = {
1831                         .action = RTE_PIPELINE_ACTION_PORT,
1832                         {.port_id = p->port_out_id[req->port_id]},
1833                 },
1834
1835                 .flags = 0,
1836                 .port_id = 0,
1837                 .ip = 0,
1838         };
1839
1840         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1841                 p->table_id[0],
1842                 (struct rte_pipeline_table_entry *) &default_entry,
1843                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1844
1845         return rsp;
1846 }
1847
1848 void *
1849 pipeline_routing_msg_req_route_del_default_handler(struct pipeline *p,
1850         void *msg)
1851 {
1852         struct pipeline_routing_route_delete_default_msg_rsp *rsp = msg;
1853
1854         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1855                 p->table_id[0],
1856                 NULL);
1857
1858         return rsp;
1859 }
1860
1861 void *
1862 pipeline_routing_msg_req_arp_add_handler(struct pipeline *p, void *msg)
1863 {
1864         struct pipeline_routing_arp_add_msg_req *req = msg;
1865         struct pipeline_routing_arp_add_msg_rsp *rsp = msg;
1866
1867         struct pipeline_routing_arp_key_ipv4 key = {
1868                 .port_id = req->key.key.ipv4.port_id,
1869                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1870         };
1871
1872         struct arp_table_entry entry = {
1873                 .head = {
1874                         .action = RTE_PIPELINE_ACTION_PORT,
1875                         {.port_id = p->port_out_id[req->key.key.ipv4.port_id]},
1876                 },
1877
1878                 .macaddr = 0, /* set below */
1879         };
1880
1881         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1882                 rsp->status = -1;
1883                 return rsp;
1884         }
1885
1886         entry.macaddr = *((uint64_t *)&(req->macaddr));
1887         entry.macaddr = entry.macaddr << 16;
1888
1889         rsp->status = rte_pipeline_table_entry_add(p->p,
1890                 p->table_id[1],
1891                 &key,
1892                 (struct rte_pipeline_table_entry *) &entry,
1893                 &rsp->key_found,
1894                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1895
1896         return rsp;
1897 }
1898
1899 void *
1900 pipeline_routing_msg_req_arp_del_handler(struct pipeline *p, void *msg)
1901 {
1902         struct pipeline_routing_arp_delete_msg_req *req = msg;
1903         struct pipeline_routing_arp_delete_msg_rsp *rsp = msg;
1904
1905         struct pipeline_routing_arp_key_ipv4 key = {
1906                 .port_id = req->key.key.ipv4.port_id,
1907                 .ip = rte_bswap32(req->key.key.ipv4.ip),
1908         };
1909
1910         if (req->key.type != PIPELINE_ROUTING_ARP_IPV4) {
1911                 rsp->status = -1;
1912                 return rsp;
1913         }
1914
1915         rsp->status = rte_pipeline_table_entry_delete(p->p,
1916                 p->table_id[1],
1917                 &key,
1918                 &rsp->key_found,
1919                 NULL);
1920
1921         return rsp;
1922 }
1923
1924 void *
1925 pipeline_routing_msg_req_arp_add_default_handler(struct pipeline *p, void *msg)
1926 {
1927         struct pipeline_routing_arp_add_default_msg_req *req = msg;
1928         struct pipeline_routing_arp_add_default_msg_rsp *rsp = msg;
1929
1930         struct arp_table_entry default_entry = {
1931                 .head = {
1932                         .action = RTE_PIPELINE_ACTION_PORT,
1933                         {.port_id = p->port_out_id[req->port_id]},
1934                 },
1935
1936                 .macaddr = 0,
1937         };
1938
1939         rsp->status = rte_pipeline_table_default_entry_add(p->p,
1940                 p->table_id[1],
1941                 (struct rte_pipeline_table_entry *) &default_entry,
1942                 (struct rte_pipeline_table_entry **) &rsp->entry_ptr);
1943
1944         return rsp;
1945 }
1946
1947 void *
1948 pipeline_routing_msg_req_arp_del_default_handler(struct pipeline *p, void *msg)
1949 {
1950         struct pipeline_routing_arp_delete_default_msg_rsp *rsp = msg;
1951
1952         rsp->status = rte_pipeline_table_default_entry_delete(p->p,
1953                 p->table_id[1],
1954                 NULL);
1955
1956         return rsp;
1957 }
1958
1959 struct pipeline_be_ops pipeline_routing_be_ops = {
1960         .f_init = pipeline_routing_init,
1961         .f_free = pipeline_routing_free,
1962         .f_run = NULL,
1963         .f_timer = pipeline_routing_timer,
1964         .f_track = pipeline_routing_track,
1965 };