app/test-flow-perf/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020 Mellanox Technologies, Ltd
   3  *
   4  * This file contain the application main file
   5  * This application provides the user the ability to test the
   6  * insertion rate for specific rte_flow rule under stress state ~4M rule/
   7  *
   8  * Then it will also provide packet per second measurement after installing
   9  * all rules, the user may send traffic to test the PPS that match the rules
  10  * after all rules are installed, to check performance or functionality after
  11  * the stress.
  12  *
  13  * The flows insertion will go for all ports first, then it will print the
  14  * results, after that the application will go into forwarding packets mode
  15  * it will start receiving traffic if any and then forwarding it back and
  16  * gives packet per second measurement.
  17  */
  18
  19 #include <stdio.h>
  20 #include <stdlib.h>
  21 #include <string.h>
  22 #include <stdint.h>
  23 #include <inttypes.h>
  24 #include <stdarg.h>
  25 #include <errno.h>
  26 #include <getopt.h>
  27 #include <stdbool.h>
  28 #include <sys/time.h>
  29 #include <signal.h>
  30 #include <unistd.h>
  31
  32 #include <rte_malloc.h>
  33 #include <rte_mempool.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_ethdev.h>
  36 #include <rte_flow.h>
  37 #include <rte_mtr.h>
  38
  39 #include "config.h"
  40 #include "flow_gen.h"
  41
  42 #define MAX_BATCHES_COUNT          100
  43 #define DEFAULT_RULES_COUNT    4000000
  44 #define DEFAULT_RULES_BATCH     100000
  45 #define DEFAULT_GROUP                0
  46
  47 struct rte_flow *flow;
  48 static uint8_t flow_group;
  49
  50 static uint64_t encap_data;
  51 static uint64_t decap_data;
  52
  53 static uint64_t flow_items[MAX_ITEMS_NUM];
  54 static uint64_t flow_actions[MAX_ACTIONS_NUM];
  55 static uint64_t flow_attrs[MAX_ATTRS_NUM];
  56 static uint8_t items_idx, actions_idx, attrs_idx;
  57
  58 static uint64_t ports_mask;
  59 static uint16_t dst_ports[RTE_MAX_ETHPORTS];
  60 static volatile bool force_quit;
  61 static bool dump_iterations;
  62 static bool delete_flag;
  63 static bool dump_socket_mem_flag;
  64 static bool enable_fwd;
  65 static bool unique_data;
  66
  67 static uint8_t rx_queues_count;
  68 static uint8_t tx_queues_count;
  69 static uint8_t rxd_count;
  70 static uint8_t txd_count;
  71 static uint32_t mbuf_size;
  72 static uint32_t mbuf_cache_size;
  73 static uint32_t total_mbuf_num;
  74
  75 static struct rte_mempool *mbuf_mp;
  76 static uint32_t nb_lcores;
  77 static uint32_t rules_count;
  78 static uint32_t rules_batch;
  79 static uint32_t hairpin_queues_num; /* total hairpin q number - default: 0 */
  80 static uint32_t nb_lcores;
  81 static uint8_t max_priority;
  82 static uint32_t rand_seed;
  83
  84 #define MAX_PKT_BURST    32
  85 #define LCORE_MODE_PKT    1
  86 #define LCORE_MODE_STATS  2
  87 #define MAX_STREAMS      64
  88 #define METER_CREATE      1
  89 #define METER_DELETE      2
  90
  91 struct stream {
  92         int tx_port;
  93         int tx_queue;
  94         int rx_port;
  95         int rx_queue;
  96 };
  97
  98 struct lcore_info {
  99         int mode;
 100         int streams_nb;
 101         struct stream streams[MAX_STREAMS];
 102         /* stats */
 103         uint64_t tx_pkts;
 104         uint64_t tx_drops;
 105         uint64_t rx_pkts;
 106         struct rte_mbuf *pkts[MAX_PKT_BURST];
 107 } __rte_cache_aligned;
 108
 109 static struct lcore_info lcore_infos[RTE_MAX_LCORE];
 110
 111 struct used_cpu_time {
 112         double insertion[MAX_PORTS][RTE_MAX_LCORE];
 113         double deletion[MAX_PORTS][RTE_MAX_LCORE];
 114 };
 115
 116 struct multi_cores_pool {
 117         uint32_t cores_count;
 118         uint32_t rules_count;
 119         struct used_cpu_time meters_record;
 120         struct used_cpu_time flows_record;
 121         int64_t last_alloc[RTE_MAX_LCORE];
 122         int64_t current_alloc[RTE_MAX_LCORE];
 123 } __rte_cache_aligned;
 124
 125 static struct multi_cores_pool mc_pool = {
 126         .cores_count = 1,
 127 };
 128
 129 static const struct option_dict {
 130         const char *str;
 131         const uint64_t mask;
 132         uint64_t *map;
 133         uint8_t *map_idx;
 134
 135 } flow_options[] = {
 136         {
 137                 .str = "ether",
 138                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_ETH),
 139                 .map = &flow_items[0],
 140                 .map_idx = &items_idx
 141         },
 142         {
 143                 .str = "ipv4",
 144                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_IPV4),
 145                 .map = &flow_items[0],
 146                 .map_idx = &items_idx
 147         },
 148         {
 149                 .str = "ipv6",
 150                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_IPV6),
 151                 .map = &flow_items[0],
 152                 .map_idx = &items_idx
 153         },
 154         {
 155                 .str = "vlan",
 156                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_VLAN),
 157                 .map = &flow_items[0],
 158                 .map_idx = &items_idx
 159         },
 160         {
 161                 .str = "tcp",
 162                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_TCP),
 163                 .map = &flow_items[0],
 164                 .map_idx = &items_idx
 165         },
 166         {
 167                 .str = "udp",
 168                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_UDP),
 169                 .map = &flow_items[0],
 170                 .map_idx = &items_idx
 171         },
 172         {
 173                 .str = "vxlan",
 174                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_VXLAN),
 175                 .map = &flow_items[0],
 176                 .map_idx = &items_idx
 177         },
 178         {
 179                 .str = "vxlan-gpe",
 180                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_VXLAN_GPE),
 181                 .map = &flow_items[0],
 182                 .map_idx = &items_idx
 183         },
 184         {
 185                 .str = "gre",
 186                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_GRE),
 187                 .map = &flow_items[0],
 188                 .map_idx = &items_idx
 189         },
 190         {
 191                 .str = "geneve",
 192                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_GENEVE),
 193                 .map = &flow_items[0],
 194                 .map_idx = &items_idx
 195         },
 196         {
 197                 .str = "gtp",
 198                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_GTP),
 199                 .map = &flow_items[0],
 200                 .map_idx = &items_idx
 201         },
 202         {
 203                 .str = "meta",
 204                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_META),
 205                 .map = &flow_items[0],
 206                 .map_idx = &items_idx
 207         },
 208         {
 209                 .str = "tag",
 210                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_TAG),
 211                 .map = &flow_items[0],
 212                 .map_idx = &items_idx
 213         },
 214         {
 215                 .str = "icmpv4",
 216                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_ICMP),
 217                 .map = &flow_items[0],
 218                 .map_idx = &items_idx
 219         },
 220         {
 221                 .str = "icmpv6",
 222                 .mask = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_ICMP6),
 223                 .map = &flow_items[0],
 224                 .map_idx = &items_idx
 225         },
 226         {
 227                 .str = "ingress",
 228                 .mask = INGRESS,
 229                 .map = &flow_attrs[0],
 230                 .map_idx = &attrs_idx
 231         },
 232         {
 233                 .str = "egress",
 234                 .mask = EGRESS,
 235                 .map = &flow_attrs[0],
 236                 .map_idx = &attrs_idx
 237         },
 238         {
 239                 .str = "transfer",
 240                 .mask = TRANSFER,
 241                 .map = &flow_attrs[0],
 242                 .map_idx = &attrs_idx
 243         },
 244         {
 245                 .str = "port-id",
 246                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_PORT_ID),
 247                 .map = &flow_actions[0],
 248                 .map_idx = &actions_idx
 249         },
 250         {
 251                 .str = "rss",
 252                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_RSS),
 253                 .map = &flow_actions[0],
 254                 .map_idx = &actions_idx
 255         },
 256         {
 257                 .str = "queue",
 258                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_QUEUE),
 259                 .map = &flow_actions[0],
 260                 .map_idx = &actions_idx
 261         },
 262         {
 263                 .str = "jump",
 264                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_JUMP),
 265                 .map = &flow_actions[0],
 266                 .map_idx = &actions_idx
 267         },
 268         {
 269                 .str = "mark",
 270                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_MARK),
 271                 .map = &flow_actions[0],
 272                 .map_idx = &actions_idx
 273         },
 274         {
 275                 .str = "count",
 276                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_COUNT),
 277                 .map = &flow_actions[0],
 278                 .map_idx = &actions_idx
 279         },
 280         {
 281                 .str = "set-meta",
 282                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_SET_META),
 283                 .map = &flow_actions[0],
 284                 .map_idx = &actions_idx
 285         },
 286         {
 287                 .str = "set-tag",
 288                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_SET_TAG),
 289                 .map = &flow_actions[0],
 290                 .map_idx = &actions_idx
 291         },
 292         {
 293                 .str = "drop",
 294                 .mask = FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_DROP),
 295                 .map = &flow_actions[0],
 296                 .map_idx = &actions_idx
 297         },
 298         {
 299                 .str = "set-src-mac",
 300                 .mask = FLOW_ACTION_MASK(
 301                         RTE_FLOW_ACTION_TYPE_SET_MAC_SRC
 302                 ),
 303                 .map = &flow_actions[0],
 304                 .map_idx = &actions_idx
 305         },
 306         {
 307                 .str = "set-dst-mac",
 308                 .mask = FLOW_ACTION_MASK(
 309                         RTE_FLOW_ACTION_TYPE_SET_MAC_DST
 310                 ),
 311                 .map = &flow_actions[0],
 312                 .map_idx = &actions_idx
 313         },
 314         {
 315                 .str = "set-src-ipv4",
 316                 .mask = FLOW_ACTION_MASK(
 317                         RTE_FLOW_ACTION_TYPE_SET_IPV4_SRC
 318                 ),
 319                 .map = &flow_actions[0],
 320                 .map_idx = &actions_idx
 321         },
 322         {
 323                 .str = "set-dst-ipv4",
 324                 .mask = FLOW_ACTION_MASK(
 325                         RTE_FLOW_ACTION_TYPE_SET_IPV4_DST
 326                 ),
 327                 .map = &flow_actions[0],
 328                 .map_idx = &actions_idx
 329         },
 330         {
 331                 .str = "set-src-ipv6",
 332                 .mask = FLOW_ACTION_MASK(
 333                         RTE_FLOW_ACTION_TYPE_SET_IPV6_SRC
 334                 ),
 335                 .map = &flow_actions[0],
 336                 .map_idx = &actions_idx
 337         },
 338         {
 339                 .str = "set-dst-ipv6",
 340                 .mask = FLOW_ACTION_MASK(
 341                         RTE_FLOW_ACTION_TYPE_SET_IPV6_DST
 342                 ),
 343                 .map = &flow_actions[0],
 344                 .map_idx = &actions_idx
 345         },
 346         {
 347                 .str = "set-src-tp",
 348                 .mask = FLOW_ACTION_MASK(
 349                         RTE_FLOW_ACTION_TYPE_SET_TP_SRC
 350                 ),
 351                 .map = &flow_actions[0],
 352                 .map_idx = &actions_idx
 353         },
 354         {
 355                 .str = "set-dst-tp",
 356                 .mask = FLOW_ACTION_MASK(
 357                         RTE_FLOW_ACTION_TYPE_SET_TP_DST
 358                 ),
 359                 .map = &flow_actions[0],
 360                 .map_idx = &actions_idx
 361         },
 362         {
 363                 .str = "inc-tcp-ack",
 364                 .mask = FLOW_ACTION_MASK(
 365                         RTE_FLOW_ACTION_TYPE_INC_TCP_ACK
 366                 ),
 367                 .map = &flow_actions[0],
 368                 .map_idx = &actions_idx
 369         },
 370         {
 371                 .str = "dec-tcp-ack",
 372                 .mask = FLOW_ACTION_MASK(
 373                         RTE_FLOW_ACTION_TYPE_DEC_TCP_ACK
 374                 ),
 375                 .map = &flow_actions[0],
 376                 .map_idx = &actions_idx
 377         },
 378         {
 379                 .str = "inc-tcp-seq",
 380                 .mask = FLOW_ACTION_MASK(
 381                         RTE_FLOW_ACTION_TYPE_INC_TCP_SEQ
 382                 ),
 383                 .map = &flow_actions[0],
 384                 .map_idx = &actions_idx
 385         },
 386         {
 387                 .str = "dec-tcp-seq",
 388                 .mask = FLOW_ACTION_MASK(
 389                         RTE_FLOW_ACTION_TYPE_DEC_TCP_SEQ
 390                 ),
 391                 .map = &flow_actions[0],
 392                 .map_idx = &actions_idx
 393         },
 394         {
 395                 .str = "set-ttl",
 396                 .mask = FLOW_ACTION_MASK(
 397                         RTE_FLOW_ACTION_TYPE_SET_TTL
 398                 ),
 399                 .map = &flow_actions[0],
 400                 .map_idx = &actions_idx
 401         },
 402         {
 403                 .str = "dec-ttl",
 404                 .mask = FLOW_ACTION_MASK(
 405                         RTE_FLOW_ACTION_TYPE_DEC_TTL
 406                 ),
 407                 .map = &flow_actions[0],
 408                 .map_idx = &actions_idx
 409         },
 410         {
 411                 .str = "set-ipv4-dscp",
 412                 .mask = FLOW_ACTION_MASK(
 413                         RTE_FLOW_ACTION_TYPE_SET_IPV4_DSCP
 414                 ),
 415                 .map = &flow_actions[0],
 416                 .map_idx = &actions_idx
 417         },
 418         {
 419                 .str = "set-ipv6-dscp",
 420                 .mask = FLOW_ACTION_MASK(
 421                         RTE_FLOW_ACTION_TYPE_SET_IPV6_DSCP
 422                 ),
 423                 .map = &flow_actions[0],
 424                 .map_idx = &actions_idx
 425         },
 426         {
 427                 .str = "flag",
 428                 .mask = FLOW_ACTION_MASK(
 429                         RTE_FLOW_ACTION_TYPE_FLAG
 430                 ),
 431                 .map = &flow_actions[0],
 432                 .map_idx = &actions_idx
 433         },
 434         {
 435                 .str = "meter",
 436                 .mask = FLOW_ACTION_MASK(
 437                         RTE_FLOW_ACTION_TYPE_METER
 438                 ),
 439                 .map = &flow_actions[0],
 440                 .map_idx = &actions_idx
 441         },
 442         {
 443                 .str = "vxlan-encap",
 444                 .mask = FLOW_ACTION_MASK(
 445                         RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP
 446                 ),
 447                 .map = &flow_actions[0],
 448                 .map_idx = &actions_idx
 449         },
 450         {
 451                 .str = "vxlan-decap",
 452                 .mask = FLOW_ACTION_MASK(
 453                         RTE_FLOW_ACTION_TYPE_VXLAN_DECAP
 454                 ),
 455                 .map = &flow_actions[0],
 456                 .map_idx = &actions_idx
 457         },
 458 };
 459
 460 static void
 461 usage(char *progname)
 462 {
 463         printf("\nusage: %s\n", progname);
 464         printf("\nControl configurations:\n");
 465         printf("  --rules-count=N: to set the number of needed"
 466                 " rules to insert, default is %d\n", DEFAULT_RULES_COUNT);
 467         printf("  --rules-batch=N: set number of batched rules,"
 468                 " default is %d\n", DEFAULT_RULES_BATCH);
 469         printf("  --dump-iterations: To print rates for each"
 470                 " iteration\n");
 471         printf("  --deletion-rate: Enable deletion rate"
 472                 " calculations\n");
 473         printf("  --dump-socket-mem: To dump all socket memory\n");
 474         printf("  --enable-fwd: To enable packets forwarding"
 475                 " after insertion\n");
 476         printf("  --portmask=N: hexadecimal bitmask of ports used\n");
 477         printf("  --random-priority=N,S: use random priority levels "
 478                 "from 0 to (N - 1) for flows "
 479                 "and S as seed for pseudo-random number generator\n");
 480         printf("  --unique-data: flag to set using unique data for all"
 481                 " actions that support data, such as header modify and encap actions\n");
 482
 483         printf("To set flow attributes:\n");
 484         printf("  --ingress: set ingress attribute in flows\n");
 485         printf("  --egress: set egress attribute in flows\n");
 486         printf("  --transfer: set transfer attribute in flows\n");
 487         printf("  --group=N: set group for all flows,"
 488                 " default is %d\n", DEFAULT_GROUP);
 489         printf("  --cores=N: to set the number of needed "
 490                 "cores to insert rte_flow rules, default is 1\n");
 491         printf("  --rxq=N: to set the count of receive queues\n");
 492         printf("  --txq=N: to set the count of send queues\n");
 493         printf("  --rxd=N: to set the count of rxd\n");
 494         printf("  --txd=N: to set the count of txd\n");
 495         printf("  --mbuf-size=N: to set the size of mbuf\n");
 496         printf("  --mbuf-cache-size=N: to set the size of mbuf cache\n");
 497         printf("  --total-mbuf-count=N: to set the count of total mbuf count\n");
 498
 499
 500         printf("To set flow items:\n");
 501         printf("  --ether: add ether layer in flow items\n");
 502         printf("  --vlan: add vlan layer in flow items\n");
 503         printf("  --ipv4: add ipv4 layer in flow items\n");
 504         printf("  --ipv6: add ipv6 layer in flow items\n");
 505         printf("  --tcp: add tcp layer in flow items\n");
 506         printf("  --udp: add udp layer in flow items\n");
 507         printf("  --vxlan: add vxlan layer in flow items\n");
 508         printf("  --vxlan-gpe: add vxlan-gpe layer in flow items\n");
 509         printf("  --gre: add gre layer in flow items\n");
 510         printf("  --geneve: add geneve layer in flow items\n");
 511         printf("  --gtp: add gtp layer in flow items\n");
 512         printf("  --meta: add meta layer in flow items\n");
 513         printf("  --tag: add tag layer in flow items\n");
 514         printf("  --icmpv4: add icmpv4 layer in flow items\n");
 515         printf("  --icmpv6: add icmpv6 layer in flow items\n");
 516
 517         printf("To set flow actions:\n");
 518         printf("  --port-id: add port-id action in flow actions\n");
 519         printf("  --rss: add rss action in flow actions\n");
 520         printf("  --queue: add queue action in flow actions\n");
 521         printf("  --jump: add jump action in flow actions\n");
 522         printf("  --mark: add mark action in flow actions\n");
 523         printf("  --count: add count action in flow actions\n");
 524         printf("  --set-meta: add set meta action in flow actions\n");
 525         printf("  --set-tag: add set tag action in flow actions\n");
 526         printf("  --drop: add drop action in flow actions\n");
 527         printf("  --hairpin-queue=N: add hairpin-queue action in flow actions\n");
 528         printf("  --hairpin-rss=N: add hairpin-rss action in flow actions\n");
 529         printf("  --set-src-mac: add set src mac action to flow actions\n"
 530                 "Src mac to be set is random each flow\n");
 531         printf("  --set-dst-mac: add set dst mac action to flow actions\n"
 532                  "Dst mac to be set is random each flow\n");
 533         printf("  --set-src-ipv4: add set src ipv4 action to flow actions\n"
 534                 "Src ipv4 to be set is random each flow\n");
 535         printf("  --set-dst-ipv4 add set dst ipv4 action to flow actions\n"
 536                 "Dst ipv4 to be set is random each flow\n");
 537         printf("  --set-src-ipv6: add set src ipv6 action to flow actions\n"
 538                 "Src ipv6 to be set is random each flow\n");
 539         printf("  --set-dst-ipv6: add set dst ipv6 action to flow actions\n"
 540                 "Dst ipv6 to be set is random each flow\n");
 541         printf("  --set-src-tp: add set src tp action to flow actions\n"
 542                 "Src tp to be set is random each flow\n");
 543         printf("  --set-dst-tp: add set dst tp action to flow actions\n"
 544                 "Dst tp to be set is random each flow\n");
 545         printf("  --inc-tcp-ack: add inc tcp ack action to flow actions\n"
 546                 "tcp ack will be increments by 1\n");
 547         printf("  --dec-tcp-ack: add dec tcp ack action to flow actions\n"
 548                 "tcp ack will be decrements by 1\n");
 549         printf("  --inc-tcp-seq: add inc tcp seq action to flow actions\n"
 550                 "tcp seq will be increments by 1\n");
 551         printf("  --dec-tcp-seq: add dec tcp seq action to flow actions\n"
 552                 "tcp seq will be decrements by 1\n");
 553         printf("  --set-ttl: add set ttl action to flow actions\n"
 554                 "L3 ttl to be set is random each flow\n");
 555         printf("  --dec-ttl: add dec ttl action to flow actions\n"
 556                 "L3 ttl will be decrements by 1\n");
 557         printf("  --set-ipv4-dscp: add set ipv4 dscp action to flow actions\n"
 558                 "ipv4 dscp value to be set is random each flow\n");
 559         printf("  --set-ipv6-dscp: add set ipv6 dscp action to flow actions\n"
 560                 "ipv6 dscp value to be set is random each flow\n");
 561         printf("  --flag: add flag action to flow actions\n");
 562         printf("  --meter: add meter action to flow actions\n");
 563         printf("  --raw-encap=<data>: add raw encap action to flow actions\n"
 564                 "Data is the data needed to be encaped\n"
 565                 "Example: raw-encap=ether,ipv4,udp,vxlan\n");
 566         printf("  --raw-decap=<data>: add raw decap action to flow actions\n"
 567                 "Data is the data needed to be decaped\n"
 568                 "Example: raw-decap=ether,ipv4,udp,vxlan\n");
 569         printf("  --vxlan-encap: add vxlan-encap action to flow actions\n"
 570                 "Encapped data is fixed with pattern: ether,ipv4,udp,vxlan\n"
 571                 "With fixed values\n");
 572         printf("  --vxlan-decap: add vxlan_decap action to flow actions\n");
 573 }
 574
 575 static void
 576 args_parse(int argc, char **argv)
 577 {
 578         uint64_t pm, seed;
 579         char **argvopt;
 580         uint32_t prio;
 581         char *token;
 582         char *end;
 583         int n, opt;
 584         int opt_idx;
 585         size_t i;
 586
 587         static const struct option lgopts[] = {
 588                 /* Control */
 589                 { "help",                       0, 0, 0 },
 590                 { "rules-count",                1, 0, 0 },
 591                 { "rules-batch",                1, 0, 0 },
 592                 { "dump-iterations",            0, 0, 0 },
 593                 { "deletion-rate",              0, 0, 0 },
 594                 { "dump-socket-mem",            0, 0, 0 },
 595                 { "enable-fwd",                 0, 0, 0 },
 596                 { "unique-data",                0, 0, 0 },
 597                 { "portmask",                   1, 0, 0 },
 598                 { "cores",                      1, 0, 0 },
 599                 { "random-priority",            1, 0, 0 },
 600                 { "meter-profile-alg",          1, 0, 0 },
 601                 { "rxq",                        1, 0, 0 },
 602                 { "txq",                        1, 0, 0 },
 603                 { "rxd",                        1, 0, 0 },
 604                 { "txd",                        1, 0, 0 },
 605                 { "mbuf-size",                  1, 0, 0 },
 606                 { "mbuf-cache-size",            1, 0, 0 },
 607                 { "total-mbuf-count",           1, 0, 0 },
 608                 /* Attributes */
 609                 { "ingress",                    0, 0, 0 },
 610                 { "egress",                     0, 0, 0 },
 611                 { "transfer",                   0, 0, 0 },
 612                 { "group",                      1, 0, 0 },
 613                 /* Items */
 614                 { "ether",                      0, 0, 0 },
 615                 { "vlan",                       0, 0, 0 },
 616                 { "ipv4",                       0, 0, 0 },
 617                 { "ipv6",                       0, 0, 0 },
 618                 { "tcp",                        0, 0, 0 },
 619                 { "udp",                        0, 0, 0 },
 620                 { "vxlan",                      0, 0, 0 },
 621                 { "vxlan-gpe",                  0, 0, 0 },
 622                 { "gre",                        0, 0, 0 },
 623                 { "geneve",                     0, 0, 0 },
 624                 { "gtp",                        0, 0, 0 },
 625                 { "meta",                       0, 0, 0 },
 626                 { "tag",                        0, 0, 0 },
 627                 { "icmpv4",                     0, 0, 0 },
 628                 { "icmpv6",                     0, 0, 0 },
 629                 /* Actions */
 630                 { "port-id",                    2, 0, 0 },
 631                 { "rss",                        0, 0, 0 },
 632                 { "queue",                      0, 0, 0 },
 633                 { "jump",                       0, 0, 0 },
 634                 { "mark",                       0, 0, 0 },
 635                 { "count",                      0, 0, 0 },
 636                 { "set-meta",                   0, 0, 0 },
 637                 { "set-tag",                    0, 0, 0 },
 638                 { "drop",                       0, 0, 0 },
 639                 { "hairpin-queue",              1, 0, 0 },
 640                 { "hairpin-rss",                1, 0, 0 },
 641                 { "set-src-mac",                0, 0, 0 },
 642                 { "set-dst-mac",                0, 0, 0 },
 643                 { "set-src-ipv4",               0, 0, 0 },
 644                 { "set-dst-ipv4",               0, 0, 0 },
 645                 { "set-src-ipv6",               0, 0, 0 },
 646                 { "set-dst-ipv6",               0, 0, 0 },
 647                 { "set-src-tp",                 0, 0, 0 },
 648                 { "set-dst-tp",                 0, 0, 0 },
 649                 { "inc-tcp-ack",                0, 0, 0 },
 650                 { "dec-tcp-ack",                0, 0, 0 },
 651                 { "inc-tcp-seq",                0, 0, 0 },
 652                 { "dec-tcp-seq",                0, 0, 0 },
 653                 { "set-ttl",                    0, 0, 0 },
 654                 { "dec-ttl",                    0, 0, 0 },
 655                 { "set-ipv4-dscp",              0, 0, 0 },
 656                 { "set-ipv6-dscp",              0, 0, 0 },
 657                 { "flag",                       0, 0, 0 },
 658                 { "meter",                      0, 0, 0 },
 659                 { "raw-encap",                  1, 0, 0 },
 660                 { "raw-decap",                  1, 0, 0 },
 661                 { "vxlan-encap",                0, 0, 0 },
 662                 { "vxlan-decap",                0, 0, 0 },
 663         };
 664
 665         RTE_ETH_FOREACH_DEV(i)
 666                 ports_mask |= 1 << i;
 667
 668         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
 669                 dst_ports[i] = PORT_ID_DST;
 670
 671         hairpin_queues_num = 0;
 672         argvopt = argv;
 673
 674         printf(":: Flow -> ");
 675         while ((opt = getopt_long(argc, argvopt, "",
 676                                 lgopts, &opt_idx)) != EOF) {
 677                 switch (opt) {
 678                 case 0:
 679                         if (strcmp(lgopts[opt_idx].name, "help") == 0) {
 680                                 usage(argv[0]);
 681                                 exit(EXIT_SUCCESS);
 682                         }
 683
 684                         if (strcmp(lgopts[opt_idx].name, "group") == 0) {
 685                                 n = atoi(optarg);
 686                                 if (n >= 0)
 687                                         flow_group = n;
 688                                 else
 689                                         rte_exit(EXIT_FAILURE,
 690                                                 "flow group should be >= 0\n");
 691                                 printf("group %d / ", flow_group);
 692                         }
 693
 694                         for (i = 0; i < RTE_DIM(flow_options); i++)
 695                                 if (strcmp(lgopts[opt_idx].name,
 696                                                 flow_options[i].str) == 0) {
 697                                         flow_options[i].map[
 698                                         (*flow_options[i].map_idx)++] =
 699                                                 flow_options[i].mask;
 700                                         printf("%s / ", flow_options[i].str);
 701                                 }
 702
 703                         if (strcmp(lgopts[opt_idx].name,
 704                                         "hairpin-rss") == 0) {
 705                                 n = atoi(optarg);
 706                                 if (n > 0)
 707                                         hairpin_queues_num = n;
 708                                 else
 709                                         rte_exit(EXIT_FAILURE,
 710                                                 "Hairpin queues should be > 0\n");
 711
 712                                 flow_actions[actions_idx++] =
 713                                         HAIRPIN_RSS_ACTION;
 714                                 printf("hairpin-rss / ");
 715                         }
 716                         if (strcmp(lgopts[opt_idx].name,
 717                                         "hairpin-queue") == 0) {
 718                                 n = atoi(optarg);
 719                                 if (n > 0)
 720                                         hairpin_queues_num = n;
 721                                 else
 722                                         rte_exit(EXIT_FAILURE,
 723                                                 "Hairpin queues should be > 0\n");
 724
 725                                 flow_actions[actions_idx++] =
 726                                         HAIRPIN_QUEUE_ACTION;
 727                                 printf("hairpin-queue / ");
 728                         }
 729
 730                         if (strcmp(lgopts[opt_idx].name, "raw-encap") == 0) {
 731                                 printf("raw-encap ");
 732                                 flow_actions[actions_idx++] =
 733                                         FLOW_ITEM_MASK(
 734                                                 RTE_FLOW_ACTION_TYPE_RAW_ENCAP
 735                                         );
 736
 737                                 token = strtok(optarg, ",");
 738                                 while (token != NULL) {
 739                                         for (i = 0; i < RTE_DIM(flow_options); i++) {
 740                                                 if (strcmp(flow_options[i].str, token) == 0) {
 741                                                         printf("%s,", token);
 742                                                         encap_data |= flow_options[i].mask;
 743                                                         break;
 744                                                 }
 745                                                 /* Reached last item with no match */
 746                                                 if (i == (RTE_DIM(flow_options) - 1))
 747                                                         rte_exit(EXIT_FAILURE,
 748                                                                 "Invalid encap item: %s\n", token);
 749                                         }
 750                                         token = strtok(NULL, ",");
 751                                 }
 752                                 printf(" / ");
 753                         }
 754                         if (strcmp(lgopts[opt_idx].name, "raw-decap") == 0) {
 755                                 printf("raw-decap ");
 756                                 flow_actions[actions_idx++] =
 757                                         FLOW_ITEM_MASK(
 758                                                 RTE_FLOW_ACTION_TYPE_RAW_DECAP
 759                                         );
 760
 761                                 token = strtok(optarg, ",");
 762                                 while (token != NULL) {
 763                                         for (i = 0; i < RTE_DIM(flow_options); i++) {
 764                                                 if (strcmp(flow_options[i].str, token) == 0) {
 765                                                         printf("%s,", token);
 766                                                         decap_data |= flow_options[i].mask;
 767                                                         break;
 768                                                 }
 769                                                 /* Reached last item with no match */
 770                                                 if (i == (RTE_DIM(flow_options) - 1))
 771                                                         rte_exit(EXIT_FAILURE,
 772                                                                 "Invalid decap item %s\n", token);
 773                                         }
 774                                         token = strtok(NULL, ",");
 775                                 }
 776                                 printf(" / ");
 777                         }
 778                         /* Control */
 779                         if (strcmp(lgopts[opt_idx].name,
 780                                         "rules-batch") == 0) {
 781                                 rules_batch = atoi(optarg);
 782                         }
 783                         if (strcmp(lgopts[opt_idx].name,
 784                                         "rules-count") == 0) {
 785                                 rules_count = atoi(optarg);
 786                         }
 787                         if (strcmp(lgopts[opt_idx].name, "random-priority") ==
 788                             0) {
 789                                 end = NULL;
 790                                 prio = strtol(optarg, &end, 10);
 791                                 if ((optarg[0] == '\0') || (end == NULL))
 792                                         rte_exit(EXIT_FAILURE,
 793                                                  "Invalid value for random-priority\n");
 794                                 max_priority = prio;
 795                                 token = end + 1;
 796                                 seed = strtoll(token, &end, 10);
 797                                 if ((token[0] == '\0') || (*end != '\0'))
 798                                         rte_exit(EXIT_FAILURE,
 799                                                  "Invalid value for random-priority\n");
 800                                 rand_seed = seed;
 801                         }
 802                         if (strcmp(lgopts[opt_idx].name,
 803                                         "dump-iterations") == 0)
 804                                 dump_iterations = true;
 805                         if (strcmp(lgopts[opt_idx].name,
 806                                         "unique-data") == 0)
 807                                 unique_data = true;
 808                         if (strcmp(lgopts[opt_idx].name,
 809                                         "deletion-rate") == 0)
 810                                 delete_flag = true;
 811                         if (strcmp(lgopts[opt_idx].name,
 812                                         "dump-socket-mem") == 0)
 813                                 dump_socket_mem_flag = true;
 814                         if (strcmp(lgopts[opt_idx].name,
 815                                         "enable-fwd") == 0)
 816                                 enable_fwd = true;
 817                         if (strcmp(lgopts[opt_idx].name,
 818                                         "portmask") == 0) {
 819                                 /* parse hexadecimal string */
 820                                 end = NULL;
 821                                 pm = strtoull(optarg, &end, 16);
 822                                 if ((optarg[0] == '\0') || (end == NULL) || (*end != '\0'))
 823                                         rte_exit(EXIT_FAILURE, "Invalid fwd port mask\n");
 824                                 ports_mask = pm;
 825                         }
 826                         if (strcmp(lgopts[opt_idx].name,
 827                                         "port-id") == 0) {
 828                                 uint16_t port_idx = 0;
 829                                 char *token;
 830
 831                                 token = strtok(optarg, ",");
 832                                 while (token != NULL) {
 833                                         dst_ports[port_idx++] = atoi(token);
 834                                         token = strtok(NULL, ",");
 835                                 }
 836                         }
 837                         if (strcmp(lgopts[opt_idx].name, "rxq") == 0) {
 838                                 n = atoi(optarg);
 839                                 rx_queues_count = (uint8_t) n;
 840                         }
 841                         if (strcmp(lgopts[opt_idx].name, "txq") == 0) {
 842                                 n = atoi(optarg);
 843                                 tx_queues_count = (uint8_t) n;
 844                         }
 845                         if (strcmp(lgopts[opt_idx].name, "rxd") == 0) {
 846                                 n = atoi(optarg);
 847                                 rxd_count = (uint8_t) n;
 848                         }
 849                         if (strcmp(lgopts[opt_idx].name, "txd") == 0) {
 850                                 n = atoi(optarg);
 851                                 txd_count = (uint8_t) n;
 852                         }
 853                         if (strcmp(lgopts[opt_idx].name, "mbuf-size") == 0) {
 854                                 n = atoi(optarg);
 855                                 mbuf_size = (uint32_t) n;
 856                         }
 857                         if (strcmp(lgopts[opt_idx].name, "mbuf-cache-size") == 0) {
 858                                 n = atoi(optarg);
 859                                 mbuf_cache_size = (uint32_t) n;
 860                         }
 861                         if (strcmp(lgopts[opt_idx].name, "total-mbuf-count") == 0) {
 862                                 n = atoi(optarg);
 863                                 total_mbuf_num = (uint32_t) n;
 864                         }
 865                         if (strcmp(lgopts[opt_idx].name, "cores") == 0) {
 866                                 n = atoi(optarg);
 867                                 if ((int) rte_lcore_count() <= n) {
 868                                         rte_exit(EXIT_FAILURE,
 869                                                 "Error: you need %d cores to run on multi-cores\n"
 870                                                 "Existing cores are: %d\n", n, rte_lcore_count());
 871                                 }
 872                                 if (n <= RTE_MAX_LCORE && n > 0)
 873                                         mc_pool.cores_count = n;
 874                                 else {
 875                                         rte_exit(EXIT_FAILURE,
 876                                                 "Error: cores count must be > 0 and < %d\n",
 877                                                 RTE_MAX_LCORE);
 878                                 }
 879                         }
 880                         break;
 881                 default:
 882                         usage(argv[0]);
 883                         rte_exit(EXIT_FAILURE, "Invalid option: %s\n",
 884                                         argv[optind]);
 885                         break;
 886                 }
 887         }
 888         if (rules_count % rules_batch != 0) {
 889                 rte_exit(EXIT_FAILURE,
 890                          "rules_count %% rules_batch should be 0\n");
 891         }
 892         if (rules_count / rules_batch > MAX_BATCHES_COUNT) {
 893                 rte_exit(EXIT_FAILURE,
 894                          "rules_count / rules_batch should be <= %d\n",
 895                          MAX_BATCHES_COUNT);
 896         }
 897
 898         printf("end_flow\n");
 899 }
 900
 901 /* Dump the socket memory statistics on console */
 902 static size_t
 903 dump_socket_mem(FILE *f)
 904 {
 905         struct rte_malloc_socket_stats socket_stats;
 906         unsigned int i = 0;
 907         size_t total = 0;
 908         size_t alloc = 0;
 909         size_t free = 0;
 910         unsigned int n_alloc = 0;
 911         unsigned int n_free = 0;
 912         bool active_nodes = false;
 913
 914
 915         for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
 916                 if (rte_malloc_get_socket_stats(i, &socket_stats) ||
 917                     !socket_stats.heap_totalsz_bytes)
 918                         continue;
 919                 active_nodes = true;
 920                 total += socket_stats.heap_totalsz_bytes;
 921                 alloc += socket_stats.heap_allocsz_bytes;
 922                 free += socket_stats.heap_freesz_bytes;
 923                 n_alloc += socket_stats.alloc_count;
 924                 n_free += socket_stats.free_count;
 925                 if (dump_socket_mem_flag) {
 926                         fprintf(f, "::::::::::::::::::::::::::::::::::::::::");
 927                         fprintf(f,
 928                                 "\nSocket %u:\nsize(M) total: %.6lf\nalloc:"
 929                                 " %.6lf(%.3lf%%)\nfree: %.6lf"
 930                                 "\nmax: %.6lf"
 931                                 "\ncount alloc: %u\nfree: %u\n",
 932                                 i,
 933                                 socket_stats.heap_totalsz_bytes / 1.0e6,
 934                                 socket_stats.heap_allocsz_bytes / 1.0e6,
 935                                 (double)socket_stats.heap_allocsz_bytes * 100 /
 936                                 (double)socket_stats.heap_totalsz_bytes,
 937                                 socket_stats.heap_freesz_bytes / 1.0e6,
 938                                 socket_stats.greatest_free_size / 1.0e6,
 939                                 socket_stats.alloc_count,
 940                                 socket_stats.free_count);
 941                                 fprintf(f, "::::::::::::::::::::::::::::::::::::::::");
 942                 }
 943         }
 944         if (dump_socket_mem_flag && active_nodes) {
 945                 fprintf(f,
 946                         "\nTotal: size(M)\ntotal: %.6lf"
 947                         "\nalloc: %.6lf(%.3lf%%)\nfree: %.6lf"
 948                         "\ncount alloc: %u\nfree: %u\n",
 949                         total / 1.0e6, alloc / 1.0e6,
 950                         (double)alloc * 100 / (double)total, free / 1.0e6,
 951                         n_alloc, n_free);
 952                 fprintf(f, "::::::::::::::::::::::::::::::::::::::::\n");
 953         }
 954         return alloc;
 955 }
 956
 957 static void
 958 print_flow_error(struct rte_flow_error error)
 959 {
 960         printf("Flow can't be created %d message: %s\n",
 961                 error.type,
 962                 error.message ? error.message : "(no stated reason)");
 963 }
 964
 965 static inline void
 966 print_rules_batches(double *cpu_time_per_batch)
 967 {
 968         uint8_t idx;
 969         double delta;
 970         double rate;
 971
 972         for (idx = 0; idx < MAX_BATCHES_COUNT; idx++) {
 973                 if (!cpu_time_per_batch[idx])
 974                         break;
 975                 delta = (double)(rules_batch / cpu_time_per_batch[idx]);
 976                 rate = delta / 1000; /* Save rate in K unit. */
 977                 printf(":: Rules batch #%d: %d rules "
 978                         "in %f sec[ Rate = %f K Rule/Sec ]\n",
 979                         idx, rules_batch,
 980                         cpu_time_per_batch[idx], rate);
 981         }
 982 }
 983
 984
 985 static inline int
 986 has_meter(void)
 987 {
 988         int i;
 989
 990         for (i = 0; i < MAX_ACTIONS_NUM; i++) {
 991                 if (flow_actions[i] == 0)
 992                         break;
 993                 if (flow_actions[i]
 994                                 & FLOW_ACTION_MASK(RTE_FLOW_ACTION_TYPE_METER))
 995                         return 1;
 996         }
 997         return 0;
 998 }
 999
1000 static void
1001 create_meter_rule(int port_id, uint32_t counter)
1002 {
1003         int ret;
1004         struct rte_mtr_params params;
1005         uint32_t default_prof_id = 100;
1006         struct rte_mtr_error error;
1007
1008         memset(&params, 0, sizeof(struct rte_mtr_params));
1009         params.meter_enable = 1;
1010         params.stats_mask = 0xffff;
1011         params.use_prev_mtr_color = 0;
1012         params.dscp_table = NULL;
1013
1014         /*create meter*/
1015         params.meter_profile_id = default_prof_id;
1016         ret = rte_mtr_create(port_id, counter, &params, 1, &error);
1017         if (ret != 0) {
1018                 printf("Port %u create meter idx(%d) error(%d) message: %s\n",
1019                         port_id, counter, error.type,
1020                         error.message ? error.message : "(no stated reason)");
1021                 rte_exit(EXIT_FAILURE, "Error in creating meter\n");
1022         }
1023 }
1024
1025 static void
1026 destroy_meter_rule(int port_id, uint32_t counter)
1027 {
1028         struct rte_mtr_error error;
1029
1030         if (rte_mtr_destroy(port_id, counter, &error)) {
1031                 printf("Port %u destroy meter(%d) error(%d) message: %s\n",
1032                         port_id, counter, error.type,
1033                         error.message ? error.message : "(no stated reason)");
1034                 rte_exit(EXIT_FAILURE, "Error in deleting meter rule\n");
1035         }
1036 }
1037
1038 static void
1039 meters_handler(int port_id, uint8_t core_id, uint8_t ops)
1040 {
1041         uint64_t start_batch;
1042         double cpu_time_used, insertion_rate;
1043         int rules_count_per_core, rules_batch_idx;
1044         uint32_t counter, start_counter = 0, end_counter;
1045         double cpu_time_per_batch[MAX_BATCHES_COUNT] = { 0 };
1046
1047         rules_count_per_core = rules_count / mc_pool.cores_count;
1048
1049         if (core_id)
1050                 start_counter = core_id * rules_count_per_core;
1051         end_counter = (core_id + 1) * rules_count_per_core;
1052
1053         cpu_time_used = 0;
1054         start_batch = rte_get_timer_cycles();
1055         for (counter = start_counter; counter < end_counter; counter++) {
1056                 if (ops == METER_CREATE)
1057                         create_meter_rule(port_id, counter);
1058                 else
1059                         destroy_meter_rule(port_id, counter);
1060                 /*
1061                  * Save the insertion rate for rules batch.
1062                  * Check if the insertion reached the rules
1063                  * patch counter, then save the insertion rate
1064                  * for this batch.
1065                  */
1066                 if (!((counter + 1) % rules_batch)) {
1067                         rules_batch_idx = ((counter + 1) / rules_batch) - 1;
1068                         cpu_time_per_batch[rules_batch_idx] =
1069                                 ((double)(rte_get_timer_cycles() - start_batch))
1070                                 / rte_get_timer_hz();
1071                         cpu_time_used += cpu_time_per_batch[rules_batch_idx];
1072                         start_batch = rte_get_timer_cycles();
1073                 }
1074         }
1075
1076         /* Print insertion rates for all batches */
1077         if (dump_iterations)
1078                 print_rules_batches(cpu_time_per_batch);
1079
1080         insertion_rate =
1081                 ((double) (rules_count_per_core / cpu_time_used) / 1000);
1082
1083         /* Insertion rate for all rules in one core */
1084         printf(":: Port %d :: Core %d Meter %s :: start @[%d] - end @[%d],"
1085                 " use:%.02fs, rate:%.02fk Rule/Sec\n",
1086                 port_id, core_id, ops == METER_CREATE ? "create" : "delete",
1087                 start_counter, end_counter - 1,
1088                 cpu_time_used, insertion_rate);
1089
1090         if (ops == METER_CREATE)
1091                 mc_pool.meters_record.insertion[port_id][core_id]
1092                         = cpu_time_used;
1093         else
1094                 mc_pool.meters_record.deletion[port_id][core_id]
1095                         = cpu_time_used;
1096 }
1097
1098 static void
1099 destroy_meter_profile(void)
1100 {
1101         struct rte_mtr_error error;
1102         uint16_t nr_ports;
1103         int port_id;
1104
1105         nr_ports = rte_eth_dev_count_avail();
1106         for (port_id = 0; port_id < nr_ports; port_id++) {
1107                 /* If port outside portmask */
1108                 if (!((ports_mask >> port_id) & 0x1))
1109                         continue;
1110
1111                 if (rte_mtr_meter_profile_delete
1112                         (port_id, DEFAULT_METER_PROF_ID, &error)) {
1113                         printf("Port %u del profile error(%d) message: %s\n",
1114                                 port_id, error.type,
1115                                 error.message ? error.message : "(no stated reason)");
1116                         rte_exit(EXIT_FAILURE, "Error: Destroy meter profile Failed!\n");
1117                 }
1118         }
1119 }
1120
1121 static void
1122 create_meter_profile(void)
1123 {
1124         uint16_t nr_ports;
1125         int ret, port_id;
1126         struct rte_mtr_meter_profile mp;
1127         struct rte_mtr_error error;
1128
1129         /*
1130          *currently , only create one meter file for one port
1131          *1 meter profile -> N meter rules -> N rte flows
1132          */
1133         memset(&mp, 0, sizeof(struct rte_mtr_meter_profile));
1134         nr_ports = rte_eth_dev_count_avail();
1135         for (port_id = 0; port_id < nr_ports; port_id++) {
1136                 /* If port outside portmask */
1137                 if (!((ports_mask >> port_id) & 0x1))
1138                         continue;
1139
1140                 mp.alg = RTE_MTR_SRTCM_RFC2697;
1141                 mp.srtcm_rfc2697.cir = METER_CIR;
1142                 mp.srtcm_rfc2697.cbs = METER_CIR / 8;
1143                 mp.srtcm_rfc2697.ebs = 0;
1144
1145                 ret = rte_mtr_meter_profile_add
1146                         (port_id, DEFAULT_METER_PROF_ID, &mp, &error);
1147                 if (ret != 0) {
1148                         printf("Port %u create Profile error(%d) message: %s\n",
1149                                 port_id, error.type,
1150                                 error.message ? error.message : "(no stated reason)");
1151                         rte_exit(EXIT_FAILURE, "Error: Creation meter profile Failed!\n");
1152                 }
1153         }
1154 }
1155
1156 static inline void
1157 destroy_flows(int port_id, uint8_t core_id, struct rte_flow **flows_list)
1158 {
1159         struct rte_flow_error error;
1160         clock_t start_batch, end_batch;
1161         double cpu_time_used = 0;
1162         double deletion_rate;
1163         double cpu_time_per_batch[MAX_BATCHES_COUNT] = { 0 };
1164         double delta;
1165         uint32_t i;
1166         int rules_batch_idx;
1167         int rules_count_per_core;
1168
1169         rules_count_per_core = rules_count / mc_pool.cores_count;
1170         /* If group > 0 , should add 1 flow which created in group 0 */
1171         if (flow_group > 0 && core_id == 0)
1172                 rules_count_per_core++;
1173
1174         start_batch = rte_get_timer_cycles();
1175         for (i = 0; i < (uint32_t) rules_count_per_core; i++) {
1176                 if (flows_list[i] == 0)
1177                         break;
1178
1179                 memset(&error, 0x33, sizeof(error));
1180                 if (rte_flow_destroy(port_id, flows_list[i], &error)) {
1181                         print_flow_error(error);
1182                         rte_exit(EXIT_FAILURE, "Error in deleting flow\n");
1183                 }
1184
1185                 /*
1186                  * Save the deletion rate for rules batch.
1187                  * Check if the deletion reached the rules
1188                  * patch counter, then save the deletion rate
1189                  * for this batch.
1190                  */
1191                 if (!((i + 1) % rules_batch)) {
1192                         end_batch = rte_get_timer_cycles();
1193                         delta = (double) (end_batch - start_batch);
1194                         rules_batch_idx = ((i + 1) / rules_batch) - 1;
1195                         cpu_time_per_batch[rules_batch_idx] = delta / rte_get_timer_hz();
1196                         cpu_time_used += cpu_time_per_batch[rules_batch_idx];
1197                         start_batch = rte_get_timer_cycles();
1198                 }
1199         }
1200
1201         /* Print deletion rates for all batches */
1202         if (dump_iterations)
1203                 print_rules_batches(cpu_time_per_batch);
1204
1205         /* Deletion rate for all rules */
1206         deletion_rate = ((double) (rules_count_per_core / cpu_time_used) / 1000);
1207         printf(":: Port %d :: Core %d :: Rules deletion rate -> %f K Rule/Sec\n",
1208                 port_id, core_id, deletion_rate);
1209         printf(":: Port %d :: Core %d :: The time for deleting %d rules is %f seconds\n",
1210                 port_id, core_id, rules_count_per_core, cpu_time_used);
1211
1212         mc_pool.flows_record.deletion[port_id][core_id] = cpu_time_used;
1213 }
1214
1215 static struct rte_flow **
1216 insert_flows(int port_id, uint8_t core_id, uint16_t dst_port_id)
1217 {
1218         struct rte_flow **flows_list;
1219         struct rte_flow_error error;
1220         clock_t start_batch, end_batch;
1221         double first_flow_latency;
1222         double cpu_time_used;
1223         double insertion_rate;
1224         double cpu_time_per_batch[MAX_BATCHES_COUNT] = { 0 };
1225         double delta;
1226         uint32_t flow_index;
1227         uint32_t counter, start_counter = 0, end_counter;
1228         uint64_t global_items[MAX_ITEMS_NUM] = { 0 };
1229         uint64_t global_actions[MAX_ACTIONS_NUM] = { 0 };
1230         int rules_batch_idx;
1231         int rules_count_per_core;
1232
1233         rules_count_per_core = rules_count / mc_pool.cores_count;
1234
1235         /* Set boundaries of rules for each core. */
1236         if (core_id)
1237                 start_counter = core_id * rules_count_per_core;
1238         end_counter = (core_id + 1) * rules_count_per_core;
1239
1240         global_items[0] = FLOW_ITEM_MASK(RTE_FLOW_ITEM_TYPE_ETH);
1241         global_actions[0] = FLOW_ITEM_MASK(RTE_FLOW_ACTION_TYPE_JUMP);
1242
1243         flows_list = rte_zmalloc("flows_list",
1244                 (sizeof(struct rte_flow *) * rules_count_per_core) + 1, 0);
1245         if (flows_list == NULL)
1246                 rte_exit(EXIT_FAILURE, "No Memory available!\n");
1247
1248         cpu_time_used = 0;
1249         flow_index = 0;
1250         if (flow_group > 0 && core_id == 0) {
1251                 /*
1252                  * Create global rule to jump into flow_group,
1253                  * this way the app will avoid the default rules.
1254                  *
1255                  * This rule will be created only once.
1256                  *
1257                  * Global rule:
1258                  * group 0 eth / end actions jump group <flow_group>
1259                  */
1260                 flow = generate_flow(port_id, 0, flow_attrs,
1261                         global_items, global_actions,
1262                         flow_group, 0, 0, 0, 0, dst_port_id, core_id,
1263                         rx_queues_count, unique_data, max_priority, &error);
1264
1265                 if (flow == NULL) {
1266                         print_flow_error(error);
1267                         rte_exit(EXIT_FAILURE, "Error in creating flow\n");
1268                 }
1269                 flows_list[flow_index++] = flow;
1270         }
1271
1272         start_batch = rte_get_timer_cycles();
1273         for (counter = start_counter; counter < end_counter; counter++) {
1274                 flow = generate_flow(port_id, flow_group,
1275                         flow_attrs, flow_items, flow_actions,
1276                         JUMP_ACTION_TABLE, counter,
1277                         hairpin_queues_num, encap_data,
1278                         decap_data, dst_port_id,
1279                         core_id, rx_queues_count,
1280                         unique_data, max_priority, &error);
1281
1282                 if (!counter) {
1283                         first_flow_latency = (double) (rte_get_timer_cycles() - start_batch);
1284                         first_flow_latency /= rte_get_timer_hz();
1285                         /* In millisecond */
1286                         first_flow_latency *= 1000;
1287                         printf(":: First Flow Latency :: Port %d :: First flow "
1288                                 "installed in %f milliseconds\n",
1289                                 port_id, first_flow_latency);
1290                 }
1291
1292                 if (force_quit)
1293                         counter = end_counter;
1294
1295                 if (!flow) {
1296                         print_flow_error(error);
1297                         rte_exit(EXIT_FAILURE, "Error in creating flow\n");
1298                 }
1299
1300                 flows_list[flow_index++] = flow;
1301
1302                 /*
1303                  * Save the insertion rate for rules batch.
1304                  * Check if the insertion reached the rules
1305                  * patch counter, then save the insertion rate
1306                  * for this batch.
1307                  */
1308                 if (!((counter + 1) % rules_batch)) {
1309                         end_batch = rte_get_timer_cycles();
1310                         delta = (double) (end_batch - start_batch);
1311                         rules_batch_idx = ((counter + 1) / rules_batch) - 1;
1312                         cpu_time_per_batch[rules_batch_idx] = delta / rte_get_timer_hz();
1313                         cpu_time_used += cpu_time_per_batch[rules_batch_idx];
1314                         start_batch = rte_get_timer_cycles();
1315                 }
1316         }
1317
1318         /* Print insertion rates for all batches */
1319         if (dump_iterations)
1320                 print_rules_batches(cpu_time_per_batch);
1321
1322         printf(":: Port %d :: Core %d boundaries :: start @[%d] - end @[%d]\n",
1323                 port_id, core_id, start_counter, end_counter - 1);
1324
1325         /* Insertion rate for all rules in one core */
1326         insertion_rate = ((double) (rules_count_per_core / cpu_time_used) / 1000);
1327         printf(":: Port %d :: Core %d :: Rules insertion rate -> %f K Rule/Sec\n",
1328                 port_id, core_id, insertion_rate);
1329         printf(":: Port %d :: Core %d :: The time for creating %d in rules %f seconds\n",
1330                 port_id, core_id, rules_count_per_core, cpu_time_used);
1331
1332         mc_pool.flows_record.insertion[port_id][core_id] = cpu_time_used;
1333         return flows_list;
1334 }
1335
1336 static void
1337 flows_handler(uint8_t core_id)
1338 {
1339         struct rte_flow **flows_list;
1340         uint16_t port_idx = 0;
1341         uint16_t nr_ports;
1342         int port_id;
1343
1344         nr_ports = rte_eth_dev_count_avail();
1345
1346         if (rules_batch > rules_count)
1347                 rules_batch = rules_count;
1348
1349         printf(":: Rules Count per port: %d\n\n", rules_count);
1350
1351         for (port_id = 0; port_id < nr_ports; port_id++) {
1352                 /* If port outside portmask */
1353                 if (!((ports_mask >> port_id) & 0x1))
1354                         continue;
1355
1356                 /* Insertion part. */
1357                 mc_pool.last_alloc[core_id] = (int64_t)dump_socket_mem(stdout);
1358                 if (has_meter())
1359                         meters_handler(port_id, core_id, METER_CREATE);
1360                 flows_list = insert_flows(port_id, core_id,
1361                                                 dst_ports[port_idx++]);
1362                 if (flows_list == NULL)
1363                         rte_exit(EXIT_FAILURE, "Error: Insertion Failed!\n");
1364                 mc_pool.current_alloc[core_id] = (int64_t)dump_socket_mem(stdout);
1365
1366                 /* Deletion part. */
1367                 if (delete_flag) {
1368                         destroy_flows(port_id, core_id, flows_list);
1369                         if (has_meter())
1370                                 meters_handler(port_id, core_id, METER_DELETE);
1371                 }
1372         }
1373 }
1374
1375 static void
1376 dump_used_cpu_time(const char *item,
1377                 uint16_t port, struct used_cpu_time *used_time)
1378 {
1379         uint32_t i;
1380         /* Latency: total count of rte rules divided
1381          * over max time used by thread between all
1382          * threads time.
1383          *
1384          * Throughput: total count of rte rules divided
1385          * over the average of the time cosumed by all
1386          * threads time.
1387          */
1388         double insertion_latency_time;
1389         double insertion_throughput_time;
1390         double deletion_latency_time;
1391         double deletion_throughput_time;
1392         double insertion_latency, insertion_throughput;
1393         double deletion_latency, deletion_throughput;
1394
1395         /* Save first insertion/deletion rates from first thread.
1396          * Start comparing with all threads, if any thread used
1397          * time more than current saved, replace it.
1398          *
1399          * Thus in the end we will have the max time used for
1400          * insertion/deletion by one thread.
1401          *
1402          * As for memory consumption, save the min of all threads
1403          * of last alloc, and save the max for all threads for
1404          * current alloc.
1405          */
1406
1407         insertion_latency_time = used_time->insertion[port][0];
1408         deletion_latency_time = used_time->deletion[port][0];
1409         insertion_throughput_time = used_time->insertion[port][0];
1410         deletion_throughput_time = used_time->deletion[port][0];
1411
1412         i = mc_pool.cores_count;
1413         while (i-- > 1) {
1414                 insertion_throughput_time += used_time->insertion[port][i];
1415                 deletion_throughput_time += used_time->deletion[port][i];
1416                 if (insertion_latency_time < used_time->insertion[port][i])
1417                         insertion_latency_time = used_time->insertion[port][i];
1418                 if (deletion_latency_time < used_time->deletion[port][i])
1419                         deletion_latency_time = used_time->deletion[port][i];
1420         }
1421
1422         insertion_latency = ((double) (mc_pool.rules_count
1423                                 / insertion_latency_time) / 1000);
1424         deletion_latency = ((double) (mc_pool.rules_count
1425                                 / deletion_latency_time) / 1000);
1426
1427         insertion_throughput_time /= mc_pool.cores_count;
1428         deletion_throughput_time /= mc_pool.cores_count;
1429         insertion_throughput = ((double) (mc_pool.rules_count
1430                                 / insertion_throughput_time) / 1000);
1431         deletion_throughput = ((double) (mc_pool.rules_count
1432                                 / deletion_throughput_time) / 1000);
1433
1434         /* Latency stats */
1435         printf("\n%s\n:: [Latency | Insertion] All Cores :: Port %d :: ",
1436                 item, port);
1437         printf("Total flows insertion rate -> %f K Rules/Sec\n",
1438                 insertion_latency);
1439         printf(":: [Latency | Insertion] All Cores :: Port %d :: ", port);
1440         printf("The time for creating %d rules is %f seconds\n",
1441                 mc_pool.rules_count, insertion_latency_time);
1442
1443         /* Throughput stats */
1444         printf(":: [Throughput | Insertion] All Cores :: Port %d :: ", port);
1445         printf("Total flows insertion rate -> %f K Rules/Sec\n",
1446                 insertion_throughput);
1447         printf(":: [Throughput | Insertion] All Cores :: Port %d :: ", port);
1448         printf("The average time for creating %d rules is %f seconds\n",
1449                 mc_pool.rules_count, insertion_throughput_time);
1450
1451         if (delete_flag) {
1452         /* Latency stats */
1453                 printf(":: [Latency | Deletion] All Cores :: Port %d :: Total "
1454                         "deletion rate -> %f K Rules/Sec\n",
1455                         port, deletion_latency);
1456                 printf(":: [Latency | Deletion] All Cores :: Port %d :: ",
1457                         port);
1458                 printf("The time for deleting %d rules is %f seconds\n",
1459                         mc_pool.rules_count, deletion_latency_time);
1460
1461                 /* Throughput stats */
1462                 printf(":: [Throughput | Deletion] All Cores :: Port %d :: Total "
1463                         "deletion rate -> %f K Rules/Sec\n",
1464                         port, deletion_throughput);
1465                 printf(":: [Throughput | Deletion] All Cores :: Port %d :: ",
1466                         port);
1467                 printf("The average time for deleting %d rules is %f seconds\n",
1468                         mc_pool.rules_count, deletion_throughput_time);
1469         }
1470 }
1471
1472 static void
1473 dump_used_mem(uint16_t port)
1474 {
1475         uint32_t i;
1476         int64_t last_alloc, current_alloc;
1477         int flow_size_in_bytes;
1478
1479         last_alloc = mc_pool.last_alloc[0];
1480         current_alloc = mc_pool.current_alloc[0];
1481
1482         i = mc_pool.cores_count;
1483         while (i-- > 1) {
1484                 if (last_alloc > mc_pool.last_alloc[i])
1485                         last_alloc = mc_pool.last_alloc[i];
1486                 if (current_alloc < mc_pool.current_alloc[i])
1487                         current_alloc = mc_pool.current_alloc[i];
1488         }
1489
1490         flow_size_in_bytes = (current_alloc - last_alloc) / mc_pool.rules_count;
1491         printf("\n:: Port %d :: rte_flow size in DPDK layer: %d Bytes\n",
1492                 port, flow_size_in_bytes);
1493 }
1494
1495 static int
1496 run_rte_flow_handler_cores(void *data __rte_unused)
1497 {
1498         uint16_t port;
1499         int lcore_counter = 0;
1500         int lcore_id = rte_lcore_id();
1501         int i;
1502
1503         RTE_LCORE_FOREACH(i) {
1504                 /*  If core not needed return. */
1505                 if (lcore_id == i) {
1506                         printf(":: lcore %d mapped with index %d\n", lcore_id, lcore_counter);
1507                         if (lcore_counter >= (int) mc_pool.cores_count)
1508                                 return 0;
1509                         break;
1510                 }
1511                 lcore_counter++;
1512         }
1513         lcore_id = lcore_counter;
1514
1515         if (lcore_id >= (int) mc_pool.cores_count)
1516                 return 0;
1517
1518         mc_pool.rules_count = rules_count;
1519
1520         flows_handler(lcore_id);
1521
1522         /* Only main core to print total results. */
1523         if (lcore_id != 0)
1524                 return 0;
1525
1526         /* Make sure all cores finished insertion/deletion process. */
1527         rte_eal_mp_wait_lcore();
1528
1529         RTE_ETH_FOREACH_DEV(port) {
1530                 /* If port outside portmask */
1531                 if (!((ports_mask >> port) & 0x1))
1532                         continue;
1533                 if (has_meter())
1534                         dump_used_cpu_time("Meters:",
1535                                 port, &mc_pool.meters_record);
1536                 dump_used_cpu_time("Flows:",
1537                         port, &mc_pool.flows_record);
1538                 dump_used_mem(port);
1539         }
1540
1541         return 0;
1542 }
1543
1544 static void
1545 signal_handler(int signum)
1546 {
1547         if (signum == SIGINT || signum == SIGTERM) {
1548                 printf("\n\nSignal %d received, preparing to exit...\n",
1549                                         signum);
1550                 printf("Error: Stats are wrong due to sudden signal!\n\n");
1551                 force_quit = true;
1552         }
1553 }
1554
1555 static inline uint16_t
1556 do_rx(struct lcore_info *li, uint16_t rx_port, uint16_t rx_queue)
1557 {
1558         uint16_t cnt = 0;
1559         cnt = rte_eth_rx_burst(rx_port, rx_queue, li->pkts, MAX_PKT_BURST);
1560         li->rx_pkts += cnt;
1561         return cnt;
1562 }
1563
1564 static inline void
1565 do_tx(struct lcore_info *li, uint16_t cnt, uint16_t tx_port,
1566                         uint16_t tx_queue)
1567 {
1568         uint16_t nr_tx = 0;
1569         uint16_t i;
1570
1571         nr_tx = rte_eth_tx_burst(tx_port, tx_queue, li->pkts, cnt);
1572         li->tx_pkts  += nr_tx;
1573         li->tx_drops += cnt - nr_tx;
1574
1575         for (i = nr_tx; i < cnt; i++)
1576                 rte_pktmbuf_free(li->pkts[i]);
1577 }
1578
1579 /*
1580  * Method to convert numbers into pretty numbers that easy
1581  * to read. The design here is to add comma after each three
1582  * digits and set all of this inside buffer.
1583  *
1584  * For example if n = 1799321, the output will be
1585  * 1,799,321 after this method which is easier to read.
1586  */
1587 static char *
1588 pretty_number(uint64_t n, char *buf)
1589 {
1590         char p[6][4];
1591         int i = 0;
1592         int off = 0;
1593
1594         while (n > 1000) {
1595                 sprintf(p[i], "%03d", (int)(n % 1000));
1596                 n /= 1000;
1597                 i += 1;
1598         }
1599
1600         sprintf(p[i++], "%d", (int)n);
1601
1602         while (i--)
1603                 off += sprintf(buf + off, "%s,", p[i]);
1604         buf[strlen(buf) - 1] = '\0';
1605
1606         return buf;
1607 }
1608
1609 static void
1610 packet_per_second_stats(void)
1611 {
1612         struct lcore_info *old;
1613         struct lcore_info *li, *oli;
1614         int nr_lines = 0;
1615         int i;
1616
1617         old = rte_zmalloc("old",
1618                 sizeof(struct lcore_info) * RTE_MAX_LCORE, 0);
1619         if (old == NULL)
1620                 rte_exit(EXIT_FAILURE, "No Memory available!\n");
1621
1622         memcpy(old, lcore_infos,
1623                 sizeof(struct lcore_info) * RTE_MAX_LCORE);
1624
1625         while (!force_quit) {
1626                 uint64_t total_tx_pkts = 0;
1627                 uint64_t total_rx_pkts = 0;
1628                 uint64_t total_tx_drops = 0;
1629                 uint64_t tx_delta, rx_delta, drops_delta;
1630                 char buf[3][32];
1631                 int nr_valid_core = 0;
1632
1633                 sleep(1);
1634
1635                 if (nr_lines) {
1636                         char go_up_nr_lines[16];
1637
1638                         sprintf(go_up_nr_lines, "%c[%dA\r", 27, nr_lines);
1639                         printf("%s\r", go_up_nr_lines);
1640                 }
1641
1642                 printf("\n%6s %16s %16s %16s\n", "core", "tx", "tx drops", "rx");
1643                 printf("%6s %16s %16s %16s\n", "------", "----------------",
1644                         "----------------", "----------------");
1645                 nr_lines = 3;
1646                 for (i = 0; i < RTE_MAX_LCORE; i++) {
1647                         li  = &lcore_infos[i];
1648                         oli = &old[i];
1649                         if (li->mode != LCORE_MODE_PKT)
1650                                 continue;
1651
1652                         tx_delta    = li->tx_pkts  - oli->tx_pkts;
1653                         rx_delta    = li->rx_pkts  - oli->rx_pkts;
1654                         drops_delta = li->tx_drops - oli->tx_drops;
1655                         printf("%6d %16s %16s %16s\n", i,
1656                                 pretty_number(tx_delta,    buf[0]),
1657                                 pretty_number(drops_delta, buf[1]),
1658                                 pretty_number(rx_delta,    buf[2]));
1659
1660                         total_tx_pkts  += tx_delta;
1661                         total_rx_pkts  += rx_delta;
1662                         total_tx_drops += drops_delta;
1663
1664                         nr_valid_core++;
1665                         nr_lines += 1;
1666                 }
1667
1668                 if (nr_valid_core > 1) {
1669                         printf("%6s %16s %16s %16s\n", "total",
1670                                 pretty_number(total_tx_pkts,  buf[0]),
1671                                 pretty_number(total_tx_drops, buf[1]),
1672                                 pretty_number(total_rx_pkts,  buf[2]));
1673                         nr_lines += 1;
1674                 }
1675
1676                 memcpy(old, lcore_infos,
1677                         sizeof(struct lcore_info) * RTE_MAX_LCORE);
1678         }
1679 }
1680
1681 static int
1682 start_forwarding(void *data __rte_unused)
1683 {
1684         int lcore = rte_lcore_id();
1685         int stream_id;
1686         uint16_t cnt;
1687         struct lcore_info *li = &lcore_infos[lcore];
1688
1689         if (!li->mode)
1690                 return 0;
1691
1692         if (li->mode == LCORE_MODE_STATS) {
1693                 printf(":: started stats on lcore %u\n", lcore);
1694                 packet_per_second_stats();
1695                 return 0;
1696         }
1697
1698         while (!force_quit)
1699                 for (stream_id = 0; stream_id < MAX_STREAMS; stream_id++) {
1700                         if (li->streams[stream_id].rx_port == -1)
1701                                 continue;
1702
1703                         cnt = do_rx(li,
1704                                         li->streams[stream_id].rx_port,
1705                                         li->streams[stream_id].rx_queue);
1706                         if (cnt)
1707                                 do_tx(li, cnt,
1708                                         li->streams[stream_id].tx_port,
1709                                         li->streams[stream_id].tx_queue);
1710                 }
1711         return 0;
1712 }
1713
1714 static void
1715 init_lcore_info(void)
1716 {
1717         int i, j;
1718         unsigned int lcore;
1719         uint16_t nr_port;
1720         uint16_t queue;
1721         int port;
1722         int stream_id = 0;
1723         int streams_per_core;
1724         int unassigned_streams;
1725         int nb_fwd_streams;
1726         nr_port = rte_eth_dev_count_avail();
1727
1728         /* First logical core is reserved for stats printing */
1729         lcore = rte_get_next_lcore(-1, 0, 0);
1730         lcore_infos[lcore].mode = LCORE_MODE_STATS;
1731
1732         /*
1733          * Initialize all cores
1734          * All cores at first must have -1 value in all streams
1735          * This means that this stream is not used, or not set
1736          * yet.
1737          */
1738         for (i = 0; i < RTE_MAX_LCORE; i++)
1739                 for (j = 0; j < MAX_STREAMS; j++) {
1740                         lcore_infos[i].streams[j].tx_port = -1;
1741                         lcore_infos[i].streams[j].rx_port = -1;
1742                         lcore_infos[i].streams[j].tx_queue = -1;
1743                         lcore_infos[i].streams[j].rx_queue = -1;
1744                         lcore_infos[i].streams_nb = 0;
1745                 }
1746
1747         /*
1748          * Calculate the total streams count.
1749          * Also distribute those streams count between the available
1750          * logical cores except first core, since it's reserved for
1751          * stats prints.
1752          */
1753         nb_fwd_streams = nr_port * rx_queues_count;
1754         if ((int)(nb_lcores - 1) >= nb_fwd_streams)
1755                 for (i = 0; i < (int)(nb_lcores - 1); i++) {
1756                         lcore = rte_get_next_lcore(lcore, 0, 0);
1757                         lcore_infos[lcore].streams_nb = 1;
1758                 }
1759         else {
1760                 streams_per_core = nb_fwd_streams / (nb_lcores - 1);
1761                 unassigned_streams = nb_fwd_streams % (nb_lcores - 1);
1762                 for (i = 0; i < (int)(nb_lcores - 1); i++) {
1763                         lcore = rte_get_next_lcore(lcore, 0, 0);
1764                         lcore_infos[lcore].streams_nb = streams_per_core;
1765                         if (unassigned_streams) {
1766                                 lcore_infos[lcore].streams_nb++;
1767                                 unassigned_streams--;
1768                         }
1769                 }
1770         }
1771
1772         /*
1773          * Set the streams for the cores according to each logical
1774          * core stream count.
1775          * The streams is built on the design of what received should
1776          * forward as well, this means that if you received packets on
1777          * port 0 queue 0 then the same queue should forward the
1778          * packets, using the same logical core.
1779          */
1780         lcore = rte_get_next_lcore(-1, 0, 0);
1781         for (port = 0; port < nr_port; port++) {
1782                 /* Create FWD stream */
1783                 for (queue = 0; queue < rx_queues_count; queue++) {
1784                         if (!lcore_infos[lcore].streams_nb ||
1785                                 !(stream_id % lcore_infos[lcore].streams_nb)) {
1786                                 lcore = rte_get_next_lcore(lcore, 0, 0);
1787                                 lcore_infos[lcore].mode = LCORE_MODE_PKT;
1788                                 stream_id = 0;
1789                         }
1790                         lcore_infos[lcore].streams[stream_id].rx_queue = queue;
1791                         lcore_infos[lcore].streams[stream_id].tx_queue = queue;
1792                         lcore_infos[lcore].streams[stream_id].rx_port = port;
1793                         lcore_infos[lcore].streams[stream_id].tx_port = port;
1794                         stream_id++;
1795                 }
1796         }
1797
1798         /* Print all streams */
1799         printf(":: Stream -> core id[N]: (rx_port, rx_queue)->(tx_port, tx_queue)\n");
1800         for (i = 0; i < RTE_MAX_LCORE; i++)
1801                 for (j = 0; j < MAX_STREAMS; j++) {
1802                         /* No streams for this core */
1803                         if (lcore_infos[i].streams[j].tx_port == -1)
1804                                 break;
1805                         printf("Stream -> core id[%d]: (%d,%d)->(%d,%d)\n",
1806                                 i,
1807                                 lcore_infos[i].streams[j].rx_port,
1808                                 lcore_infos[i].streams[j].rx_queue,
1809                                 lcore_infos[i].streams[j].tx_port,
1810                                 lcore_infos[i].streams[j].tx_queue);
1811                 }
1812 }
1813
1814 static void
1815 init_port(void)
1816 {
1817         int ret;
1818         uint16_t std_queue;
1819         uint16_t hairpin_queue;
1820         uint16_t port_id;
1821         uint16_t nr_ports;
1822         uint16_t nr_queues;
1823         struct rte_eth_hairpin_conf hairpin_conf = {
1824                 .peer_count = 1,
1825         };
1826         struct rte_eth_conf port_conf = {
1827                 .rx_adv_conf = {
1828                         .rss_conf.rss_hf =
1829                                 GET_RSS_HF(),
1830                 }
1831         };
1832         struct rte_eth_txconf txq_conf;
1833         struct rte_eth_rxconf rxq_conf;
1834         struct rte_eth_dev_info dev_info;
1835
1836         nr_queues = rx_queues_count;
1837         if (hairpin_queues_num != 0)
1838                 nr_queues = rx_queues_count + hairpin_queues_num;
1839
1840         nr_ports = rte_eth_dev_count_avail();
1841         if (nr_ports == 0)
1842                 rte_exit(EXIT_FAILURE, "Error: no port detected\n");
1843
1844         mbuf_mp = rte_pktmbuf_pool_create("mbuf_pool",
1845                                         total_mbuf_num, mbuf_cache_size,
1846                                         0, mbuf_size,
1847                                         rte_socket_id());
1848         if (mbuf_mp == NULL)
1849                 rte_exit(EXIT_FAILURE, "Error: can't init mbuf pool\n");
1850
1851         for (port_id = 0; port_id < nr_ports; port_id++) {
1852                 uint64_t rx_metadata = 0;
1853
1854                 rx_metadata |= RTE_ETH_RX_METADATA_USER_FLAG;
1855                 rx_metadata |= RTE_ETH_RX_METADATA_USER_MARK;
1856
1857                 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_metadata);
1858                 if (ret == 0) {
1859                         if (!(rx_metadata & RTE_ETH_RX_METADATA_USER_FLAG)) {
1860                                 printf(":: flow action FLAG will not affect Rx mbufs on port=%u\n",
1861                                        port_id);
1862                         }
1863
1864                         if (!(rx_metadata & RTE_ETH_RX_METADATA_USER_MARK)) {
1865                                 printf(":: flow action MARK will not affect Rx mbufs on port=%u\n",
1866                                        port_id);
1867                         }
1868                 } else if (ret != -ENOTSUP) {
1869                         rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port=%u: %s\n",
1870                                  port_id, rte_strerror(-ret));
1871                 }
1872
1873                 ret = rte_eth_dev_info_get(port_id, &dev_info);
1874                 if (ret != 0)
1875                         rte_exit(EXIT_FAILURE,
1876                                 "Error during getting device"
1877                                 " (port %u) info: %s\n",
1878                                 port_id, strerror(-ret));
1879
1880                 port_conf.txmode.offloads &= dev_info.tx_offload_capa;
1881                 port_conf.rxmode.offloads &= dev_info.rx_offload_capa;
1882
1883                 printf(":: initializing port: %d\n", port_id);
1884
1885                 ret = rte_eth_dev_configure(port_id, nr_queues,
1886                                 nr_queues, &port_conf);
1887                 if (ret < 0)
1888                         rte_exit(EXIT_FAILURE,
1889                                 ":: cannot configure device: err=%d, port=%u\n",
1890                                 ret, port_id);
1891
1892                 rxq_conf = dev_info.default_rxconf;
1893                 for (std_queue = 0; std_queue < rx_queues_count; std_queue++) {
1894                         ret = rte_eth_rx_queue_setup(port_id, std_queue, rxd_count,
1895                                         rte_eth_dev_socket_id(port_id),
1896                                         &rxq_conf,
1897                                         mbuf_mp);
1898                         if (ret < 0)
1899                                 rte_exit(EXIT_FAILURE,
1900                                         ":: Rx queue setup failed: err=%d, port=%u\n",
1901                                         ret, port_id);
1902                 }
1903
1904                 txq_conf = dev_info.default_txconf;
1905                 for (std_queue = 0; std_queue < tx_queues_count; std_queue++) {
1906                         ret = rte_eth_tx_queue_setup(port_id, std_queue, txd_count,
1907                                         rte_eth_dev_socket_id(port_id),
1908                                         &txq_conf);
1909                         if (ret < 0)
1910                                 rte_exit(EXIT_FAILURE,
1911                                         ":: Tx queue setup failed: err=%d, port=%u\n",
1912                                         ret, port_id);
1913                 }
1914
1915                 /* Catch all packets from traffic generator. */
1916                 ret = rte_eth_promiscuous_enable(port_id);
1917                 if (ret != 0)
1918                         rte_exit(EXIT_FAILURE,
1919                                 ":: promiscuous mode enable failed: err=%s, port=%u\n",
1920                                 rte_strerror(-ret), port_id);
1921
1922                 if (hairpin_queues_num != 0) {
1923                         /*
1924                          * Configure peer which represents hairpin Tx.
1925                          * Hairpin queue numbers start after standard queues
1926                          * (rx_queues_count and tx_queues_count).
1927                          */
1928                         for (hairpin_queue = rx_queues_count, std_queue = 0;
1929                                         hairpin_queue < nr_queues;
1930                                         hairpin_queue++, std_queue++) {
1931                                 hairpin_conf.peers[0].port = port_id;
1932                                 hairpin_conf.peers[0].queue =
1933                                         std_queue + tx_queues_count;
1934                                 ret = rte_eth_rx_hairpin_queue_setup(
1935                                                 port_id, hairpin_queue,
1936                                                 rxd_count, &hairpin_conf);
1937                                 if (ret != 0)
1938                                         rte_exit(EXIT_FAILURE,
1939                                                 ":: Hairpin rx queue setup failed: err=%d, port=%u\n",
1940                                                 ret, port_id);
1941                         }
1942
1943                         for (hairpin_queue = tx_queues_count, std_queue = 0;
1944                                         hairpin_queue < nr_queues;
1945                                         hairpin_queue++, std_queue++) {
1946                                 hairpin_conf.peers[0].port = port_id;
1947                                 hairpin_conf.peers[0].queue =
1948                                         std_queue + rx_queues_count;
1949                                 ret = rte_eth_tx_hairpin_queue_setup(
1950                                                 port_id, hairpin_queue,
1951                                                 txd_count, &hairpin_conf);
1952                                 if (ret != 0)
1953                                         rte_exit(EXIT_FAILURE,
1954                                                 ":: Hairpin tx queue setup failed: err=%d, port=%u\n",
1955                                                 ret, port_id);
1956                         }
1957                 }
1958
1959                 ret = rte_eth_dev_start(port_id);
1960                 if (ret < 0)
1961                         rte_exit(EXIT_FAILURE,
1962                                 "rte_eth_dev_start:err=%d, port=%u\n",
1963                                 ret, port_id);
1964
1965                 printf(":: initializing port: %d done\n", port_id);
1966         }
1967 }
1968
1969 int
1970 main(int argc, char **argv)
1971 {
1972         int ret;
1973         uint16_t port;
1974         struct rte_flow_error error;
1975
1976         ret = rte_eal_init(argc, argv);
1977         if (ret < 0)
1978                 rte_exit(EXIT_FAILURE, "EAL init failed\n");
1979
1980         force_quit = false;
1981         dump_iterations = false;
1982         rules_count = DEFAULT_RULES_COUNT;
1983         rules_batch = DEFAULT_RULES_BATCH;
1984         delete_flag = false;
1985         dump_socket_mem_flag = false;
1986         flow_group = DEFAULT_GROUP;
1987         unique_data = false;
1988
1989         rx_queues_count = (uint8_t) RXQ_NUM;
1990         tx_queues_count = (uint8_t) TXQ_NUM;
1991         rxd_count = (uint8_t) NR_RXD;
1992         txd_count = (uint8_t) NR_TXD;
1993         mbuf_size = (uint32_t) MBUF_SIZE;
1994         mbuf_cache_size = (uint32_t) MBUF_CACHE_SIZE;
1995         total_mbuf_num = (uint32_t) TOTAL_MBUF_NUM;
1996
1997         signal(SIGINT, signal_handler);
1998         signal(SIGTERM, signal_handler);
1999
2000         argc -= ret;
2001         argv += ret;
2002         if (argc > 1)
2003                 args_parse(argc, argv);
2004
2005         init_port();
2006
2007         nb_lcores = rte_lcore_count();
2008         if (nb_lcores <= 1)
2009                 rte_exit(EXIT_FAILURE, "This app needs at least two cores\n");
2010
2011         printf(":: Flows Count per port: %d\n\n", rules_count);
2012
2013         rte_srand(rand_seed);
2014
2015         if (has_meter())
2016                 create_meter_profile();
2017         rte_eal_mp_remote_launch(run_rte_flow_handler_cores, NULL, CALL_MAIN);
2018
2019         if (enable_fwd) {
2020                 init_lcore_info();
2021                 rte_eal_mp_remote_launch(start_forwarding, NULL, CALL_MAIN);
2022         }
2023         if (has_meter() && delete_flag)
2024                 destroy_meter_profile();
2025
2026         RTE_ETH_FOREACH_DEV(port) {
2027                 rte_flow_flush(port, &error);
2028                 if (rte_eth_dev_stop(port) != 0)
2029                         printf("Failed to stop device on port %u\n", port);
2030                 rte_eth_dev_close(port);
2031         }
2032         printf("\nBye ...\n");
2033         return 0;
2034 }