app/test-pmd/csumonly.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdarg.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdint.h>
  38 #include <unistd.h>
  39 #include <inttypes.h>
  40
  41 #include <sys/queue.h>
  42 #include <sys/stat.h>
  43
  44 #include <rte_common.h>
  45 #include <rte_byteorder.h>
  46 #include <rte_log.h>
  47 #include <rte_debug.h>
  48 #include <rte_cycles.h>
  49 #include <rte_memory.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_memzone.h>
  52 #include <rte_launch.h>
  53 #include <rte_tailq.h>
  54 #include <rte_eal.h>
  55 #include <rte_per_lcore.h>
  56 #include <rte_lcore.h>
  57 #include <rte_atomic.h>
  58 #include <rte_branch_prediction.h>
  59 #include <rte_ring.h>
  60 #include <rte_memory.h>
  61 #include <rte_mempool.h>
  62 #include <rte_mbuf.h>
  63 #include <rte_memcpy.h>
  64 #include <rte_interrupts.h>
  65 #include <rte_pci.h>
  66 #include <rte_ether.h>
  67 #include <rte_ethdev.h>
  68 #include <rte_ip.h>
  69 #include <rte_tcp.h>
  70 #include <rte_udp.h>
  71 #include <rte_sctp.h>
  72 #include <rte_prefetch.h>
  73 #include <rte_string_fns.h>
  74 #include "testpmd.h"
  75
  76
  77
  78 #define IP_DEFTTL  64   /* from RFC 1340. */
  79 #define IP_VERSION 0x40
  80 #define IP_HDRLEN  0x05 /* default IP header length == five 32-bits words. */
  81 #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
  82
  83 static inline uint16_t
  84 get_16b_sum(uint16_t *ptr16, uint32_t nr)
  85 {
  86         uint32_t sum = 0;
  87         while (nr > 1)
  88         {
  89                 sum +=*ptr16;
  90                 nr -= sizeof(uint16_t);
  91                 ptr16++;
  92                 if (sum > UINT16_MAX)
  93                         sum -= UINT16_MAX;
  94         }
  95
  96         /* If length is in odd bytes */
  97         if (nr)
  98                 sum += *((uint8_t*)ptr16);
  99
 100         sum = ((sum & 0xffff0000) >> 16) + (sum & 0xffff);
 101         sum &= 0x0ffff;
 102         return (uint16_t)sum;
 103 }
 104
 105 static inline uint16_t
 106 get_ipv4_cksum(struct ipv4_hdr *ipv4_hdr)
 107 {
 108         uint16_t cksum;
 109         cksum = get_16b_sum((uint16_t*)ipv4_hdr, sizeof(struct ipv4_hdr));
 110         return (uint16_t)((cksum == 0xffff)?cksum:~cksum);
 111 }
 112
 113
 114 static inline uint16_t
 115 get_ipv4_psd_sum (struct ipv4_hdr * ip_hdr)
 116 {
 117         /* Pseudo Header for IPv4/UDP/TCP checksum */
 118         union ipv4_psd_header {
 119                 struct {
 120                         uint32_t src_addr; /* IP address of source host. */
 121                         uint32_t dst_addr; /* IP address of destination host(s). */
 122                         uint8_t  zero;     /* zero. */
 123                         uint8_t  proto;    /* L4 protocol type. */
 124                         uint16_t len;      /* L4 length. */
 125                 } __attribute__((__packed__));
 126                 uint16_t u16_arr[0];
 127         } psd_hdr;
 128
 129         psd_hdr.src_addr = ip_hdr->src_addr;
 130         psd_hdr.dst_addr = ip_hdr->dst_addr;
 131         psd_hdr.zero     = 0;
 132         psd_hdr.proto    = ip_hdr->next_proto_id;
 133         psd_hdr.len      = rte_cpu_to_be_16((uint16_t)(rte_be_to_cpu_16(ip_hdr->total_length)
 134                                 - sizeof(struct ipv4_hdr)));
 135         return get_16b_sum(psd_hdr.u16_arr, sizeof(psd_hdr));
 136 }
 137
 138 static inline uint16_t
 139 get_ipv6_psd_sum (struct ipv6_hdr * ip_hdr)
 140 {
 141         /* Pseudo Header for IPv6/UDP/TCP checksum */
 142         union ipv6_psd_header {
 143                 struct {
 144                         uint8_t src_addr[16]; /* IP address of source host. */
 145                         uint8_t dst_addr[16]; /* IP address of destination host(s). */
 146                         uint32_t len;         /* L4 length. */
 147                         uint32_t proto;       /* L4 protocol - top 3 bytes must be zero */
 148                 } __attribute__((__packed__));
 149
 150                 uint16_t u16_arr[0]; /* allow use as 16-bit values with safe aliasing */
 151         } psd_hdr;
 152
 153         rte_memcpy(&psd_hdr.src_addr, ip_hdr->src_addr,
 154                         sizeof(ip_hdr->src_addr) + sizeof(ip_hdr->dst_addr));
 155         psd_hdr.len       = ip_hdr->payload_len;
 156         psd_hdr.proto     = (ip_hdr->proto << 24);
 157
 158         return get_16b_sum(psd_hdr.u16_arr, sizeof(psd_hdr));
 159 }
 160
 161 static inline uint16_t
 162 get_ipv4_udptcp_checksum(struct ipv4_hdr *ipv4_hdr, uint16_t *l4_hdr)
 163 {
 164         uint32_t cksum;
 165         uint32_t l4_len;
 166
 167         l4_len = rte_be_to_cpu_16(ipv4_hdr->total_length) - sizeof(struct ipv4_hdr);
 168
 169         cksum = get_16b_sum(l4_hdr, l4_len);
 170         cksum += get_ipv4_psd_sum(ipv4_hdr);
 171
 172         cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
 173         cksum = (~cksum) & 0xffff;
 174         if (cksum == 0)
 175                 cksum = 0xffff;
 176         return (uint16_t)cksum;
 177
 178 }
 179
 180 static inline uint16_t
 181 get_ipv6_udptcp_checksum(struct ipv6_hdr *ipv6_hdr, uint16_t *l4_hdr)
 182 {
 183         uint32_t cksum;
 184         uint32_t l4_len;
 185
 186         l4_len = rte_be_to_cpu_16(ipv6_hdr->payload_len);
 187
 188         cksum = get_16b_sum(l4_hdr, l4_len);
 189         cksum += get_ipv6_psd_sum(ipv6_hdr);
 190
 191         cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
 192         cksum = (~cksum) & 0xffff;
 193         if (cksum == 0)
 194                 cksum = 0xffff;
 195
 196         return (uint16_t)cksum;
 197 }
 198
 199
 200 /*
 201  * Forwarding of packets. Change the checksum field with HW or SW methods
 202  * The HW/SW method selection depends on the ol_flags on every packet
 203  */
 204 static void
 205 pkt_burst_checksum_forward(struct fwd_stream *fs)
 206 {
 207         struct rte_mbuf  *pkts_burst[MAX_PKT_BURST];
 208         struct rte_port  *txp;
 209         struct rte_mbuf  *mb;
 210         struct ether_hdr *eth_hdr;
 211         struct ipv4_hdr  *ipv4_hdr;
 212         struct ether_hdr *inner_eth_hdr;
 213         struct ipv4_hdr  *inner_ipv4_hdr = NULL;
 214         struct ipv6_hdr  *ipv6_hdr;
 215         struct ipv6_hdr  *inner_ipv6_hdr = NULL;
 216         struct udp_hdr   *udp_hdr;
 217         struct udp_hdr   *inner_udp_hdr;
 218         struct tcp_hdr   *tcp_hdr;
 219         struct tcp_hdr   *inner_tcp_hdr;
 220         struct sctp_hdr  *sctp_hdr;
 221         struct sctp_hdr  *inner_sctp_hdr;
 222
 223         uint16_t nb_rx;
 224         uint16_t nb_tx;
 225         uint16_t i;
 226         uint64_t ol_flags;
 227         uint64_t pkt_ol_flags;
 228         uint64_t tx_ol_flags;
 229         uint16_t l4_proto;
 230         uint16_t inner_l4_proto = 0;
 231         uint16_t eth_type;
 232         uint8_t  l2_len;
 233         uint8_t  l3_len;
 234         uint8_t  inner_l2_len = 0;
 235         uint8_t  inner_l3_len = 0;
 236
 237         uint32_t rx_bad_ip_csum;
 238         uint32_t rx_bad_l4_csum;
 239         uint8_t  ipv4_tunnel;
 240         uint8_t  ipv6_tunnel;
 241
 242 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 243         uint64_t start_tsc;
 244         uint64_t end_tsc;
 245         uint64_t core_cycles;
 246 #endif
 247
 248 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 249         start_tsc = rte_rdtsc();
 250 #endif
 251
 252         /*
 253          * Receive a burst of packets and forward them.
 254          */
 255         nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, pkts_burst,
 256                                  nb_pkt_per_burst);
 257         if (unlikely(nb_rx == 0))
 258                 return;
 259
 260 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 261         fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 262 #endif
 263         fs->rx_packets += nb_rx;
 264         rx_bad_ip_csum = 0;
 265         rx_bad_l4_csum = 0;
 266
 267         txp = &ports[fs->tx_port];
 268         tx_ol_flags = txp->tx_ol_flags;
 269
 270         for (i = 0; i < nb_rx; i++) {
 271
 272                 mb = pkts_burst[i];
 273                 l2_len  = sizeof(struct ether_hdr);
 274                 pkt_ol_flags = mb->ol_flags;
 275                 ol_flags = (pkt_ol_flags & (~PKT_TX_L4_MASK));
 276                 ipv4_tunnel = (pkt_ol_flags & PKT_RX_TUNNEL_IPV4_HDR) ?
 277                                 1 : 0;
 278                 ipv6_tunnel = (pkt_ol_flags & PKT_RX_TUNNEL_IPV6_HDR) ?
 279                                 1 : 0;
 280                 eth_hdr = rte_pktmbuf_mtod(mb, struct ether_hdr *);
 281                 eth_type = rte_be_to_cpu_16(eth_hdr->ether_type);
 282                 if (eth_type == ETHER_TYPE_VLAN) {
 283                         /* Only allow single VLAN label here */
 284                         l2_len  += sizeof(struct vlan_hdr);
 285                          eth_type = rte_be_to_cpu_16(*(uint16_t *)
 286                                 ((uintptr_t)&eth_hdr->ether_type +
 287                                 sizeof(struct vlan_hdr)));
 288                 }
 289
 290                 /* Update the L3/L4 checksum error packet count  */
 291                 rx_bad_ip_csum += (uint16_t) ((pkt_ol_flags & PKT_RX_IP_CKSUM_BAD) != 0);
 292                 rx_bad_l4_csum += (uint16_t) ((pkt_ol_flags & PKT_RX_L4_CKSUM_BAD) != 0);
 293
 294                 /*
 295                  * Try to figure out L3 packet type by SW.
 296                  */
 297                 if ((pkt_ol_flags & (PKT_RX_IPV4_HDR | PKT_RX_IPV4_HDR_EXT |
 298                                 PKT_RX_IPV6_HDR | PKT_RX_IPV6_HDR_EXT)) == 0) {
 299                         if (eth_type == ETHER_TYPE_IPv4)
 300                                 pkt_ol_flags |= PKT_RX_IPV4_HDR;
 301                         else if (eth_type == ETHER_TYPE_IPv6)
 302                                 pkt_ol_flags |= PKT_RX_IPV6_HDR;
 303                 }
 304
 305                 /*
 306                  * Simplify the protocol parsing
 307                  * Assuming the incoming packets format as
 308                  *      Ethernet2 + optional single VLAN
 309                  *      + ipv4 or ipv6
 310                  *      + udp or tcp or sctp or others
 311                  */
 312                 if (pkt_ol_flags & (PKT_RX_IPV4_HDR | PKT_RX_TUNNEL_IPV4_HDR)) {
 313
 314                         /* Do not support ipv4 option field */
 315                         l3_len = sizeof(struct ipv4_hdr) ;
 316
 317                         ipv4_hdr = (struct ipv4_hdr *) (rte_pktmbuf_mtod(mb,
 318                                         unsigned char *) + l2_len);
 319
 320                         l4_proto = ipv4_hdr->next_proto_id;
 321
 322                         /* Do not delete, this is required by HW*/
 323                         ipv4_hdr->hdr_checksum = 0;
 324
 325                         if (tx_ol_flags & TESTPMD_TX_OFFLOAD_IP_CKSUM) {
 326                                 /* HW checksum */
 327                                 ol_flags |= PKT_TX_IP_CKSUM;
 328                         }
 329                         else {
 330                                 ol_flags |= PKT_TX_IPV4;
 331                                 /* SW checksum calculation */
 332                                 ipv4_hdr->src_addr++;
 333                                 ipv4_hdr->hdr_checksum = get_ipv4_cksum(ipv4_hdr);
 334                         }
 335
 336                         if (l4_proto == IPPROTO_UDP) {
 337                                 udp_hdr = (struct udp_hdr*) (rte_pktmbuf_mtod(mb,
 338                                                 unsigned char *) + l2_len + l3_len);
 339                                 if (tx_ol_flags & TESTPMD_TX_OFFLOAD_UDP_CKSUM) {
 340                                         /* HW Offload */
 341                                         ol_flags |= PKT_TX_UDP_CKSUM;
 342                                         if (ipv4_tunnel)
 343                                                 udp_hdr->dgram_cksum = 0;
 344                                         else
 345                                                 /* Pseudo header sum need be set properly */
 346                                                 udp_hdr->dgram_cksum =
 347                                                         get_ipv4_psd_sum(ipv4_hdr);
 348                                 }
 349                                 else {
 350                                         /* SW Implementation, clear checksum field first */
 351                                         udp_hdr->dgram_cksum = 0;
 352                                         udp_hdr->dgram_cksum = get_ipv4_udptcp_checksum(ipv4_hdr,
 353                                                                         (uint16_t *)udp_hdr);
 354                                 }
 355
 356                                 if (ipv4_tunnel) {
 357
 358                                         uint16_t len;
 359
 360                                         /* Check if inner L3/L4 checkum flag is set */
 361                                         if (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_CKSUM_MASK)
 362                                                 ol_flags |= PKT_TX_VXLAN_CKSUM;
 363
 364                                         inner_l2_len  = sizeof(struct ether_hdr);
 365                                         inner_eth_hdr = (struct ether_hdr *) (rte_pktmbuf_mtod(mb,
 366                                                                 unsigned char *) + l2_len + l3_len
 367                                                                  + ETHER_VXLAN_HLEN);
 368
 369                                         eth_type = rte_be_to_cpu_16(inner_eth_hdr->ether_type);
 370                                         if (eth_type == ETHER_TYPE_VLAN) {
 371                                                 inner_l2_len += sizeof(struct vlan_hdr);
 372                                                 eth_type = rte_be_to_cpu_16(*(uint16_t *)
 373                                                         ((uintptr_t)&eth_hdr->ether_type +
 374                                                                 sizeof(struct vlan_hdr)));
 375                                         }
 376
 377                                         len = l2_len + l3_len + ETHER_VXLAN_HLEN + inner_l2_len;
 378                                         if (eth_type == ETHER_TYPE_IPv4) {
 379                                                 inner_l3_len = sizeof(struct ipv4_hdr);
 380                                                 inner_ipv4_hdr = (struct ipv4_hdr *) (rte_pktmbuf_mtod(mb,
 381                                                                 unsigned char *) + len);
 382                                                 inner_l4_proto = inner_ipv4_hdr->next_proto_id;
 383
 384                                                 if (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_IP_CKSUM) {
 385
 386                                                         /* Do not delete, this is required by HW*/
 387                                                         inner_ipv4_hdr->hdr_checksum = 0;
 388                                                         ol_flags |= PKT_TX_IPV4_CSUM;
 389                                                 }
 390
 391                                         } else if (eth_type == ETHER_TYPE_IPv6) {
 392                                                 inner_l3_len = sizeof(struct ipv6_hdr);
 393                                                 inner_ipv6_hdr = (struct ipv6_hdr *) (rte_pktmbuf_mtod(mb,
 394                                                                 unsigned char *) + len);
 395                                                 inner_l4_proto = inner_ipv6_hdr->proto;
 396                                         }
 397                                         if ((inner_l4_proto == IPPROTO_UDP) &&
 398                                                 (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_UDP_CKSUM)) {
 399
 400                                                 /* HW Offload */
 401                                                 ol_flags |= PKT_TX_UDP_CKSUM;
 402                                                 inner_udp_hdr = (struct udp_hdr *) (rte_pktmbuf_mtod(mb,
 403                                                                 unsigned char *) + len + inner_l3_len);
 404                                                 if (eth_type == ETHER_TYPE_IPv4)
 405                                                         inner_udp_hdr->dgram_cksum = get_ipv4_psd_sum(inner_ipv4_hdr);
 406                                                 else if (eth_type == ETHER_TYPE_IPv6)
 407                                                         inner_udp_hdr->dgram_cksum = get_ipv6_psd_sum(inner_ipv6_hdr);
 408
 409                                         } else if ((inner_l4_proto == IPPROTO_TCP) &&
 410                                                 (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_TCP_CKSUM)) {
 411                                                 /* HW Offload */
 412                                                 ol_flags |= PKT_TX_TCP_CKSUM;
 413                                                 inner_tcp_hdr = (struct tcp_hdr *) (rte_pktmbuf_mtod(mb,
 414                                                                 unsigned char *) + len + inner_l3_len);
 415                                                 if (eth_type == ETHER_TYPE_IPv4)
 416                                                         inner_tcp_hdr->cksum = get_ipv4_psd_sum(inner_ipv4_hdr);
 417                                                 else if (eth_type == ETHER_TYPE_IPv6)
 418                                                         inner_tcp_hdr->cksum = get_ipv6_psd_sum(inner_ipv6_hdr);
 419                                         } else if ((inner_l4_proto == IPPROTO_SCTP) &&
 420                                                 (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_SCTP_CKSUM)) {
 421                                                 /* HW Offload */
 422                                                 ol_flags |= PKT_TX_SCTP_CKSUM;
 423                                                 inner_sctp_hdr = (struct sctp_hdr *) (rte_pktmbuf_mtod(mb,
 424                                                                 unsigned char *) + len + inner_l3_len);
 425                                                 inner_sctp_hdr->cksum = 0;
 426                                         }
 427
 428                                 }
 429
 430                         } else if (l4_proto == IPPROTO_TCP) {
 431                                 tcp_hdr = (struct tcp_hdr*) (rte_pktmbuf_mtod(mb,
 432                                                 unsigned char *) + l2_len + l3_len);
 433                                 if (tx_ol_flags & TESTPMD_TX_OFFLOAD_TCP_CKSUM) {
 434                                         ol_flags |= PKT_TX_TCP_CKSUM;
 435                                         tcp_hdr->cksum = get_ipv4_psd_sum(ipv4_hdr);
 436                                 }
 437                                 else {
 438                                         tcp_hdr->cksum = 0;
 439                                         tcp_hdr->cksum = get_ipv4_udptcp_checksum(ipv4_hdr,
 440                                                         (uint16_t*)tcp_hdr);
 441                                 }
 442                         } else if (l4_proto == IPPROTO_SCTP) {
 443                                 sctp_hdr = (struct sctp_hdr*) (rte_pktmbuf_mtod(mb,
 444                                                 unsigned char *) + l2_len + l3_len);
 445
 446                                 if (tx_ol_flags & TESTPMD_TX_OFFLOAD_SCTP_CKSUM) {
 447                                         ol_flags |= PKT_TX_SCTP_CKSUM;
 448                                         sctp_hdr->cksum = 0;
 449
 450                                         /* Sanity check, only number of 4 bytes supported */
 451                                         if ((rte_be_to_cpu_16(ipv4_hdr->total_length) % 4) != 0)
 452                                                 printf("sctp payload must be a multiple "
 453                                                         "of 4 bytes for checksum offload");
 454                                 }
 455                                 else {
 456                                         sctp_hdr->cksum = 0;
 457                                         /* CRC32c sample code available in RFC3309 */
 458                                 }
 459                         }
 460                         /* End of L4 Handling*/
 461                 } else if (pkt_ol_flags & (PKT_RX_IPV6_HDR | PKT_RX_TUNNEL_IPV6_HDR)) {
 462                         ipv6_hdr = (struct ipv6_hdr *) (rte_pktmbuf_mtod(mb,
 463                                         unsigned char *) + l2_len);
 464                         l3_len = sizeof(struct ipv6_hdr) ;
 465                         l4_proto = ipv6_hdr->proto;
 466                         ol_flags |= PKT_TX_IPV6;
 467
 468                         if (l4_proto == IPPROTO_UDP) {
 469                                 udp_hdr = (struct udp_hdr*) (rte_pktmbuf_mtod(mb,
 470                                                 unsigned char *) + l2_len + l3_len);
 471                                 if (tx_ol_flags & TESTPMD_TX_OFFLOAD_UDP_CKSUM) {
 472                                         /* HW Offload */
 473                                         ol_flags |= PKT_TX_UDP_CKSUM;
 474                                         if (ipv6_tunnel)
 475                                                 udp_hdr->dgram_cksum = 0;
 476                                         else
 477                                                 udp_hdr->dgram_cksum =
 478                                                         get_ipv6_psd_sum(ipv6_hdr);
 479                                 }
 480                                 else {
 481                                         /* SW Implementation */
 482                                         /* checksum field need be clear first */
 483                                         udp_hdr->dgram_cksum = 0;
 484                                         udp_hdr->dgram_cksum = get_ipv6_udptcp_checksum(ipv6_hdr,
 485                                                                 (uint16_t *)udp_hdr);
 486                                 }
 487
 488                                 if (ipv6_tunnel) {
 489
 490                                         uint16_t len;
 491
 492                                         /* Check if inner L3/L4 checksum flag is set */
 493                                         if (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_CKSUM_MASK)
 494                                                 ol_flags |= PKT_TX_VXLAN_CKSUM;
 495
 496                                         inner_l2_len  = sizeof(struct ether_hdr);
 497                                         inner_eth_hdr = (struct ether_hdr *) (rte_pktmbuf_mtod(mb,
 498                                                 unsigned char *) + l2_len + l3_len + ETHER_VXLAN_HLEN);
 499                                         eth_type = rte_be_to_cpu_16(inner_eth_hdr->ether_type);
 500
 501                                         if (eth_type == ETHER_TYPE_VLAN) {
 502                                                 inner_l2_len += sizeof(struct vlan_hdr);
 503                                                 eth_type = rte_be_to_cpu_16(*(uint16_t *)
 504                                                         ((uintptr_t)&eth_hdr->ether_type +
 505                                                         sizeof(struct vlan_hdr)));
 506                                         }
 507
 508                                         len = l2_len + l3_len + ETHER_VXLAN_HLEN + inner_l2_len;
 509
 510                                         if (eth_type == ETHER_TYPE_IPv4) {
 511                                                 inner_l3_len = sizeof(struct ipv4_hdr);
 512                                                 inner_ipv4_hdr = (struct ipv4_hdr *) (rte_pktmbuf_mtod(mb,
 513                                                                 unsigned char *) + len);
 514                                                 inner_l4_proto = inner_ipv4_hdr->next_proto_id;
 515
 516                                                 /* HW offload */
 517                                                 if (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_IP_CKSUM) {
 518
 519                                                         /* Do not delete, this is required by HW*/
 520                                                         inner_ipv4_hdr->hdr_checksum = 0;
 521                                                         ol_flags |= PKT_TX_IPV4_CSUM;
 522                                                 }
 523                                         } else if (eth_type == ETHER_TYPE_IPv6) {
 524                                                 inner_l3_len = sizeof(struct ipv6_hdr);
 525                                                 inner_ipv6_hdr = (struct ipv6_hdr *) (rte_pktmbuf_mtod(mb,
 526                                                         unsigned char *) + len);
 527                                                 inner_l4_proto = inner_ipv6_hdr->proto;
 528                                         }
 529
 530                                         if ((inner_l4_proto == IPPROTO_UDP) &&
 531                                                 (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_UDP_CKSUM)) {
 532                                                 inner_udp_hdr = (struct udp_hdr *) (rte_pktmbuf_mtod(mb,
 533                                                         unsigned char *) + len + inner_l3_len);
 534                                                 /* HW offload */
 535                                                 ol_flags |= PKT_TX_UDP_CKSUM;
 536                                                 inner_udp_hdr->dgram_cksum = 0;
 537                                                 if (eth_type == ETHER_TYPE_IPv4)
 538                                                         inner_udp_hdr->dgram_cksum = get_ipv4_psd_sum(inner_ipv4_hdr);
 539                                                 else if (eth_type == ETHER_TYPE_IPv6)
 540                                                         inner_udp_hdr->dgram_cksum = get_ipv6_psd_sum(inner_ipv6_hdr);
 541                                         } else if ((inner_l4_proto == IPPROTO_TCP) &&
 542                                                 (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_TCP_CKSUM)) {
 543                                                 /* HW offload */
 544                                                 ol_flags |= PKT_TX_TCP_CKSUM;
 545                                                 inner_tcp_hdr = (struct tcp_hdr *) (rte_pktmbuf_mtod(mb,
 546                                                                 unsigned char *) + len + inner_l3_len);
 547
 548                                                 if (eth_type == ETHER_TYPE_IPv4)
 549                                                         inner_tcp_hdr->cksum = get_ipv4_psd_sum(inner_ipv4_hdr);
 550                                                 else if (eth_type == ETHER_TYPE_IPv6)
 551                                                         inner_tcp_hdr->cksum = get_ipv6_psd_sum(inner_ipv6_hdr);
 552
 553                                         } else if ((inner_l4_proto == IPPROTO_SCTP) &&
 554                                                 (tx_ol_flags & TESTPMD_TX_OFFLOAD_INNER_SCTP_CKSUM)) {
 555                                                 /* HW offload */
 556                                                 ol_flags |= PKT_TX_SCTP_CKSUM;
 557                                                 inner_sctp_hdr = (struct sctp_hdr *) (rte_pktmbuf_mtod(mb,
 558                                                                 unsigned char *) + len + inner_l3_len);
 559                                                 inner_sctp_hdr->cksum = 0;
 560                                         }
 561
 562                                 }
 563
 564                         }
 565                         else if (l4_proto == IPPROTO_TCP) {
 566                                 tcp_hdr = (struct tcp_hdr*) (rte_pktmbuf_mtod(mb,
 567                                                 unsigned char *) + l2_len + l3_len);
 568                                 if (tx_ol_flags & TESTPMD_TX_OFFLOAD_TCP_CKSUM) {
 569                                         ol_flags |= PKT_TX_TCP_CKSUM;
 570                                         tcp_hdr->cksum = get_ipv6_psd_sum(ipv6_hdr);
 571                                 }
 572                                 else {
 573                                         tcp_hdr->cksum = 0;
 574                                         tcp_hdr->cksum = get_ipv6_udptcp_checksum(ipv6_hdr,
 575                                                         (uint16_t*)tcp_hdr);
 576                                 }
 577                         }
 578                         else if (l4_proto == IPPROTO_SCTP) {
 579                                 sctp_hdr = (struct sctp_hdr*) (rte_pktmbuf_mtod(mb,
 580                                                 unsigned char *) + l2_len + l3_len);
 581
 582                                 if (tx_ol_flags & TESTPMD_TX_OFFLOAD_SCTP_CKSUM) {
 583                                         ol_flags |= PKT_TX_SCTP_CKSUM;
 584                                         sctp_hdr->cksum = 0;
 585                                         /* Sanity check, only number of 4 bytes supported by HW */
 586                                         if ((rte_be_to_cpu_16(ipv6_hdr->payload_len) % 4) != 0)
 587                                                 printf("sctp payload must be a multiple "
 588                                                         "of 4 bytes for checksum offload");
 589                                 }
 590                                 else {
 591                                         /* CRC32c sample code available in RFC3309 */
 592                                         sctp_hdr->cksum = 0;
 593                                 }
 594                         } else {
 595                                 printf("Test flow control for 1G PMD \n");
 596                         }
 597                         /* End of L6 Handling*/
 598                 }
 599                 else {
 600                         l3_len = 0;
 601                         printf("Unhandled packet type: %#hx\n", eth_type);
 602                 }
 603
 604                 /* Combine the packet header write. VLAN is not consider here */
 605                 mb->l2_len = l2_len;
 606                 mb->l3_len = l3_len;
 607                 mb->inner_l2_len = inner_l2_len;
 608                 mb->inner_l3_len = inner_l3_len;
 609                 mb->ol_flags = ol_flags;
 610         }
 611         nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx);
 612         fs->tx_packets += nb_tx;
 613         fs->rx_bad_ip_csum += rx_bad_ip_csum;
 614         fs->rx_bad_l4_csum += rx_bad_l4_csum;
 615
 616 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 617         fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
 618 #endif
 619         if (unlikely(nb_tx < nb_rx)) {
 620                 fs->fwd_dropped += (nb_rx - nb_tx);
 621                 do {
 622                         rte_pktmbuf_free(pkts_burst[nb_tx]);
 623                 } while (++nb_tx < nb_rx);
 624         }
 625 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 626         end_tsc = rte_rdtsc();
 627         core_cycles = (end_tsc - start_tsc);
 628         fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles);
 629 #endif
 630 }
 631
 632
 633 struct fwd_engine csum_fwd_engine = {
 634         .fwd_mode_name  = "csum",
 635         .port_fwd_begin = NULL,
 636         .port_fwd_end   = NULL,
 637         .packet_fwd     = pkt_burst_checksum_forward,
 638 };
 639