lib/net/net_crc_neon.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Cavium, Inc
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_branch_prediction.h>
   9 #include <rte_net_crc.h>
  10 #include <rte_vect.h>
  11 #include <rte_cpuflags.h>
  12
  13 #include "net_crc.h"
  14
  15 /** PMULL CRC computation context structure */
  16 struct crc_pmull_ctx {
  17         uint64x2_t rk1_rk2;
  18         uint64x2_t rk5_rk6;
  19         uint64x2_t rk7_rk8;
  20 };
  21
  22 struct crc_pmull_ctx crc32_eth_pmull __rte_aligned(16);
  23 struct crc_pmull_ctx crc16_ccitt_pmull __rte_aligned(16);
  24
  25 /**
  26  * @brief Performs one folding round
  27  *
  28  * Logically function operates as follows:
  29  *     DATA = READ_NEXT_16BYTES();
  30  *     F1 = LSB8(FOLD)
  31  *     F2 = MSB8(FOLD)
  32  *     T1 = CLMUL(F1, RK1)
  33  *     T2 = CLMUL(F2, RK2)
  34  *     FOLD = XOR(T1, T2, DATA)
  35  *
  36  * @param data_block 16 byte data block
  37  * @param precomp precomputed rk1 constant
  38  * @param fold running 16 byte folded data
  39  *
  40  * @return New 16 byte folded data
  41  */
  42 static inline uint64x2_t
  43 crcr32_folding_round(uint64x2_t data_block, uint64x2_t precomp,
  44         uint64x2_t fold)
  45 {
  46         uint64x2_t tmp0 = vreinterpretq_u64_p128(vmull_p64(
  47                         vgetq_lane_p64(vreinterpretq_p64_u64(fold), 1),
  48                         vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
  49
  50         uint64x2_t tmp1 = vreinterpretq_u64_p128(vmull_p64(
  51                         vgetq_lane_p64(vreinterpretq_p64_u64(fold), 0),
  52                         vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
  53
  54         return veorq_u64(tmp1, veorq_u64(data_block, tmp0));
  55 }
  56
  57 /**
  58  * Performs reduction from 128 bits to 64 bits
  59  *
  60  * @param data128 128 bits data to be reduced
  61  * @param precomp rk5 and rk6 precomputed constants
  62  *
  63  * @return data reduced to 64 bits
  64  */
  65 static inline uint64x2_t
  66 crcr32_reduce_128_to_64(uint64x2_t data128,
  67         uint64x2_t precomp)
  68 {
  69         uint64x2_t tmp0, tmp1, tmp2;
  70
  71         /* 64b fold */
  72         tmp0 = vreinterpretq_u64_p128(vmull_p64(
  73                 vgetq_lane_p64(vreinterpretq_p64_u64(data128), 0),
  74                 vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
  75         tmp1 = vshift_bytes_right(data128, 8);
  76         tmp0 = veorq_u64(tmp0, tmp1);
  77
  78         /* 32b fold */
  79         tmp2 = vshift_bytes_left(tmp0, 4);
  80         tmp1 = vreinterpretq_u64_p128(vmull_p64(
  81                 vgetq_lane_p64(vreinterpretq_p64_u64(tmp2), 0),
  82                 vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
  83
  84         return veorq_u64(tmp1, tmp0);
  85 }
  86
  87 /**
  88  * Performs Barret's reduction from 64 bits to 32 bits
  89  *
  90  * @param data64 64 bits data to be reduced
  91  * @param precomp rk7 precomputed constant
  92  *
  93  * @return data reduced to 32 bits
  94  */
  95 static inline uint32_t
  96 crcr32_reduce_64_to_32(uint64x2_t data64,
  97         uint64x2_t precomp)
  98 {
  99         static uint32_t mask1[4] __rte_aligned(16) = {
 100                 0xffffffff, 0xffffffff, 0x00000000, 0x00000000
 101         };
 102         static uint32_t mask2[4] __rte_aligned(16) = {
 103                 0x00000000, 0xffffffff, 0xffffffff, 0xffffffff
 104         };
 105         uint64x2_t tmp0, tmp1, tmp2;
 106
 107         tmp0 = vandq_u64(data64, vld1q_u64((uint64_t *)mask2));
 108
 109         tmp1 = vreinterpretq_u64_p128(vmull_p64(
 110                 vgetq_lane_p64(vreinterpretq_p64_u64(tmp0), 0),
 111                 vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 0)));
 112         tmp1 = veorq_u64(tmp1, tmp0);
 113         tmp1 = vandq_u64(tmp1, vld1q_u64((uint64_t *)mask1));
 114
 115         tmp2 = vreinterpretq_u64_p128(vmull_p64(
 116                 vgetq_lane_p64(vreinterpretq_p64_u64(tmp1), 0),
 117                 vgetq_lane_p64(vreinterpretq_p64_u64(precomp), 1)));
 118         tmp2 = veorq_u64(tmp2, tmp1);
 119         tmp2 = veorq_u64(tmp2, tmp0);
 120
 121         return vgetq_lane_u32(vreinterpretq_u32_u64(tmp2), 2);
 122 }
 123
 124 static inline uint32_t
 125 crc32_eth_calc_pmull(
 126         const uint8_t *data,
 127         uint32_t data_len,
 128         uint32_t crc,
 129         const struct crc_pmull_ctx *params)
 130 {
 131         uint64x2_t temp, fold, k;
 132         uint32_t n;
 133
 134         /* Get CRC init value */
 135         temp = vreinterpretq_u64_u32(vsetq_lane_u32(crc, vmovq_n_u32(0), 0));
 136
 137         /**
 138          * Folding all data into single 16 byte data block
 139          * Assumes: fold holds first 16 bytes of data
 140          */
 141         if (unlikely(data_len < 32)) {
 142                 if (unlikely(data_len == 16)) {
 143                         /* 16 bytes */
 144                         fold = vld1q_u64((const uint64_t *)data);
 145                         fold = veorq_u64(fold, temp);
 146                         goto reduction_128_64;
 147                 }
 148
 149                 if (unlikely(data_len < 16)) {
 150                         /* 0 to 15 bytes */
 151                         uint8_t buffer[16] __rte_aligned(16);
 152
 153                         memset(buffer, 0, sizeof(buffer));
 154                         memcpy(buffer, data, data_len);
 155
 156                         fold = vld1q_u64((uint64_t *)buffer);
 157                         fold = veorq_u64(fold, temp);
 158                         if (unlikely(data_len < 4)) {
 159                                 fold = vshift_bytes_left(fold, 8 - data_len);
 160                                 goto barret_reduction;
 161                         }
 162                         fold = vshift_bytes_left(fold, 16 - data_len);
 163                         goto reduction_128_64;
 164                 }
 165                 /* 17 to 31 bytes */
 166                 fold = vld1q_u64((const uint64_t *)data);
 167                 fold = veorq_u64(fold, temp);
 168                 n = 16;
 169                 k = params->rk1_rk2;
 170                 goto partial_bytes;
 171         }
 172
 173         /** At least 32 bytes in the buffer */
 174         /** Apply CRC initial value */
 175         fold = vld1q_u64((const uint64_t *)data);
 176         fold = veorq_u64(fold, temp);
 177
 178         /** Main folding loop - the last 16 bytes is processed separately */
 179         k = params->rk1_rk2;
 180         for (n = 16; (n + 16) <= data_len; n += 16) {
 181                 temp = vld1q_u64((const uint64_t *)&data[n]);
 182                 fold = crcr32_folding_round(temp, k, fold);
 183         }
 184
 185 partial_bytes:
 186         if (likely(n < data_len)) {
 187                 uint64x2_t last16, a, b, mask;
 188                 uint32_t rem = data_len & 15;
 189
 190                 last16 = vld1q_u64((const uint64_t *)&data[data_len - 16]);
 191                 a = vshift_bytes_left(fold, 16 - rem);
 192                 b = vshift_bytes_right(fold, rem);
 193                 mask = vshift_bytes_left(vdupq_n_u64(-1), 16 - rem);
 194                 b = vorrq_u64(b, vandq_u64(mask, last16));
 195
 196                 /* k = rk1 & rk2 */
 197                 temp = vreinterpretq_u64_p128(vmull_p64(
 198                                 vgetq_lane_p64(vreinterpretq_p64_u64(a), 1),
 199                                 vgetq_lane_p64(vreinterpretq_p64_u64(k), 0)));
 200                 fold = vreinterpretq_u64_p128(vmull_p64(
 201                                 vgetq_lane_p64(vreinterpretq_p64_u64(a), 0),
 202                                 vgetq_lane_p64(vreinterpretq_p64_u64(k), 1)));
 203                 fold = veorq_u64(fold, temp);
 204                 fold = veorq_u64(fold, b);
 205         }
 206
 207         /** Reduction 128 -> 32 Assumes: fold holds 128bit folded data */
 208 reduction_128_64:
 209         k = params->rk5_rk6;
 210         fold = crcr32_reduce_128_to_64(fold, k);
 211
 212 barret_reduction:
 213         k = params->rk7_rk8;
 214         n = crcr32_reduce_64_to_32(fold, k);
 215
 216         return n;
 217 }
 218
 219 void
 220 rte_net_crc_neon_init(void)
 221 {
 222         /* Initialize CRC16 data */
 223         uint64_t ccitt_k1_k2[2] = {0x189aeLLU, 0x8e10LLU};
 224         uint64_t ccitt_k5_k6[2] = {0x189aeLLU, 0x114aaLLU};
 225         uint64_t ccitt_k7_k8[2] = {0x11c581910LLU, 0x10811LLU};
 226
 227         /* Initialize CRC32 data */
 228         uint64_t eth_k1_k2[2] = {0xccaa009eLLU, 0x1751997d0LLU};
 229         uint64_t eth_k5_k6[2] = {0xccaa009eLLU, 0x163cd6124LLU};
 230         uint64_t eth_k7_k8[2] = {0x1f7011640LLU, 0x1db710641LLU};
 231
 232         /** Save the params in context structure */
 233         crc16_ccitt_pmull.rk1_rk2 = vld1q_u64(ccitt_k1_k2);
 234         crc16_ccitt_pmull.rk5_rk6 = vld1q_u64(ccitt_k5_k6);
 235         crc16_ccitt_pmull.rk7_rk8 = vld1q_u64(ccitt_k7_k8);
 236
 237         /** Save the params in context structure */
 238         crc32_eth_pmull.rk1_rk2 = vld1q_u64(eth_k1_k2);
 239         crc32_eth_pmull.rk5_rk6 = vld1q_u64(eth_k5_k6);
 240         crc32_eth_pmull.rk7_rk8 = vld1q_u64(eth_k7_k8);
 241 }
 242
 243 uint32_t
 244 rte_crc16_ccitt_neon_handler(const uint8_t *data, uint32_t data_len)
 245 {
 246         return (uint16_t)~crc32_eth_calc_pmull(data,
 247                 data_len,
 248                 0xffff,
 249                 &crc16_ccitt_pmull);
 250 }
 251
 252 uint32_t
 253 rte_crc32_eth_neon_handler(const uint8_t *data, uint32_t data_len)
 254 {
 255         return ~crc32_eth_calc_pmull(data,
 256                 data_len,
 257                 0xffffffffUL,
 258                 &crc32_eth_pmull);
 259 }