lib/librte_acl/acl_run_sse.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include "acl_run.h"
   6 #include "acl_vect.h"
   7
   8 enum {
   9         SHUFFLE32_SLOT1 = 0xe5,
  10         SHUFFLE32_SLOT2 = 0xe6,
  11         SHUFFLE32_SLOT3 = 0xe7,
  12         SHUFFLE32_SWAP64 = 0x4e,
  13 };
  14
  15 static const rte_xmm_t xmm_shuffle_input = {
  16         .u32 = {0x00000000, 0x04040404, 0x08080808, 0x0c0c0c0c},
  17 };
  18
  19 static const rte_xmm_t xmm_ones_16 = {
  20         .u16 = {1, 1, 1, 1, 1, 1, 1, 1},
  21 };
  22
  23 static const rte_xmm_t xmm_match_mask = {
  24         .u32 = {
  25                 RTE_ACL_NODE_MATCH,
  26                 RTE_ACL_NODE_MATCH,
  27                 RTE_ACL_NODE_MATCH,
  28                 RTE_ACL_NODE_MATCH,
  29         },
  30 };
  31
  32 static const rte_xmm_t xmm_index_mask = {
  33         .u32 = {
  34                 RTE_ACL_NODE_INDEX,
  35                 RTE_ACL_NODE_INDEX,
  36                 RTE_ACL_NODE_INDEX,
  37                 RTE_ACL_NODE_INDEX,
  38         },
  39 };
  40
  41 static const rte_xmm_t xmm_range_base = {
  42         .u32 = {
  43                 0xffffff00, 0xffffff04, 0xffffff08, 0xffffff0c,
  44         },
  45 };
  46
  47 /*
  48  * Resolve priority for multiple results (sse version).
  49  * This consists comparing the priority of the current traversal with the
  50  * running set of results for the packet.
  51  * For each result, keep a running array of the result (rule number) and
  52  * its priority for each category.
  53  */
  54 static inline void
  55 resolve_priority_sse(uint64_t transition, int n, const struct rte_acl_ctx *ctx,
  56         struct parms *parms, const struct rte_acl_match_results *p,
  57         uint32_t categories)
  58 {
  59         uint32_t x;
  60         xmm_t results, priority, results1, priority1, selector;
  61         xmm_t *saved_results, *saved_priority;
  62
  63         for (x = 0; x < categories; x += RTE_ACL_RESULTS_MULTIPLIER) {
  64
  65                 saved_results = (xmm_t *)(&parms[n].cmplt->results[x]);
  66                 saved_priority =
  67                         (xmm_t *)(&parms[n].cmplt->priority[x]);
  68
  69                 /* get results and priorities for completed trie */
  70                 results = _mm_loadu_si128(
  71                         (const xmm_t *)&p[transition].results[x]);
  72                 priority = _mm_loadu_si128(
  73                         (const xmm_t *)&p[transition].priority[x]);
  74
  75                 /* if this is not the first completed trie */
  76                 if (parms[n].cmplt->count != ctx->num_tries) {
  77
  78                         /* get running best results and their priorities */
  79                         results1 = _mm_loadu_si128(saved_results);
  80                         priority1 = _mm_loadu_si128(saved_priority);
  81
  82                         /* select results that are highest priority */
  83                         selector = _mm_cmpgt_epi32(priority1, priority);
  84                         results = _mm_blendv_epi8(results, results1, selector);
  85                         priority = _mm_blendv_epi8(priority, priority1,
  86                                 selector);
  87                 }
  88
  89                 /* save running best results and their priorities */
  90                 _mm_storeu_si128(saved_results, results);
  91                 _mm_storeu_si128(saved_priority, priority);
  92         }
  93 }
  94
  95 /*
  96  * Extract transitions from an XMM register and check for any matches
  97  */
  98 static void
  99 acl_process_matches(xmm_t *indices, int slot, const struct rte_acl_ctx *ctx,
 100         struct parms *parms, struct acl_flow_data *flows)
 101 {
 102         uint64_t transition1, transition2;
 103
 104         /* extract transition from low 64 bits. */
 105         transition1 = _mm_cvtsi128_si64(*indices);
 106
 107         /* extract transition from high 64 bits. */
 108         *indices = _mm_shuffle_epi32(*indices, SHUFFLE32_SWAP64);
 109         transition2 = _mm_cvtsi128_si64(*indices);
 110
 111         transition1 = acl_match_check(transition1, slot, ctx,
 112                 parms, flows, resolve_priority_sse);
 113         transition2 = acl_match_check(transition2, slot + 1, ctx,
 114                 parms, flows, resolve_priority_sse);
 115
 116         /* update indices with new transitions. */
 117         *indices = _mm_set_epi64x(transition2, transition1);
 118 }
 119
 120 /*
 121  * Check for any match in 4 transitions (contained in 2 SSE registers)
 122  */
 123 static __rte_always_inline void
 124 acl_match_check_x4(int slot, const struct rte_acl_ctx *ctx, struct parms *parms,
 125         struct acl_flow_data *flows, xmm_t *indices1, xmm_t *indices2,
 126         xmm_t match_mask)
 127 {
 128         xmm_t temp;
 129
 130         /* put low 32 bits of each transition into one register */
 131         temp = (xmm_t)_mm_shuffle_ps((__m128)*indices1, (__m128)*indices2,
 132                 0x88);
 133         /* test for match node */
 134         temp = _mm_and_si128(match_mask, temp);
 135
 136         while (!_mm_testz_si128(temp, temp)) {
 137                 acl_process_matches(indices1, slot, ctx, parms, flows);
 138                 acl_process_matches(indices2, slot + 2, ctx, parms, flows);
 139
 140                 temp = (xmm_t)_mm_shuffle_ps((__m128)*indices1,
 141                                         (__m128)*indices2,
 142                                         0x88);
 143                 temp = _mm_and_si128(match_mask, temp);
 144         }
 145 }
 146
 147 /*
 148  * Process 4 transitions (in 2 XMM registers) in parallel
 149  */
 150 static __rte_always_inline xmm_t
 151 transition4(xmm_t next_input, const uint64_t *trans,
 152         xmm_t *indices1, xmm_t *indices2)
 153 {
 154         xmm_t addr, tr_lo, tr_hi;
 155         uint64_t trans0, trans2;
 156
 157         /* Shuffle low 32 into tr_lo and high 32 into tr_hi */
 158         ACL_TR_HILO(mm, __m128, *indices1, *indices2, tr_lo, tr_hi);
 159
 160          /* Calculate the address (array index) for all 4 transitions. */
 161         ACL_TR_CALC_ADDR(mm, 128, addr, xmm_index_mask.x, next_input,
 162                 xmm_shuffle_input.x, xmm_ones_16.x, xmm_range_base.x,
 163                 tr_lo, tr_hi);
 164
 165          /* Gather 64 bit transitions and pack back into 2 registers. */
 166
 167         trans0 = trans[_mm_cvtsi128_si32(addr)];
 168
 169         /* get slot 2 */
 170
 171         /* {x0, x1, x2, x3} -> {x2, x1, x2, x3} */
 172         addr = _mm_shuffle_epi32(addr, SHUFFLE32_SLOT2);
 173         trans2 = trans[_mm_cvtsi128_si32(addr)];
 174
 175         /* get slot 1 */
 176
 177         /* {x2, x1, x2, x3} -> {x1, x1, x2, x3} */
 178         addr = _mm_shuffle_epi32(addr, SHUFFLE32_SLOT1);
 179         *indices1 = _mm_set_epi64x(trans[_mm_cvtsi128_si32(addr)], trans0);
 180
 181         /* get slot 3 */
 182
 183         /* {x1, x1, x2, x3} -> {x3, x1, x2, x3} */
 184         addr = _mm_shuffle_epi32(addr, SHUFFLE32_SLOT3);
 185         *indices2 = _mm_set_epi64x(trans[_mm_cvtsi128_si32(addr)], trans2);
 186
 187         return _mm_srli_epi32(next_input, CHAR_BIT);
 188 }
 189
 190 /*
 191  * Execute trie traversal with 8 traversals in parallel
 192  */
 193 static inline int
 194 search_sse_8(const struct rte_acl_ctx *ctx, const uint8_t **data,
 195         uint32_t *results, uint32_t total_packets, uint32_t categories)
 196 {
 197         int n;
 198         struct acl_flow_data flows;
 199         uint64_t index_array[MAX_SEARCHES_SSE8];
 200         struct completion cmplt[MAX_SEARCHES_SSE8];
 201         struct parms parms[MAX_SEARCHES_SSE8];
 202         xmm_t input0, input1;
 203         xmm_t indices1, indices2, indices3, indices4;
 204
 205         acl_set_flow(&flows, cmplt, RTE_DIM(cmplt), data, results,
 206                 total_packets, categories, ctx->trans_table);
 207
 208         for (n = 0; n < MAX_SEARCHES_SSE8; n++) {
 209                 cmplt[n].count = 0;
 210                 index_array[n] = acl_start_next_trie(&flows, parms, n, ctx);
 211         }
 212
 213         /*
 214          * indices1 contains index_array[0,1]
 215          * indices2 contains index_array[2,3]
 216          * indices3 contains index_array[4,5]
 217          * indices4 contains index_array[6,7]
 218          */
 219
 220         indices1 = _mm_loadu_si128((xmm_t *) &index_array[0]);
 221         indices2 = _mm_loadu_si128((xmm_t *) &index_array[2]);
 222
 223         indices3 = _mm_loadu_si128((xmm_t *) &index_array[4]);
 224         indices4 = _mm_loadu_si128((xmm_t *) &index_array[6]);
 225
 226          /* Check for any matches. */
 227         acl_match_check_x4(0, ctx, parms, &flows,
 228                 &indices1, &indices2, xmm_match_mask.x);
 229         acl_match_check_x4(4, ctx, parms, &flows,
 230                 &indices3, &indices4, xmm_match_mask.x);
 231
 232         while (flows.started > 0) {
 233
 234                 /* Gather 4 bytes of input data for each stream. */
 235                 input0 = _mm_cvtsi32_si128(GET_NEXT_4BYTES(parms, 0));
 236                 input1 = _mm_cvtsi32_si128(GET_NEXT_4BYTES(parms, 4));
 237
 238                 input0 = _mm_insert_epi32(input0, GET_NEXT_4BYTES(parms, 1), 1);
 239                 input1 = _mm_insert_epi32(input1, GET_NEXT_4BYTES(parms, 5), 1);
 240
 241                 input0 = _mm_insert_epi32(input0, GET_NEXT_4BYTES(parms, 2), 2);
 242                 input1 = _mm_insert_epi32(input1, GET_NEXT_4BYTES(parms, 6), 2);
 243
 244                 input0 = _mm_insert_epi32(input0, GET_NEXT_4BYTES(parms, 3), 3);
 245                 input1 = _mm_insert_epi32(input1, GET_NEXT_4BYTES(parms, 7), 3);
 246
 247                  /* Process the 4 bytes of input on each stream. */
 248
 249                 input0 = transition4(input0, flows.trans,
 250                         &indices1, &indices2);
 251                 input1 = transition4(input1, flows.trans,
 252                         &indices3, &indices4);
 253
 254                 input0 = transition4(input0, flows.trans,
 255                         &indices1, &indices2);
 256                 input1 = transition4(input1, flows.trans,
 257                         &indices3, &indices4);
 258
 259                 input0 = transition4(input0, flows.trans,
 260                         &indices1, &indices2);
 261                 input1 = transition4(input1, flows.trans,
 262                         &indices3, &indices4);
 263
 264                 input0 = transition4(input0, flows.trans,
 265                         &indices1, &indices2);
 266                 input1 = transition4(input1, flows.trans,
 267                         &indices3, &indices4);
 268
 269                  /* Check for any matches. */
 270                 acl_match_check_x4(0, ctx, parms, &flows,
 271                         &indices1, &indices2, xmm_match_mask.x);
 272                 acl_match_check_x4(4, ctx, parms, &flows,
 273                         &indices3, &indices4, xmm_match_mask.x);
 274         }
 275
 276         return 0;
 277 }
 278
 279 /*
 280  * Execute trie traversal with 4 traversals in parallel
 281  */
 282 static inline int
 283 search_sse_4(const struct rte_acl_ctx *ctx, const uint8_t **data,
 284          uint32_t *results, int total_packets, uint32_t categories)
 285 {
 286         int n;
 287         struct acl_flow_data flows;
 288         uint64_t index_array[MAX_SEARCHES_SSE4];
 289         struct completion cmplt[MAX_SEARCHES_SSE4];
 290         struct parms parms[MAX_SEARCHES_SSE4];
 291         xmm_t input, indices1, indices2;
 292
 293         acl_set_flow(&flows, cmplt, RTE_DIM(cmplt), data, results,
 294                 total_packets, categories, ctx->trans_table);
 295
 296         for (n = 0; n < MAX_SEARCHES_SSE4; n++) {
 297                 cmplt[n].count = 0;
 298                 index_array[n] = acl_start_next_trie(&flows, parms, n, ctx);
 299         }
 300
 301         indices1 = _mm_loadu_si128((xmm_t *) &index_array[0]);
 302         indices2 = _mm_loadu_si128((xmm_t *) &index_array[2]);
 303
 304         /* Check for any matches. */
 305         acl_match_check_x4(0, ctx, parms, &flows,
 306                 &indices1, &indices2, xmm_match_mask.x);
 307
 308         while (flows.started > 0) {
 309
 310                 /* Gather 4 bytes of input data for each stream. */
 311                 input = _mm_cvtsi32_si128(GET_NEXT_4BYTES(parms, 0));
 312                 input = _mm_insert_epi32(input, GET_NEXT_4BYTES(parms, 1), 1);
 313                 input = _mm_insert_epi32(input, GET_NEXT_4BYTES(parms, 2), 2);
 314                 input = _mm_insert_epi32(input, GET_NEXT_4BYTES(parms, 3), 3);
 315
 316                 /* Process the 4 bytes of input on each stream. */
 317                 input = transition4(input, flows.trans, &indices1, &indices2);
 318                 input = transition4(input, flows.trans, &indices1, &indices2);
 319                 input = transition4(input, flows.trans, &indices1, &indices2);
 320                 input = transition4(input, flows.trans, &indices1, &indices2);
 321
 322                 /* Check for any matches. */
 323                 acl_match_check_x4(0, ctx, parms, &flows,
 324                         &indices1, &indices2, xmm_match_mask.x);
 325         }
 326
 327         return 0;
 328 }