drivers/common/sfc_efx/base/efx_types.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2019-2020 Xilinx, Inc.
   4  * Copyright(c) 2007-2019 Solarflare Communications Inc.
   5  *
   6  * Ackowledgement to Fen Systems Ltd.
   7  */
   8
   9 #ifndef _SYS_EFX_TYPES_H
  10 #define _SYS_EFX_TYPES_H
  11
  12 #include "efsys.h"
  13 #include "efx_annote.h"
  14
  15 #ifdef  __cplusplus
  16 extern "C" {
  17 #endif
  18
  19 /*
  20  * Bitfield access
  21  *
  22  * Solarflare NICs make extensive use of bitfields up to 128 bits
  23  * wide.  Since there is no native 128-bit datatype on most systems,
  24  * and since 64-bit datatypes are inefficient on 32-bit systems and
  25  * vice versa, we wrap accesses in a way that uses the most efficient
  26  * datatype.
  27  *
  28  * The NICs are PCI devices and therefore little-endian.  Since most
  29  * of the quantities that we deal with are DMAed to/from host memory,
  30  * we define    our datatypes (efx_oword_t, efx_qword_t and efx_dword_t)
  31  * to be little-endian.
  32  *
  33  * In the less common case of using PIO for individual register
  34  * writes, we construct the little-endian datatype in host memory and
  35  * then use non-swapping register access primitives, rather than
  36  * constructing a native-endian datatype and relying on implicit
  37  * byte-swapping.  (We use a similar strategy for register reads.)
  38  */
  39
  40 /*
  41  * NOTE: Field definitions here and elsewhere are done in terms of a lowest
  42  *       bit number (LBN) and a width.
  43  */
  44
  45 #define EFX_DUMMY_FIELD_LBN 0
  46 #define EFX_DUMMY_FIELD_WIDTH 0
  47
  48 #define EFX_BYTE_0_LBN 0
  49 #define EFX_BYTE_0_WIDTH 8
  50
  51 #define EFX_BYTE_1_LBN 8
  52 #define EFX_BYTE_1_WIDTH 8
  53
  54 #define EFX_BYTE_2_LBN 16
  55 #define EFX_BYTE_2_WIDTH 8
  56
  57 #define EFX_BYTE_3_LBN 24
  58 #define EFX_BYTE_3_WIDTH 8
  59
  60 #define EFX_BYTE_4_LBN 32
  61 #define EFX_BYTE_4_WIDTH 8
  62
  63 #define EFX_BYTE_5_LBN 40
  64 #define EFX_BYTE_5_WIDTH 8
  65
  66 #define EFX_BYTE_6_LBN 48
  67 #define EFX_BYTE_6_WIDTH 8
  68
  69 #define EFX_BYTE_7_LBN 56
  70 #define EFX_BYTE_7_WIDTH 8
  71
  72 #define EFX_WORD_0_LBN 0
  73 #define EFX_WORD_0_WIDTH 16
  74
  75 #define EFX_WORD_1_LBN 16
  76 #define EFX_WORD_1_WIDTH 16
  77
  78 #define EFX_WORD_2_LBN 32
  79 #define EFX_WORD_2_WIDTH 16
  80
  81 #define EFX_WORD_3_LBN 48
  82 #define EFX_WORD_3_WIDTH 16
  83
  84 #define EFX_DWORD_0_LBN 0
  85 #define EFX_DWORD_0_WIDTH 32
  86
  87 #define EFX_DWORD_1_LBN 32
  88 #define EFX_DWORD_1_WIDTH 32
  89
  90 #define EFX_DWORD_2_LBN 64
  91 #define EFX_DWORD_2_WIDTH 32
  92
  93 #define EFX_DWORD_3_LBN 96
  94 #define EFX_DWORD_3_WIDTH 32
  95
  96 /*
  97  * There are intentionally no EFX_QWORD_0 or EFX_QWORD_1 field definitions
  98  * here as the implementaion of EFX_QWORD_FIELD and EFX_OWORD_FIELD do not
  99  * support field widths larger than 32 bits.
 100  */
 101
 102 /* Specified attribute (i.e. LBN ow WIDTH) of the specified field */
 103 #define EFX_VAL(_field, _attribute)                                     \
 104         _field ## _ ## _attribute
 105
 106 /* Lowest bit number of the specified field */
 107 #define EFX_LOW_BIT(_field)                                             \
 108         EFX_VAL(_field, LBN)
 109
 110 /* Width of the specified field */
 111 #define EFX_WIDTH(_field)                                               \
 112         EFX_VAL(_field, WIDTH)
 113
 114 /* Highest bit number of the specified field */
 115 #define EFX_HIGH_BIT(_field)                                            \
 116         (EFX_LOW_BIT(_field) + EFX_WIDTH(_field) - 1)
 117
 118 /*
 119  * 64-bit mask equal in width to the specified field.
 120  *
 121  * For example, a field with width 5 would have a mask of 0x000000000000001f.
 122  */
 123 #define EFX_MASK64(_field)                                              \
 124         ((EFX_WIDTH(_field) == 64) ? ~((uint64_t)0) :                   \
 125             (((((uint64_t)1) << EFX_WIDTH(_field))) - 1))
 126 /*
 127  * 32-bit mask equal in width to the specified field.
 128  *
 129  * For example, a field with width 5 would have a mask of 0x0000001f.
 130  */
 131 #define EFX_MASK32(_field)                                              \
 132         ((EFX_WIDTH(_field) == 32) ? ~((uint32_t)0) :                   \
 133             (((((uint32_t)1) << EFX_WIDTH(_field))) - 1))
 134
 135 /*
 136  * 16-bit mask equal in width to the specified field.
 137  *
 138  * For example, a field with width 5 would have a mask of 0x001f.
 139  */
 140 #define EFX_MASK16(_field)                                              \
 141         ((EFX_WIDTH(_field) == 16) ? 0xffffu :                          \
 142             (uint16_t)((1 << EFX_WIDTH(_field)) - 1))
 143
 144 /*
 145  * 8-bit mask equal in width to the specified field.
 146  *
 147  * For example, a field with width 5 would have a mask of 0x1f.
 148  */
 149 #define EFX_MASK8(_field)                                               \
 150         ((uint8_t)((1 << EFX_WIDTH(_field)) - 1))
 151
 152 #pragma pack(1)
 153
 154 /*
 155  * A byte (i.e. 8-bit) datatype
 156  */
 157 typedef union efx_byte_u {
 158         uint8_t eb_u8[1];
 159 } efx_byte_t;
 160
 161 /*
 162  * A word (i.e. 16-bit) datatype
 163  *
 164  * This datatype is defined to be little-endian.
 165  */
 166 typedef union efx_word_u {
 167         efx_byte_t ew_byte[2];
 168         uint16_t ew_u16[1];
 169         uint8_t ew_u8[2];
 170 } efx_word_t;
 171
 172 /*
 173  * A doubleword (i.e. 32-bit) datatype
 174  *
 175  * This datatype is defined to be little-endian.
 176  */
 177 typedef union efx_dword_u {
 178         efx_byte_t ed_byte[4];
 179         efx_word_t ed_word[2];
 180         uint32_t ed_u32[1];
 181         uint16_t ed_u16[2];
 182         uint8_t ed_u8[4];
 183 } efx_dword_t;
 184
 185 /*
 186  * A quadword (i.e. 64-bit) datatype
 187  *
 188  * This datatype is defined to be little-endian.
 189  */
 190 typedef union efx_qword_u {
 191         efx_byte_t eq_byte[8];
 192         efx_word_t eq_word[4];
 193         efx_dword_t eq_dword[2];
 194 #if EFSYS_HAS_UINT64
 195         uint64_t eq_u64[1];
 196 #endif
 197         uint32_t eq_u32[2];
 198         uint16_t eq_u16[4];
 199         uint8_t eq_u8[8];
 200 } efx_qword_t;
 201
 202 /*
 203  * An octword (i.e. 128-bit) datatype
 204  *
 205  * This datatype is defined to be little-endian.
 206  */
 207 typedef union efx_oword_u {
 208         efx_byte_t eo_byte[16];
 209         efx_word_t eo_word[8];
 210         efx_dword_t eo_dword[4];
 211         efx_qword_t eo_qword[2];
 212 #if EFSYS_HAS_SSE2_M128
 213         __m128i eo_u128[1];
 214 #endif
 215 #if EFSYS_HAS_UINT64
 216         uint64_t eo_u64[2];
 217 #endif
 218         uint32_t eo_u32[4];
 219         uint16_t eo_u16[8];
 220         uint8_t eo_u8[16];
 221 } efx_oword_t;
 222
 223 #pragma pack()
 224
 225 #define __SWAP16(_x)                            \
 226         ((((_x) & 0xff) << 8) |                 \
 227         (((_x) >> 8) & 0xff))
 228
 229 #define __SWAP32(_x)                            \
 230         ((__SWAP16((_x) & 0xffff) << 16) |      \
 231         __SWAP16(((_x) >> 16) & 0xffff))
 232
 233 #define __SWAP64(_x)                            \
 234         ((__SWAP32((_x) & 0xffffffff) << 32) |  \
 235         __SWAP32(((_x) >> 32) & 0xffffffff))
 236
 237 #define __NOSWAP16(_x)          (_x)
 238 #define __NOSWAP32(_x)          (_x)
 239 #define __NOSWAP64(_x)          (_x)
 240
 241 #if EFSYS_IS_BIG_ENDIAN
 242
 243 #define __CPU_TO_LE_16(_x)      ((uint16_t)__SWAP16(_x))
 244 #define __LE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 245 #define __CPU_TO_BE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 246 #define __BE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 247
 248 #define __CPU_TO_LE_32(_x)      ((uint32_t)__SWAP32(_x))
 249 #define __LE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 250 #define __CPU_TO_BE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 251 #define __BE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 252
 253 #define __CPU_TO_LE_64(_x)      ((uint64_t)__SWAP64(_x))
 254 #define __LE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 255 #define __CPU_TO_BE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 256 #define __BE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 257
 258 #elif EFSYS_IS_LITTLE_ENDIAN
 259
 260 #define __CPU_TO_LE_16(_x)      ((uint16_t)__NOSWAP16(_x))
 261 #define __LE_TO_CPU_16(_x)      ((uint16_t)__NOSWAP16(_x))
 262 #define __CPU_TO_BE_16(_x)      ((uint16_t)__SWAP16(_x))
 263 #define __BE_TO_CPU_16(_x)      ((uint16_t)__SWAP16(_x))
 264
 265 #define __CPU_TO_LE_32(_x)      ((uint32_t)__NOSWAP32(_x))
 266 #define __LE_TO_CPU_32(_x)      ((uint32_t)__NOSWAP32(_x))
 267 #define __CPU_TO_BE_32(_x)      ((uint32_t)__SWAP32(_x))
 268 #define __BE_TO_CPU_32(_x)      ((uint32_t)__SWAP32(_x))
 269
 270 #define __CPU_TO_LE_64(_x)      ((uint64_t)__NOSWAP64(_x))
 271 #define __LE_TO_CPU_64(_x)      ((uint64_t)__NOSWAP64(_x))
 272 #define __CPU_TO_BE_64(_x)      ((uint64_t)__SWAP64(_x))
 273 #define __BE_TO_CPU_64(_x)      ((uint64_t)__SWAP64(_x))
 274
 275 #else
 276
 277 #error "Neither of EFSYS_IS_{BIG,LITTLE}_ENDIAN is set"
 278
 279 #endif
 280
 281 #define __NATIVE_8(_x)  (uint8_t)(_x)
 282
 283 /* Format string for printing an efx_byte_t */
 284 #define EFX_BYTE_FMT "0x%02x"
 285
 286 /* Format string for printing an efx_word_t */
 287 #define EFX_WORD_FMT "0x%04x"
 288
 289 /* Format string for printing an efx_dword_t */
 290 #define EFX_DWORD_FMT "0x%08x"
 291
 292 /* Format string for printing an efx_qword_t */
 293 #define EFX_QWORD_FMT "0x%08x:%08x"
 294
 295 /* Format string for printing an efx_oword_t */
 296 #define EFX_OWORD_FMT "0x%08x:%08x:%08x:%08x"
 297
 298 /* Parameters for printing an efx_byte_t */
 299 #define EFX_BYTE_VAL(_byte)                                     \
 300         ((unsigned int)__NATIVE_8((_byte).eb_u8[0]))
 301
 302 /* Parameters for printing an efx_word_t */
 303 #define EFX_WORD_VAL(_word)                                     \
 304         ((unsigned int)__LE_TO_CPU_16((_word).ew_u16[0]))
 305
 306 /* Parameters for printing an efx_dword_t */
 307 #define EFX_DWORD_VAL(_dword)                                   \
 308         ((unsigned int)__LE_TO_CPU_32((_dword).ed_u32[0]))
 309
 310 /* Parameters for printing an efx_qword_t */
 311 #define EFX_QWORD_VAL(_qword)                                   \
 312         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[1])),     \
 313         ((unsigned int)__LE_TO_CPU_32((_qword).eq_u32[0]))
 314
 315 /* Parameters for printing an efx_oword_t */
 316 #define EFX_OWORD_VAL(_oword)                                   \
 317         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[3])),     \
 318         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[2])),     \
 319         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[1])),     \
 320         ((unsigned int)__LE_TO_CPU_32((_oword).eo_u32[0]))
 321
 322 /*
 323  * Stop lint complaining about some shifts.
 324  */
 325 #ifdef  __lint
 326 extern int fix_lint;
 327 #define FIX_LINT(_x)    (_x + fix_lint)
 328 #else
 329 #define FIX_LINT(_x)    (_x)
 330 #endif
 331
 332 /*
 333  * Saturation arithmetic subtract with minimum equal to zero.
 334  *
 335  * Use saturating arithmetic to ensure a non-negative result. This
 336  * avoids undefined behaviour (and compiler warnings) when used as a
 337  * shift count.
 338  */
 339 #define EFX_SSUB(_val, _sub) \
 340         ((_val) > (_sub) ? ((_val) - (_sub)) : 0)
 341
 342 /*
 343  * Extract bit field portion [low,high) from the native-endian element
 344  * which contains bits [min,max).
 345  *
 346  * For example, suppose "element" represents the high 32 bits of a
 347  * 64-bit value, and we wish to extract the bits belonging to the bit
 348  * field occupying bits 28-45 of this 64-bit value.
 349  *
 350  * Then EFX_EXTRACT(_element, 32, 63, 28, 45) would give
 351  *
 352  *   (_element) << 4
 353  *
 354  * The result will contain the relevant bits filled in in the range
 355  * [0,high-low), with garbage in bits [high-low+1,...).
 356  */
 357 #define EFX_EXTRACT_NATIVE(_element, _min, _max, _low, _high)           \
 358         ((FIX_LINT(_low > _max) || FIX_LINT(_high < _min)) ?            \
 359                 0U :                                                    \
 360                 ((_low > _min) ?                                        \
 361                         ((_element) >> EFX_SSUB(_low, _min)) :          \
 362                         ((_element) << EFX_SSUB(_min, _low))))
 363
 364 /*
 365  * Extract bit field portion [low,high) from the 64-bit little-endian
 366  * element which contains bits [min,max)
 367  */
 368 #define EFX_EXTRACT64(_element, _min, _max, _low, _high)                \
 369         EFX_EXTRACT_NATIVE(__LE_TO_CPU_64(_element), _min, _max, _low, _high)
 370
 371 /*
 372  * Extract bit field portion [low,high) from the 32-bit little-endian
 373  * element which contains bits [min,max)
 374  */
 375 #define EFX_EXTRACT32(_element, _min, _max, _low, _high)                \
 376         EFX_EXTRACT_NATIVE(__LE_TO_CPU_32(_element), _min, _max, _low, _high)
 377
 378 /*
 379  * Extract bit field portion [low,high) from the 16-bit little-endian
 380  * element which contains bits [min,max)
 381  */
 382 #define EFX_EXTRACT16(_element, _min, _max, _low, _high)                \
 383         EFX_EXTRACT_NATIVE(__LE_TO_CPU_16(_element), _min, _max, _low, _high)
 384
 385 /*
 386  * Extract bit field portion [low,high) from the 8-bit
 387  * element which contains bits [min,max)
 388  */
 389 #define EFX_EXTRACT8(_element, _min, _max, _low, _high)                 \
 390         EFX_EXTRACT_NATIVE(__NATIVE_8(_element), _min, _max, _low, _high)
 391
 392 #define EFX_EXTRACT_OWORD64(_oword, _low, _high)                        \
 393         (EFX_EXTRACT64((_oword).eo_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 394             _low, _high) |                                              \
 395         EFX_EXTRACT64((_oword).eo_u64[1], FIX_LINT(64), FIX_LINT(127),  \
 396             _low, _high))
 397
 398 #define EFX_EXTRACT_OWORD32(_oword, _low, _high)                        \
 399         (EFX_EXTRACT32((_oword).eo_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 400             _low, _high) |                                              \
 401         EFX_EXTRACT32((_oword).eo_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 402             _low, _high) |                                              \
 403         EFX_EXTRACT32((_oword).eo_u32[2], FIX_LINT(64), FIX_LINT(95),   \
 404             _low, _high) |                                              \
 405         EFX_EXTRACT32((_oword).eo_u32[3], FIX_LINT(96), FIX_LINT(127),  \
 406             _low, _high))
 407
 408 #define EFX_EXTRACT_QWORD64(_qword, _low, _high)                        \
 409         (EFX_EXTRACT64((_qword).eq_u64[0], FIX_LINT(0), FIX_LINT(63),   \
 410             _low, _high))
 411
 412 #define EFX_EXTRACT_QWORD32(_qword, _low, _high)                        \
 413         (EFX_EXTRACT32((_qword).eq_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 414             _low, _high) |                                              \
 415         EFX_EXTRACT32((_qword).eq_u32[1], FIX_LINT(32), FIX_LINT(63),   \
 416             _low, _high))
 417
 418 #define EFX_EXTRACT_DWORD(_dword, _low, _high)                          \
 419         (EFX_EXTRACT32((_dword).ed_u32[0], FIX_LINT(0), FIX_LINT(31),   \
 420             _low, _high))
 421
 422 #define EFX_EXTRACT_WORD(_word, _low, _high)                            \
 423         (EFX_EXTRACT16((_word).ew_u16[0], FIX_LINT(0), FIX_LINT(15),    \
 424             _low, _high))
 425
 426 #define EFX_EXTRACT_BYTE(_byte, _low, _high)                            \
 427         (EFX_EXTRACT8((_byte).eb_u8[0], FIX_LINT(0), FIX_LINT(7),       \
 428             _low, _high))
 429
 430
 431 #define EFX_OWORD_FIELD64(_oword, _field)                               \
 432         ((uint32_t)EFX_EXTRACT_OWORD64(_oword, EFX_LOW_BIT(_field),     \
 433             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 434
 435 #define EFX_OWORD_FIELD32(_oword, _field)                               \
 436         (EFX_EXTRACT_OWORD32(_oword, EFX_LOW_BIT(_field),               \
 437             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 438
 439 #define EFX_QWORD_FIELD64(_qword, _field)                               \
 440         ((uint32_t)EFX_EXTRACT_QWORD64(_qword, EFX_LOW_BIT(_field),     \
 441             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 442
 443 #define EFX_QWORD_FIELD32(_qword, _field)                               \
 444         (EFX_EXTRACT_QWORD32(_qword, EFX_LOW_BIT(_field),               \
 445             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 446
 447 #define EFX_DWORD_FIELD(_dword, _field)                                 \
 448         (EFX_EXTRACT_DWORD(_dword, EFX_LOW_BIT(_field),                 \
 449             EFX_HIGH_BIT(_field)) & EFX_MASK32(_field))
 450
 451 #define EFX_WORD_FIELD(_word, _field)                                   \
 452         (EFX_EXTRACT_WORD(_word, EFX_LOW_BIT(_field),                   \
 453             EFX_HIGH_BIT(_field)) & EFX_MASK16(_field))
 454
 455 #define EFX_BYTE_FIELD(_byte, _field)                                   \
 456         (EFX_EXTRACT_BYTE(_byte, EFX_LOW_BIT(_field),                   \
 457             EFX_HIGH_BIT(_field)) & EFX_MASK8(_field))
 458
 459
 460 #define EFX_OWORD_IS_EQUAL64(_oword_a, _oword_b)                        \
 461         ((_oword_a).eo_u64[0] == (_oword_b).eo_u64[0] &&                \
 462             (_oword_a).eo_u64[1] == (_oword_b).eo_u64[1])
 463
 464 #define EFX_OWORD_IS_EQUAL32(_oword_a, _oword_b)                        \
 465         ((_oword_a).eo_u32[0] == (_oword_b).eo_u32[0] &&                \
 466             (_oword_a).eo_u32[1] == (_oword_b).eo_u32[1] &&             \
 467             (_oword_a).eo_u32[2] == (_oword_b).eo_u32[2] &&             \
 468             (_oword_a).eo_u32[3] == (_oword_b).eo_u32[3])
 469
 470 #define EFX_QWORD_IS_EQUAL64(_qword_a, _qword_b)                        \
 471         ((_qword_a).eq_u64[0] == (_qword_b).eq_u64[0])
 472
 473 #define EFX_QWORD_IS_EQUAL32(_qword_a, _qword_b)                        \
 474         ((_qword_a).eq_u32[0] == (_qword_b).eq_u32[0] &&                \
 475             (_qword_a).eq_u32[1] == (_qword_b).eq_u32[1])
 476
 477 #define EFX_DWORD_IS_EQUAL(_dword_a, _dword_b)                          \
 478         ((_dword_a).ed_u32[0] == (_dword_b).ed_u32[0])
 479
 480 #define EFX_WORD_IS_EQUAL(_word_a, _word_b)                             \
 481         ((_word_a).ew_u16[0] == (_word_b).ew_u16[0])
 482
 483 #define EFX_BYTE_IS_EQUAL(_byte_a, _byte_b)                             \
 484         ((_byte_a).eb_u8[0] == (_byte_b).eb_u8[0])
 485
 486
 487 #define EFX_OWORD_IS_ZERO64(_oword)                                     \
 488         (((_oword).eo_u64[0] |                                          \
 489             (_oword).eo_u64[1]) == 0)
 490
 491 #define EFX_OWORD_IS_ZERO32(_oword)                                     \
 492         (((_oword).eo_u32[0] |                                          \
 493             (_oword).eo_u32[1] |                                        \
 494             (_oword).eo_u32[2] |                                        \
 495             (_oword).eo_u32[3]) == 0)
 496
 497 #define EFX_QWORD_IS_ZERO64(_qword)                                     \
 498         (((_qword).eq_u64[0]) == 0)
 499
 500 #define EFX_QWORD_IS_ZERO32(_qword)                                     \
 501         (((_qword).eq_u32[0] |                                          \
 502             (_qword).eq_u32[1]) == 0)
 503
 504 #define EFX_DWORD_IS_ZERO(_dword)                                       \
 505         (((_dword).ed_u32[0]) == 0)
 506
 507 #define EFX_WORD_IS_ZERO(_word)                                         \
 508         (((_word).ew_u16[0]) == 0)
 509
 510 #define EFX_BYTE_IS_ZERO(_byte)                                         \
 511         (((_byte).eb_u8[0]) == 0)
 512
 513
 514 #define EFX_OWORD_IS_SET64(_oword)                                      \
 515         (((_oword).eo_u64[0] &                                          \
 516             (_oword).eo_u64[1]) == ~((uint64_t)0))
 517
 518 #define EFX_OWORD_IS_SET32(_oword)                                      \
 519         (((_oword).eo_u32[0] &                                          \
 520             (_oword).eo_u32[1] &                                        \
 521             (_oword).eo_u32[2] &                                        \
 522             (_oword).eo_u32[3]) == ~((uint32_t)0))
 523
 524 #define EFX_QWORD_IS_SET64(_qword)                                      \
 525         (((_qword).eq_u64[0]) == ~((uint64_t)0))
 526
 527 #define EFX_QWORD_IS_SET32(_qword)                                      \
 528         (((_qword).eq_u32[0] &                                          \
 529             (_qword).eq_u32[1]) == ~((uint32_t)0))
 530
 531 #define EFX_DWORD_IS_SET(_dword)                                        \
 532         ((_dword).ed_u32[0] == ~((uint32_t)0))
 533
 534 #define EFX_WORD_IS_SET(_word)                                          \
 535         ((_word).ew_u16[0] == ~((uint16_t)0))
 536
 537 #define EFX_BYTE_IS_SET(_byte)                                          \
 538         ((_byte).eb_u8[0] == ~((uint8_t)0))
 539
 540 /*
 541  * Construct bit field portion
 542  *
 543  * Creates the portion of the bit field [low,high) that lies within
 544  * the range [min,max).
 545  */
 546
 547 #define EFX_INSERT_NATIVE64(_min, _max, _low, _high, _value)            \
 548         (((_low > _max) || (_high < _min)) ?                            \
 549                 0U :                                                    \
 550                 ((_low > _min) ?                                        \
 551                         (((uint64_t)(_value)) << EFX_SSUB(_low, _min)) :\
 552                         (((uint64_t)(_value)) >> EFX_SSUB(_min, _low))))
 553
 554 #define EFX_INSERT_NATIVE32(_min, _max, _low, _high, _value)            \
 555         (((_low > _max) || (_high < _min)) ?                            \
 556                 0U :                                                    \
 557                 ((_low > _min) ?                                        \
 558                         (((uint32_t)(_value)) << EFX_SSUB(_low, _min)) :\
 559                         (((uint32_t)(_value)) >> EFX_SSUB(_min, _low))))
 560
 561 #define EFX_INSERT_NATIVE16(_min, _max, _low, _high, _value)            \
 562         (((_low > _max) || (_high < _min)) ?                            \
 563                 0U :                                                    \
 564                 (uint16_t)((_low > _min) ?                              \
 565                                 ((_value) << EFX_SSUB(_low, _min)) :    \
 566                                 ((_value) >> EFX_SSUB(_min, _low))))
 567
 568 #define EFX_INSERT_NATIVE8(_min, _max, _low, _high, _value)             \
 569         (((_low > _max) || (_high < _min)) ?                            \
 570                 0U :                                                    \
 571                 (uint8_t)((_low > _min) ?                               \
 572                                 ((_value) << EFX_SSUB(_low, _min)) :    \
 573                                 ((_value) >> EFX_SSUB(_min, _low))))
 574
 575 /*
 576  * Construct bit field portion
 577  *
 578  * Creates the portion of the named bit field that lies within the
 579  * range [min,max).
 580  */
 581 #define EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value)           \
 582         EFX_INSERT_NATIVE64(_min, _max, EFX_LOW_BIT(_field),            \
 583             EFX_HIGH_BIT(_field), _value)
 584
 585 #define EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value)           \
 586         EFX_INSERT_NATIVE32(_min, _max, EFX_LOW_BIT(_field),            \
 587             EFX_HIGH_BIT(_field), _value)
 588
 589 #define EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value)           \
 590         EFX_INSERT_NATIVE16(_min, _max, EFX_LOW_BIT(_field),            \
 591             EFX_HIGH_BIT(_field), _value)
 592
 593 #define EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value)            \
 594         EFX_INSERT_NATIVE8(_min, _max, EFX_LOW_BIT(_field),             \
 595             EFX_HIGH_BIT(_field), _value)
 596
 597 /*
 598  * Construct bit field
 599  *
 600  * Creates the portion of the named bit fields that lie within the
 601  * range [min,max).
 602  */
 603 #define EFX_INSERT_FIELDS64(_min, _max,                                 \
 604             _field1, _value1, _field2, _value2, _field3, _value3,       \
 605             _field4, _value4, _field5, _value5, _field6, _value6,       \
 606             _field7, _value7, _field8, _value8, _field9, _value9,       \
 607             _field10, _value10)                                         \
 608         __CPU_TO_LE_64(                                                 \
 609             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field1, _value1) |   \
 610             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field2, _value2) |   \
 611             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field3, _value3) |   \
 612             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field4, _value4) |   \
 613             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field5, _value5) |   \
 614             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field6, _value6) |   \
 615             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field7, _value7) |   \
 616             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field8, _value8) |   \
 617             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field9, _value9) |   \
 618             EFX_INSERT_FIELD_NATIVE64(_min, _max, _field10, _value10))
 619
 620 #define EFX_INSERT_FIELDS32(_min, _max,                                 \
 621             _field1, _value1, _field2, _value2, _field3, _value3,       \
 622             _field4, _value4, _field5, _value5, _field6, _value6,       \
 623             _field7, _value7, _field8, _value8, _field9, _value9,       \
 624             _field10, _value10)                                         \
 625         __CPU_TO_LE_32(                                                 \
 626             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field1, _value1) |   \
 627             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field2, _value2) |   \
 628             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field3, _value3) |   \
 629             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field4, _value4) |   \
 630             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field5, _value5) |   \
 631             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field6, _value6) |   \
 632             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field7, _value7) |   \
 633             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field8, _value8) |   \
 634             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field9, _value9) |   \
 635             EFX_INSERT_FIELD_NATIVE32(_min, _max, _field10, _value10))
 636
 637 #define EFX_INSERT_FIELDS16(_min, _max,                                 \
 638             _field1, _value1, _field2, _value2, _field3, _value3,       \
 639             _field4, _value4, _field5, _value5, _field6, _value6,       \
 640             _field7, _value7, _field8, _value8, _field9, _value9,       \
 641             _field10, _value10)                                         \
 642         __CPU_TO_LE_16(                                                 \
 643             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field1, _value1) |   \
 644             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field2, _value2) |   \
 645             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field3, _value3) |   \
 646             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field4, _value4) |   \
 647             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field5, _value5) |   \
 648             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field6, _value6) |   \
 649             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field7, _value7) |   \
 650             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field8, _value8) |   \
 651             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field9, _value9) |   \
 652             EFX_INSERT_FIELD_NATIVE16(_min, _max, _field10, _value10))
 653
 654 #define EFX_INSERT_FIELDS8(_min, _max,                                  \
 655             _field1, _value1, _field2, _value2, _field3, _value3,       \
 656             _field4, _value4, _field5, _value5, _field6, _value6,       \
 657             _field7, _value7, _field8, _value8, _field9, _value9,       \
 658             _field10, _value10)                                         \
 659         __NATIVE_8(                                                     \
 660             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field1, _value1) |    \
 661             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field2, _value2) |    \
 662             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field3, _value3) |    \
 663             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field4, _value4) |    \
 664             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field5, _value5) |    \
 665             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field6, _value6) |    \
 666             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field7, _value7) |    \
 667             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field8, _value8) |    \
 668             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field9, _value9) |    \
 669             EFX_INSERT_FIELD_NATIVE8(_min, _max, _field10, _value10))
 670
 671 #define EFX_POPULATE_OWORD64(_oword,                                    \
 672             _field1, _value1, _field2, _value2, _field3, _value3,       \
 673             _field4, _value4, _field5, _value5, _field6, _value6,       \
 674             _field7, _value7, _field8, _value8, _field9, _value9,       \
 675             _field10, _value10)                                         \
 676         do {                                                            \
 677                 _NOTE(CONSTANTCONDITION)                                \
 678                 (_oword).eo_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 679                     _field1, _value1, _field2, _value2,                 \
 680                     _field3, _value3, _field4, _value4,                 \
 681                     _field5, _value5, _field6, _value6,                 \
 682                     _field7, _value7, _field8, _value8,                 \
 683                     _field9, _value9, _field10, _value10);              \
 684                 _NOTE(CONSTANTCONDITION)                                \
 685                 (_oword).eo_u64[1] = EFX_INSERT_FIELDS64(64, 127,       \
 686                     _field1, _value1, _field2, _value2,                 \
 687                     _field3, _value3, _field4, _value4,                 \
 688                     _field5, _value5, _field6, _value6,                 \
 689                     _field7, _value7, _field8, _value8,                 \
 690                     _field9, _value9, _field10, _value10);              \
 691         _NOTE(CONSTANTCONDITION)                                        \
 692         } while (B_FALSE)
 693
 694 #define EFX_POPULATE_OWORD32(_oword,                                    \
 695             _field1, _value1, _field2, _value2, _field3, _value3,       \
 696             _field4, _value4, _field5, _value5, _field6, _value6,       \
 697             _field7, _value7, _field8, _value8, _field9, _value9,       \
 698             _field10, _value10)                                         \
 699         do {                                                            \
 700                 _NOTE(CONSTANTCONDITION)                                \
 701                 (_oword).eo_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 702                     _field1, _value1, _field2, _value2,                 \
 703                     _field3, _value3, _field4, _value4,                 \
 704                     _field5, _value5, _field6, _value6,                 \
 705                     _field7, _value7, _field8, _value8,                 \
 706                     _field9, _value9, _field10, _value10);              \
 707                 _NOTE(CONSTANTCONDITION)                                \
 708                 (_oword).eo_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 709                     _field1, _value1, _field2, _value2,                 \
 710                     _field3, _value3, _field4, _value4,                 \
 711                     _field5, _value5, _field6, _value6,                 \
 712                     _field7, _value7, _field8, _value8,                 \
 713                     _field9, _value9, _field10, _value10);              \
 714                 _NOTE(CONSTANTCONDITION)                                \
 715                 (_oword).eo_u32[2] = EFX_INSERT_FIELDS32(64, 95,        \
 716                     _field1, _value1, _field2, _value2,                 \
 717                     _field3, _value3, _field4, _value4,                 \
 718                     _field5, _value5, _field6, _value6,                 \
 719                     _field7, _value7, _field8, _value8,                 \
 720                     _field9, _value9, _field10, _value10);              \
 721                 _NOTE(CONSTANTCONDITION)                                \
 722                 (_oword).eo_u32[3] = EFX_INSERT_FIELDS32(96, 127,       \
 723                     _field1, _value1, _field2, _value2,                 \
 724                     _field3, _value3, _field4, _value4,                 \
 725                     _field5, _value5, _field6, _value6,                 \
 726                     _field7, _value7, _field8, _value8,                 \
 727                     _field9, _value9, _field10, _value10);              \
 728         _NOTE(CONSTANTCONDITION)                                        \
 729         } while (B_FALSE)
 730
 731 #define EFX_POPULATE_QWORD64(_qword,                                    \
 732             _field1, _value1, _field2, _value2, _field3, _value3,       \
 733             _field4, _value4, _field5, _value5, _field6, _value6,       \
 734             _field7, _value7, _field8, _value8, _field9, _value9,       \
 735             _field10, _value10)                                         \
 736         do {                                                            \
 737                 _NOTE(CONSTANTCONDITION)                                \
 738                 (_qword).eq_u64[0] = EFX_INSERT_FIELDS64(0, 63,         \
 739                     _field1, _value1, _field2, _value2,                 \
 740                     _field3, _value3, _field4, _value4,                 \
 741                     _field5, _value5, _field6, _value6,                 \
 742                     _field7, _value7, _field8, _value8,                 \
 743                     _field9, _value9, _field10, _value10);              \
 744         _NOTE(CONSTANTCONDITION)                                        \
 745         } while (B_FALSE)
 746
 747 #define EFX_POPULATE_QWORD32(_qword,                                    \
 748             _field1, _value1, _field2, _value2, _field3, _value3,       \
 749             _field4, _value4, _field5, _value5, _field6, _value6,       \
 750             _field7, _value7, _field8, _value8, _field9, _value9,       \
 751             _field10, _value10)                                         \
 752         do {                                                            \
 753                 _NOTE(CONSTANTCONDITION)                                \
 754                 (_qword).eq_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 755                     _field1, _value1, _field2, _value2,                 \
 756                     _field3, _value3, _field4, _value4,                 \
 757                     _field5, _value5, _field6, _value6,                 \
 758                     _field7, _value7, _field8, _value8,                 \
 759                     _field9, _value9, _field10, _value10);              \
 760                 _NOTE(CONSTANTCONDITION)                                \
 761                 (_qword).eq_u32[1] = EFX_INSERT_FIELDS32(32, 63,        \
 762                     _field1, _value1, _field2, _value2,                 \
 763                     _field3, _value3, _field4, _value4,                 \
 764                     _field5, _value5, _field6, _value6,                 \
 765                     _field7, _value7, _field8, _value8,                 \
 766                     _field9, _value9, _field10, _value10);              \
 767         _NOTE(CONSTANTCONDITION)                                        \
 768         } while (B_FALSE)
 769
 770 #define EFX_POPULATE_DWORD(_dword,                                      \
 771             _field1, _value1, _field2, _value2, _field3, _value3,       \
 772             _field4, _value4, _field5, _value5, _field6, _value6,       \
 773             _field7, _value7, _field8, _value8, _field9, _value9,       \
 774             _field10, _value10)                                         \
 775         do {                                                            \
 776                 _NOTE(CONSTANTCONDITION)                                \
 777                 (_dword).ed_u32[0] = EFX_INSERT_FIELDS32(0, 31,         \
 778                     _field1, _value1, _field2, _value2,                 \
 779                     _field3, _value3, _field4, _value4,                 \
 780                     _field5, _value5, _field6, _value6,                 \
 781                     _field7, _value7, _field8, _value8,                 \
 782                     _field9, _value9, _field10, _value10);              \
 783         _NOTE(CONSTANTCONDITION)                                        \
 784         } while (B_FALSE)
 785
 786 #define EFX_POPULATE_WORD(_word,                                        \
 787             _field1, _value1, _field2, _value2, _field3, _value3,       \
 788             _field4, _value4, _field5, _value5, _field6, _value6,       \
 789             _field7, _value7, _field8, _value8, _field9, _value9,       \
 790             _field10, _value10)                                         \
 791         do {                                                            \
 792                 _NOTE(CONSTANTCONDITION)                                \
 793                 (_word).ew_u16[0] = EFX_INSERT_FIELDS16(0, 15,          \
 794                     _field1, _value1, _field2, _value2,                 \
 795                     _field3, _value3, _field4, _value4,                 \
 796                     _field5, _value5, _field6, _value6,                 \
 797                     _field7, _value7, _field8, _value8,                 \
 798                     _field9, _value9, _field10, _value10);              \
 799         _NOTE(CONSTANTCONDITION)                                        \
 800         } while (B_FALSE)
 801
 802 #define EFX_POPULATE_BYTE(_byte,                                        \
 803             _field1, _value1, _field2, _value2, _field3, _value3,       \
 804             _field4, _value4, _field5, _value5, _field6, _value6,       \
 805             _field7, _value7, _field8, _value8, _field9, _value9,       \
 806             _field10, _value10)                                         \
 807         do {                                                            \
 808                 _NOTE(CONSTANTCONDITION)                                \
 809                 (_byte).eb_u8[0] = EFX_INSERT_FIELDS8(0, 7,             \
 810                     _field1, _value1, _field2, _value2,                 \
 811                     _field3, _value3, _field4, _value4,                 \
 812                     _field5, _value5, _field6, _value6,                 \
 813                     _field7, _value7, _field8, _value8,                 \
 814                     _field9, _value9, _field10, _value10);              \
 815         _NOTE(CONSTANTCONDITION)                                        \
 816         } while (B_FALSE)
 817
 818 /* Populate an octword field with various numbers of arguments */
 819 #define EFX_POPULATE_OWORD_10 EFX_POPULATE_OWORD
 820
 821 #define EFX_POPULATE_OWORD_9(_oword,                                    \
 822             _field1, _value1, _field2, _value2, _field3, _value3,       \
 823             _field4, _value4, _field5, _value5, _field6, _value6,       \
 824             _field7, _value7, _field8, _value8, _field9, _value9)       \
 825         EFX_POPULATE_OWORD_10(_oword, EFX_DUMMY_FIELD, 0,               \
 826             _field1, _value1, _field2, _value2, _field3, _value3,       \
 827             _field4, _value4, _field5, _value5, _field6, _value6,       \
 828             _field7, _value7, _field8, _value8, _field9, _value9)
 829
 830 #define EFX_POPULATE_OWORD_8(_oword,                                    \
 831             _field1, _value1, _field2, _value2, _field3, _value3,       \
 832             _field4, _value4, _field5, _value5, _field6, _value6,       \
 833             _field7, _value7, _field8, _value8)                         \
 834         EFX_POPULATE_OWORD_9(_oword, EFX_DUMMY_FIELD, 0,                \
 835             _field1, _value1, _field2, _value2, _field3, _value3,       \
 836             _field4, _value4, _field5, _value5, _field6, _value6,       \
 837             _field7, _value7, _field8, _value8)
 838
 839 #define EFX_POPULATE_OWORD_7(_oword,                                    \
 840             _field1, _value1, _field2, _value2, _field3, _value3,       \
 841             _field4, _value4, _field5, _value5, _field6, _value6,       \
 842             _field7, _value7)                                           \
 843         EFX_POPULATE_OWORD_8(_oword, EFX_DUMMY_FIELD, 0,                \
 844             _field1, _value1, _field2, _value2, _field3, _value3,       \
 845             _field4, _value4, _field5, _value5, _field6, _value6,       \
 846             _field7, _value7)
 847
 848 #define EFX_POPULATE_OWORD_6(_oword,                                    \
 849             _field1, _value1, _field2, _value2, _field3, _value3,       \
 850             _field4, _value4, _field5, _value5, _field6, _value6)       \
 851         EFX_POPULATE_OWORD_7(_oword, EFX_DUMMY_FIELD, 0,                \
 852             _field1, _value1, _field2, _value2, _field3, _value3,       \
 853             _field4, _value4, _field5, _value5, _field6, _value6)
 854
 855 #define EFX_POPULATE_OWORD_5(_oword,                                    \
 856             _field1, _value1, _field2, _value2, _field3, _value3,       \
 857             _field4, _value4, _field5, _value5)                         \
 858         EFX_POPULATE_OWORD_6(_oword, EFX_DUMMY_FIELD, 0,                \
 859             _field1, _value1, _field2, _value2, _field3, _value3,       \
 860             _field4, _value4, _field5, _value5)
 861
 862 #define EFX_POPULATE_OWORD_4(_oword,                                    \
 863             _field1, _value1, _field2, _value2, _field3, _value3,       \
 864             _field4, _value4)                                           \
 865         EFX_POPULATE_OWORD_5(_oword, EFX_DUMMY_FIELD, 0,                \
 866             _field1, _value1, _field2, _value2, _field3, _value3,       \
 867             _field4, _value4)
 868
 869 #define EFX_POPULATE_OWORD_3(_oword,                                    \
 870             _field1, _value1, _field2, _value2, _field3, _value3)       \
 871         EFX_POPULATE_OWORD_4(_oword, EFX_DUMMY_FIELD, 0,                \
 872             _field1, _value1, _field2, _value2, _field3, _value3)
 873
 874 #define EFX_POPULATE_OWORD_2(_oword,                                    \
 875             _field1, _value1, _field2, _value2)                         \
 876         EFX_POPULATE_OWORD_3(_oword, EFX_DUMMY_FIELD, 0,                \
 877             _field1, _value1, _field2, _value2)
 878
 879 #define EFX_POPULATE_OWORD_1(_oword,                                    \
 880             _field1, _value1)                                           \
 881         EFX_POPULATE_OWORD_2(_oword, EFX_DUMMY_FIELD, 0,                \
 882             _field1, _value1)
 883
 884 #define EFX_ZERO_OWORD(_oword)                                          \
 885         EFX_POPULATE_OWORD_1(_oword, EFX_DUMMY_FIELD, 0)
 886
 887 #define EFX_SET_OWORD(_oword)                                           \
 888         EFX_POPULATE_OWORD_4(_oword,                                    \
 889             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff,           \
 890             EFX_DWORD_2, 0xffffffff, EFX_DWORD_3, 0xffffffff)
 891
 892 /* Populate a quadword field with various numbers of arguments */
 893 #define EFX_POPULATE_QWORD_10 EFX_POPULATE_QWORD
 894
 895 #define EFX_POPULATE_QWORD_9(_qword,                                    \
 896             _field1, _value1, _field2, _value2, _field3, _value3,       \
 897             _field4, _value4, _field5, _value5, _field6, _value6,       \
 898             _field7, _value7, _field8, _value8, _field9, _value9)       \
 899         EFX_POPULATE_QWORD_10(_qword, EFX_DUMMY_FIELD, 0,               \
 900             _field1, _value1, _field2, _value2, _field3, _value3,       \
 901             _field4, _value4, _field5, _value5, _field6, _value6,       \
 902             _field7, _value7, _field8, _value8, _field9, _value9)
 903
 904 #define EFX_POPULATE_QWORD_8(_qword,                                    \
 905             _field1, _value1, _field2, _value2, _field3, _value3,       \
 906             _field4, _value4, _field5, _value5, _field6, _value6,       \
 907             _field7, _value7, _field8, _value8)                         \
 908         EFX_POPULATE_QWORD_9(_qword, EFX_DUMMY_FIELD, 0,                \
 909             _field1, _value1, _field2, _value2, _field3, _value3,       \
 910             _field4, _value4, _field5, _value5, _field6, _value6,       \
 911             _field7, _value7, _field8, _value8)
 912
 913 #define EFX_POPULATE_QWORD_7(_qword,                                    \
 914             _field1, _value1, _field2, _value2, _field3, _value3,       \
 915             _field4, _value4, _field5, _value5, _field6, _value6,       \
 916             _field7, _value7)                                           \
 917         EFX_POPULATE_QWORD_8(_qword, EFX_DUMMY_FIELD, 0,                \
 918             _field1, _value1, _field2, _value2, _field3, _value3,       \
 919             _field4, _value4, _field5, _value5, _field6, _value6,       \
 920             _field7, _value7)
 921
 922 #define EFX_POPULATE_QWORD_6(_qword,                                    \
 923             _field1, _value1, _field2, _value2, _field3, _value3,       \
 924             _field4, _value4, _field5, _value5, _field6, _value6)       \
 925         EFX_POPULATE_QWORD_7(_qword, EFX_DUMMY_FIELD, 0,                \
 926             _field1, _value1, _field2, _value2, _field3, _value3,       \
 927             _field4, _value4, _field5, _value5, _field6, _value6)
 928
 929 #define EFX_POPULATE_QWORD_5(_qword,                                    \
 930             _field1, _value1, _field2, _value2, _field3, _value3,       \
 931             _field4, _value4, _field5, _value5)                         \
 932         EFX_POPULATE_QWORD_6(_qword, EFX_DUMMY_FIELD, 0,                \
 933             _field1, _value1, _field2, _value2, _field3, _value3,       \
 934             _field4, _value4, _field5, _value5)
 935
 936 #define EFX_POPULATE_QWORD_4(_qword,                                    \
 937             _field1, _value1, _field2, _value2, _field3, _value3,       \
 938             _field4, _value4)                                           \
 939         EFX_POPULATE_QWORD_5(_qword, EFX_DUMMY_FIELD, 0,                \
 940             _field1, _value1, _field2, _value2, _field3, _value3,       \
 941             _field4, _value4)
 942
 943 #define EFX_POPULATE_QWORD_3(_qword,                                    \
 944             _field1, _value1, _field2, _value2, _field3, _value3)       \
 945         EFX_POPULATE_QWORD_4(_qword, EFX_DUMMY_FIELD, 0,                \
 946             _field1, _value1, _field2, _value2, _field3, _value3)
 947
 948 #define EFX_POPULATE_QWORD_2(_qword,                                    \
 949             _field1, _value1, _field2, _value2)                         \
 950         EFX_POPULATE_QWORD_3(_qword, EFX_DUMMY_FIELD, 0,                \
 951             _field1, _value1, _field2, _value2)
 952
 953 #define EFX_POPULATE_QWORD_1(_qword,                                    \
 954             _field1, _value1)                                           \
 955         EFX_POPULATE_QWORD_2(_qword, EFX_DUMMY_FIELD, 0,                \
 956             _field1, _value1)
 957
 958 #define EFX_ZERO_QWORD(_qword)                                          \
 959         EFX_POPULATE_QWORD_1(_qword, EFX_DUMMY_FIELD, 0)
 960
 961 #define EFX_SET_QWORD(_qword)                                           \
 962         EFX_POPULATE_QWORD_2(_qword,                                    \
 963             EFX_DWORD_0, 0xffffffff, EFX_DWORD_1, 0xffffffff)
 964
 965 /* Populate a dword field with various numbers of arguments */
 966 #define EFX_POPULATE_DWORD_10 EFX_POPULATE_DWORD
 967
 968 #define EFX_POPULATE_DWORD_9(_dword,                                    \
 969             _field1, _value1, _field2, _value2, _field3, _value3,       \
 970             _field4, _value4, _field5, _value5, _field6, _value6,       \
 971             _field7, _value7, _field8, _value8, _field9, _value9)       \
 972         EFX_POPULATE_DWORD_10(_dword, EFX_DUMMY_FIELD, 0,               \
 973             _field1, _value1, _field2, _value2, _field3, _value3,       \
 974             _field4, _value4, _field5, _value5, _field6, _value6,       \
 975             _field7, _value7, _field8, _value8, _field9, _value9)
 976
 977 #define EFX_POPULATE_DWORD_8(_dword,                                    \
 978             _field1, _value1, _field2, _value2, _field3, _value3,       \
 979             _field4, _value4, _field5, _value5, _field6, _value6,       \
 980             _field7, _value7, _field8, _value8)                         \
 981         EFX_POPULATE_DWORD_9(_dword, EFX_DUMMY_FIELD, 0,                \
 982             _field1, _value1, _field2, _value2, _field3, _value3,       \
 983             _field4, _value4, _field5, _value5, _field6, _value6,       \
 984             _field7, _value7, _field8, _value8)
 985
 986 #define EFX_POPULATE_DWORD_7(_dword,                                    \
 987             _field1, _value1, _field2, _value2, _field3, _value3,       \
 988             _field4, _value4, _field5, _value5, _field6, _value6,       \
 989             _field7, _value7)                                           \
 990         EFX_POPULATE_DWORD_8(_dword, EFX_DUMMY_FIELD, 0,                \
 991             _field1, _value1, _field2, _value2, _field3, _value3,       \
 992             _field4, _value4, _field5, _value5, _field6, _value6,       \
 993             _field7, _value7)
 994
 995 #define EFX_POPULATE_DWORD_6(_dword,                                    \
 996             _field1, _value1, _field2, _value2, _field3, _value3,       \
 997             _field4, _value4, _field5, _value5, _field6, _value6)       \
 998         EFX_POPULATE_DWORD_7(_dword, EFX_DUMMY_FIELD, 0,                \
 999             _field1, _value1, _field2, _value2, _field3, _value3,       \
1000             _field4, _value4, _field5, _value5, _field6, _value6)
1001
1002 #define EFX_POPULATE_DWORD_5(_dword,                                    \
1003             _field1, _value1, _field2, _value2, _field3, _value3,       \
1004             _field4, _value4, _field5, _value5)                         \
1005         EFX_POPULATE_DWORD_6(_dword, EFX_DUMMY_FIELD, 0,                \
1006             _field1, _value1, _field2, _value2, _field3, _value3,       \
1007             _field4, _value4, _field5, _value5)
1008
1009 #define EFX_POPULATE_DWORD_4(_dword,                                    \
1010             _field1, _value1, _field2, _value2, _field3, _value3,       \
1011             _field4, _value4)                                           \
1012         EFX_POPULATE_DWORD_5(_dword, EFX_DUMMY_FIELD, 0,                \
1013             _field1, _value1, _field2, _value2, _field3, _value3,       \
1014             _field4, _value4)
1015
1016 #define EFX_POPULATE_DWORD_3(_dword,                                    \
1017             _field1, _value1, _field2, _value2, _field3, _value3)       \
1018         EFX_POPULATE_DWORD_4(_dword, EFX_DUMMY_FIELD, 0,                \
1019             _field1, _value1, _field2, _value2, _field3, _value3)
1020
1021 #define EFX_POPULATE_DWORD_2(_dword,                                    \
1022             _field1, _value1, _field2, _value2)                         \
1023         EFX_POPULATE_DWORD_3(_dword, EFX_DUMMY_FIELD, 0,                \
1024             _field1, _value1, _field2, _value2)
1025
1026 #define EFX_POPULATE_DWORD_1(_dword,                                    \
1027             _field1, _value1)                                           \
1028         EFX_POPULATE_DWORD_2(_dword, EFX_DUMMY_FIELD, 0,                \
1029             _field1, _value1)
1030
1031 #define EFX_ZERO_DWORD(_dword)                                          \
1032         EFX_POPULATE_DWORD_1(_dword, EFX_DUMMY_FIELD, 0)
1033
1034 #define EFX_SET_DWORD(_dword)                                           \
1035         EFX_POPULATE_DWORD_1(_dword,                                    \
1036             EFX_DWORD_0, 0xffffffff)
1037
1038 /* Populate a word field with various numbers of arguments */
1039 #define EFX_POPULATE_WORD_10 EFX_POPULATE_WORD
1040
1041 #define EFX_POPULATE_WORD_9(_word,                                      \
1042             _field1, _value1, _field2, _value2, _field3, _value3,       \
1043             _field4, _value4, _field5, _value5, _field6, _value6,       \
1044             _field7, _value7, _field8, _value8, _field9, _value9)       \
1045         EFX_POPULATE_WORD_10(_word, EFX_DUMMY_FIELD, 0,                 \
1046             _field1, _value1, _field2, _value2, _field3, _value3,       \
1047             _field4, _value4, _field5, _value5, _field6, _value6,       \
1048             _field7, _value7, _field8, _value8, _field9, _value9)
1049
1050 #define EFX_POPULATE_WORD_8(_word,                                      \
1051             _field1, _value1, _field2, _value2, _field3, _value3,       \
1052             _field4, _value4, _field5, _value5, _field6, _value6,       \
1053             _field7, _value7, _field8, _value8)                         \
1054         EFX_POPULATE_WORD_9(_word, EFX_DUMMY_FIELD, 0,                  \
1055             _field1, _value1, _field2, _value2, _field3, _value3,       \
1056             _field4, _value4, _field5, _value5, _field6, _value6,       \
1057             _field7, _value7, _field8, _value8)
1058
1059 #define EFX_POPULATE_WORD_7(_word,                                      \
1060             _field1, _value1, _field2, _value2, _field3, _value3,       \
1061             _field4, _value4, _field5, _value5, _field6, _value6,       \
1062             _field7, _value7)                                           \
1063         EFX_POPULATE_WORD_8(_word, EFX_DUMMY_FIELD, 0,                  \
1064             _field1, _value1, _field2, _value2, _field3, _value3,       \
1065             _field4, _value4, _field5, _value5, _field6, _value6,       \
1066             _field7, _value7)
1067
1068 #define EFX_POPULATE_WORD_6(_word,                                      \
1069             _field1, _value1, _field2, _value2, _field3, _value3,       \
1070             _field4, _value4, _field5, _value5, _field6, _value6)       \
1071         EFX_POPULATE_WORD_7(_word, EFX_DUMMY_FIELD, 0,                  \
1072             _field1, _value1, _field2, _value2, _field3, _value3,       \
1073             _field4, _value4, _field5, _value5, _field6, _value6)
1074
1075 #define EFX_POPULATE_WORD_5(_word,                                      \
1076             _field1, _value1, _field2, _value2, _field3, _value3,       \
1077             _field4, _value4, _field5, _value5)                         \
1078         EFX_POPULATE_WORD_6(_word, EFX_DUMMY_FIELD, 0,                  \
1079             _field1, _value1, _field2, _value2, _field3, _value3,       \
1080             _field4, _value4, _field5, _value5)
1081
1082 #define EFX_POPULATE_WORD_4(_word,                                      \
1083             _field1, _value1, _field2, _value2, _field3, _value3,       \
1084             _field4, _value4)                                           \
1085         EFX_POPULATE_WORD_5(_word, EFX_DUMMY_FIELD, 0,                  \
1086             _field1, _value1, _field2, _value2, _field3, _value3,       \
1087             _field4, _value4)
1088
1089 #define EFX_POPULATE_WORD_3(_word,                                      \
1090             _field1, _value1, _field2, _value2, _field3, _value3)       \
1091         EFX_POPULATE_WORD_4(_word, EFX_DUMMY_FIELD, 0,                  \
1092             _field1, _value1, _field2, _value2, _field3, _value3)
1093
1094 #define EFX_POPULATE_WORD_2(_word,                                      \
1095             _field1, _value1, _field2, _value2)                         \
1096         EFX_POPULATE_WORD_3(_word, EFX_DUMMY_FIELD, 0,                  \
1097             _field1, _value1, _field2, _value2)
1098
1099 #define EFX_POPULATE_WORD_1(_word,                                      \
1100             _field1, _value1)                                           \
1101         EFX_POPULATE_WORD_2(_word, EFX_DUMMY_FIELD, 0,                  \
1102             _field1, _value1)
1103
1104 #define EFX_ZERO_WORD(_word)                                            \
1105         EFX_POPULATE_WORD_1(_word, EFX_DUMMY_FIELD, 0)
1106
1107 #define EFX_SET_WORD(_word)                                             \
1108         EFX_POPULATE_WORD_1(_word,                                      \
1109             EFX_WORD_0, 0xffff)
1110
1111 /* Populate a byte field with various numbers of arguments */
1112 #define EFX_POPULATE_BYTE_10 EFX_POPULATE_BYTE
1113
1114 #define EFX_POPULATE_BYTE_9(_byte,                                      \
1115             _field1, _value1, _field2, _value2, _field3, _value3,       \
1116             _field4, _value4, _field5, _value5, _field6, _value6,       \
1117             _field7, _value7, _field8, _value8, _field9, _value9)       \
1118         EFX_POPULATE_BYTE_10(_byte, EFX_DUMMY_FIELD, 0,                 \
1119             _field1, _value1, _field2, _value2, _field3, _value3,       \
1120             _field4, _value4, _field5, _value5, _field6, _value6,       \
1121             _field7, _value7, _field8, _value8, _field9, _value9)
1122
1123 #define EFX_POPULATE_BYTE_8(_byte,                                      \
1124             _field1, _value1, _field2, _value2, _field3, _value3,       \
1125             _field4, _value4, _field5, _value5, _field6, _value6,       \
1126             _field7, _value7, _field8, _value8)                         \
1127         EFX_POPULATE_BYTE_9(_byte, EFX_DUMMY_FIELD, 0,                  \
1128             _field1, _value1, _field2, _value2, _field3, _value3,       \
1129             _field4, _value4, _field5, _value5, _field6, _value6,       \
1130             _field7, _value7, _field8, _value8)
1131
1132 #define EFX_POPULATE_BYTE_7(_byte,                                      \
1133             _field1, _value1, _field2, _value2, _field3, _value3,       \
1134             _field4, _value4, _field5, _value5, _field6, _value6,       \
1135             _field7, _value7)                                           \
1136         EFX_POPULATE_BYTE_8(_byte, EFX_DUMMY_FIELD, 0,                  \
1137             _field1, _value1, _field2, _value2, _field3, _value3,       \
1138             _field4, _value4, _field5, _value5, _field6, _value6,       \
1139             _field7, _value7)
1140
1141 #define EFX_POPULATE_BYTE_6(_byte,                                      \
1142             _field1, _value1, _field2, _value2, _field3, _value3,       \
1143             _field4, _value4, _field5, _value5, _field6, _value6)       \
1144         EFX_POPULATE_BYTE_7(_byte, EFX_DUMMY_FIELD, 0,                  \
1145             _field1, _value1, _field2, _value2, _field3, _value3,       \
1146             _field4, _value4, _field5, _value5, _field6, _value6)
1147
1148 #define EFX_POPULATE_BYTE_5(_byte,                                      \
1149             _field1, _value1, _field2, _value2, _field3, _value3,       \
1150             _field4, _value4, _field5, _value5)                         \
1151         EFX_POPULATE_BYTE_6(_byte, EFX_DUMMY_FIELD, 0,                  \
1152             _field1, _value1, _field2, _value2, _field3, _value3,       \
1153             _field4, _value4, _field5, _value5)
1154
1155 #define EFX_POPULATE_BYTE_4(_byte,                                      \
1156             _field1, _value1, _field2, _value2, _field3, _value3,       \
1157             _field4, _value4)                                           \
1158         EFX_POPULATE_BYTE_5(_byte, EFX_DUMMY_FIELD, 0,                  \
1159             _field1, _value1, _field2, _value2, _field3, _value3,       \
1160             _field4, _value4)
1161
1162 #define EFX_POPULATE_BYTE_3(_byte,                                      \
1163             _field1, _value1, _field2, _value2, _field3, _value3)       \
1164         EFX_POPULATE_BYTE_4(_byte, EFX_DUMMY_FIELD, 0,                  \
1165             _field1, _value1, _field2, _value2, _field3, _value3)
1166
1167 #define EFX_POPULATE_BYTE_2(_byte,                                      \
1168             _field1, _value1, _field2, _value2)                         \
1169         EFX_POPULATE_BYTE_3(_byte, EFX_DUMMY_FIELD, 0,                  \
1170             _field1, _value1, _field2, _value2)
1171
1172 #define EFX_POPULATE_BYTE_1(_byte,                                      \
1173             _field1, _value1)                                           \
1174         EFX_POPULATE_BYTE_2(_byte, EFX_DUMMY_FIELD, 0,                  \
1175             _field1, _value1)
1176
1177 #define EFX_ZERO_BYTE(_byte)                                            \
1178         EFX_POPULATE_BYTE_1(_byte, EFX_DUMMY_FIELD, 0)
1179
1180 #define EFX_SET_BYTE(_byte)                                             \
1181         EFX_POPULATE_BYTE_1(_byte,                                      \
1182             EFX_BYTE_0, 0xff)
1183
1184 /*
1185  * Modify a named field within an already-populated structure.  Used
1186  * for read-modify-write operations.
1187  */
1188
1189 #define EFX_INSERT_FIELD64(_min, _max, _field, _value)                  \
1190         __CPU_TO_LE_64(EFX_INSERT_FIELD_NATIVE64(_min, _max, _field, _value))
1191
1192 #define EFX_INSERT_FIELD32(_min, _max, _field, _value)                  \
1193         __CPU_TO_LE_32(EFX_INSERT_FIELD_NATIVE32(_min, _max, _field, _value))
1194
1195 #define EFX_INSERT_FIELD16(_min, _max, _field, _value)                  \
1196         __CPU_TO_LE_16(EFX_INSERT_FIELD_NATIVE16(_min, _max, _field, _value))
1197
1198 #define EFX_INSERT_FIELD8(_min, _max, _field, _value)                   \
1199         __NATIVE_8(EFX_INSERT_FIELD_NATIVE8(_min, _max, _field, _value))
1200
1201 #define EFX_INPLACE_MASK64(_min, _max, _field)                          \
1202         EFX_INSERT_FIELD64(_min, _max, _field, EFX_MASK64(_field))
1203
1204 #define EFX_INPLACE_MASK32(_min, _max, _field)                          \
1205         EFX_INSERT_FIELD32(_min, _max, _field, EFX_MASK32(_field))
1206
1207 #define EFX_INPLACE_MASK16(_min, _max, _field)                          \
1208         EFX_INSERT_FIELD16(_min, _max, _field, EFX_MASK16(_field))
1209
1210 #define EFX_INPLACE_MASK8(_min, _max, _field)                           \
1211         EFX_INSERT_FIELD8(_min, _max, _field, EFX_MASK8(_field))
1212
1213 #define EFX_SET_OWORD_FIELD64(_oword, _field, _value)                   \
1214         do {                                                            \
1215                 _NOTE(CONSTANTCONDITION)                                \
1216                 (_oword).eo_u64[0] = (((_oword).eo_u64[0] &             \
1217                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1218                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1219                 _NOTE(CONSTANTCONDITION)                                \
1220                 (_oword).eo_u64[1] = (((_oword).eo_u64[1] &             \
1221                     ~EFX_INPLACE_MASK64(64, 127, _field)) |             \
1222                     EFX_INSERT_FIELD64(64, 127, _field, _value));       \
1223         _NOTE(CONSTANTCONDITION)                                        \
1224         } while (B_FALSE)
1225
1226 #define EFX_SET_OWORD_FIELD32(_oword, _field, _value)                   \
1227         do {                                                            \
1228                 _NOTE(CONSTANTCONDITION)                                \
1229                 (_oword).eo_u32[0] = (((_oword).eo_u32[0] &             \
1230                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1231                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1232                 _NOTE(CONSTANTCONDITION)                                \
1233                 (_oword).eo_u32[1] = (((_oword).eo_u32[1] &             \
1234                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1235                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1236                 _NOTE(CONSTANTCONDITION)                                \
1237                 (_oword).eo_u32[2] = (((_oword).eo_u32[2] &             \
1238                     ~EFX_INPLACE_MASK32(64, 95, _field)) |              \
1239                     EFX_INSERT_FIELD32(64, 95, _field, _value));        \
1240                 _NOTE(CONSTANTCONDITION)                                \
1241                 (_oword).eo_u32[3] = (((_oword).eo_u32[3] &             \
1242                     ~EFX_INPLACE_MASK32(96, 127, _field)) |             \
1243                     EFX_INSERT_FIELD32(96, 127, _field, _value));       \
1244         _NOTE(CONSTANTCONDITION)                                        \
1245         } while (B_FALSE)
1246
1247 #define EFX_SET_QWORD_FIELD64(_qword, _field, _value)                   \
1248         do {                                                            \
1249                 _NOTE(CONSTANTCONDITION)                                \
1250                 (_qword).eq_u64[0] = (((_qword).eq_u64[0] &             \
1251                     ~EFX_INPLACE_MASK64(0, 63, _field)) |               \
1252                     EFX_INSERT_FIELD64(0, 63, _field, _value));         \
1253         _NOTE(CONSTANTCONDITION)                                        \
1254         } while (B_FALSE)
1255
1256 #define EFX_SET_QWORD_FIELD32(_qword, _field, _value)                   \
1257         do {                                                            \
1258                 _NOTE(CONSTANTCONDITION)                                \
1259                 (_qword).eq_u32[0] = (((_qword).eq_u32[0] &             \
1260                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1261                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1262                 _NOTE(CONSTANTCONDITION)                                \
1263                 (_qword).eq_u32[1] = (((_qword).eq_u32[1] &             \
1264                     ~EFX_INPLACE_MASK32(32, 63, _field)) |              \
1265                     EFX_INSERT_FIELD32(32, 63, _field, _value));        \
1266         _NOTE(CONSTANTCONDITION)                                        \
1267         } while (B_FALSE)
1268
1269 #define EFX_SET_DWORD_FIELD(_dword, _field, _value)                     \
1270         do {                                                            \
1271                 _NOTE(CONSTANTCONDITION)                                \
1272                 (_dword).ed_u32[0] = (((_dword).ed_u32[0] &             \
1273                     ~EFX_INPLACE_MASK32(0, 31, _field)) |               \
1274                     EFX_INSERT_FIELD32(0, 31, _field, _value));         \
1275         _NOTE(CONSTANTCONDITION)                                        \
1276         } while (B_FALSE)
1277
1278 #define EFX_SET_WORD_FIELD(_word, _field, _value)                       \
1279         do {                                                            \
1280                 _NOTE(CONSTANTCONDITION)                                \
1281                 (_word).ew_u16[0] = (((_word).ew_u16[0] &               \
1282                     ~EFX_INPLACE_MASK16(0, 15, _field)) |               \
1283                     EFX_INSERT_FIELD16(0, 15, _field, _value));         \
1284         _NOTE(CONSTANTCONDITION)                                        \
1285         } while (B_FALSE)
1286
1287 #define EFX_SET_BYTE_FIELD(_byte, _field, _value)                       \
1288         do {                                                            \
1289                 _NOTE(CONSTANTCONDITION)                                \
1290                 (_byte).eb_u8[0] = (((_byte).eb_u8[0] &                 \
1291                     ~EFX_INPLACE_MASK8(0, 7, _field)) |                 \
1292                     EFX_INSERT_FIELD8(0, 7, _field, _value));           \
1293         _NOTE(CONSTANTCONDITION)                                        \
1294         } while (B_FALSE)
1295
1296 /*
1297  * Set or clear a numbered bit within an octword.
1298  */
1299
1300 #define EFX_SHIFT64(_bit, _base)                                        \
1301         (((_bit) >= (_base) && (_bit) < (_base) + 64) ?                 \
1302                 ((uint64_t)1 << EFX_SSUB((_bit), (_base))) :            \
1303                 0U)
1304
1305 #define EFX_SHIFT32(_bit, _base)                                        \
1306         (((_bit) >= (_base) && (_bit) < (_base) + 32) ?                 \
1307                 ((uint32_t)1 << EFX_SSUB((_bit),(_base))) :             \
1308                 0U)
1309
1310 #define EFX_SHIFT16(_bit, _base)                                        \
1311         (((_bit) >= (_base) && (_bit) < (_base) + 16) ?                 \
1312                 (uint16_t)(1 << EFX_SSUB((_bit), (_base))) :            \
1313                 0U)
1314
1315 #define EFX_SHIFT8(_bit, _base)                                         \
1316         (((_bit) >= (_base) && (_bit) < (_base) + 8) ?                  \
1317                 (uint8_t)(1 << EFX_SSUB((_bit), (_base))) :             \
1318                 0U)
1319
1320 #define EFX_SET_OWORD_BIT64(_oword, _bit)                               \
1321         do {                                                            \
1322                 _NOTE(CONSTANTCONDITION)                                \
1323                 (_oword).eo_u64[0] |=                                   \
1324                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1325                 (_oword).eo_u64[1] |=                                   \
1326                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)));    \
1327         _NOTE(CONSTANTCONDITION)                                        \
1328         } while (B_FALSE)
1329
1330 #define EFX_SET_OWORD_BIT32(_oword, _bit)                               \
1331         do {                                                            \
1332                 _NOTE(CONSTANTCONDITION)                                \
1333                 (_oword).eo_u32[0] |=                                   \
1334                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1335                 (_oword).eo_u32[1] |=                                   \
1336                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1337                 (_oword).eo_u32[2] |=                                   \
1338                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)));    \
1339                 (_oword).eo_u32[3] |=                                   \
1340                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)));    \
1341         _NOTE(CONSTANTCONDITION)                                        \
1342         } while (B_FALSE)
1343
1344 #define EFX_CLEAR_OWORD_BIT64(_oword, _bit)                             \
1345         do {                                                            \
1346                 _NOTE(CONSTANTCONDITION)                                \
1347                 (_oword).eo_u64[0] &=                                   \
1348                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1349                 (_oword).eo_u64[1] &=                                   \
1350                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(64)));   \
1351         _NOTE(CONSTANTCONDITION)                                        \
1352         } while (B_FALSE)
1353
1354 #define EFX_CLEAR_OWORD_BIT32(_oword, _bit)                             \
1355         do {                                                            \
1356                 _NOTE(CONSTANTCONDITION)                                \
1357                 (_oword).eo_u32[0] &=                                   \
1358                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1359                 (_oword).eo_u32[1] &=                                   \
1360                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1361                 (_oword).eo_u32[2] &=                                   \
1362                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(64)));   \
1363                 (_oword).eo_u32[3] &=                                   \
1364                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(96)));   \
1365         _NOTE(CONSTANTCONDITION)                                        \
1366         } while (B_FALSE)
1367
1368 #define EFX_TEST_OWORD_BIT64(_oword, _bit)                              \
1369         (((_oword).eo_u64[0] &                                          \
1370                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) ||  \
1371         ((_oword).eo_u64[1] &                                           \
1372                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(64)))))
1373
1374 #define EFX_TEST_OWORD_BIT32(_oword, _bit)                              \
1375         (((_oword).eo_u32[0] &                                          \
1376                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1377         ((_oword).eo_u32[1] &                                           \
1378                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))) || \
1379         ((_oword).eo_u32[2] &                                           \
1380                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(64)))) || \
1381         ((_oword).eo_u32[3] &                                           \
1382                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(96)))))
1383
1384
1385 #define EFX_SET_QWORD_BIT64(_qword, _bit)                               \
1386         do {                                                            \
1387                 _NOTE(CONSTANTCONDITION)                                \
1388                 (_qword).eq_u64[0] |=                                   \
1389                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)));     \
1390         _NOTE(CONSTANTCONDITION)                                        \
1391         } while (B_FALSE)
1392
1393 #define EFX_SET_QWORD_BIT32(_qword, _bit)                               \
1394         do {                                                            \
1395                 _NOTE(CONSTANTCONDITION)                                \
1396                 (_qword).eq_u32[0] |=                                   \
1397                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1398                 (_qword).eq_u32[1] |=                                   \
1399                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)));    \
1400         _NOTE(CONSTANTCONDITION)                                        \
1401         } while (B_FALSE)
1402
1403 #define EFX_CLEAR_QWORD_BIT64(_qword, _bit)                             \
1404         do {                                                            \
1405                 _NOTE(CONSTANTCONDITION)                                \
1406                 (_qword).eq_u64[0] &=                                   \
1407                     __CPU_TO_LE_64(~EFX_SHIFT64(_bit, FIX_LINT(0)));    \
1408         _NOTE(CONSTANTCONDITION)                                        \
1409         } while (B_FALSE)
1410
1411 #define EFX_CLEAR_QWORD_BIT32(_qword, _bit)                             \
1412         do {                                                            \
1413                 _NOTE(CONSTANTCONDITION)                                \
1414                 (_qword).eq_u32[0] &=                                   \
1415                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1416                 (_qword).eq_u32[1] &=                                   \
1417                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(32)));   \
1418         _NOTE(CONSTANTCONDITION)                                        \
1419         } while (B_FALSE)
1420
1421 #define EFX_TEST_QWORD_BIT64(_qword, _bit)                              \
1422         (((_qword).eq_u64[0] &                                          \
1423                     __CPU_TO_LE_64(EFX_SHIFT64(_bit, FIX_LINT(0)))) != 0)
1424
1425 #define EFX_TEST_QWORD_BIT32(_qword, _bit)                              \
1426         (((_qword).eq_u32[0] &                                          \
1427                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) ||  \
1428         ((_qword).eq_u32[1] &                                           \
1429                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(32)))))
1430
1431
1432 #define EFX_SET_DWORD_BIT(_dword, _bit)                                 \
1433         do {                                                            \
1434                 (_dword).ed_u32[0] |=                                   \
1435                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)));     \
1436         _NOTE(CONSTANTCONDITION)                                        \
1437         } while (B_FALSE)
1438
1439 #define EFX_CLEAR_DWORD_BIT(_dword, _bit)                               \
1440         do {                                                            \
1441                 (_dword).ed_u32[0] &=                                   \
1442                     __CPU_TO_LE_32(~EFX_SHIFT32(_bit, FIX_LINT(0)));    \
1443         _NOTE(CONSTANTCONDITION)                                        \
1444         } while (B_FALSE)
1445
1446 #define EFX_TEST_DWORD_BIT(_dword, _bit)                                \
1447         (((_dword).ed_u32[0] &                                          \
1448                     __CPU_TO_LE_32(EFX_SHIFT32(_bit, FIX_LINT(0)))) != 0)
1449
1450
1451 #define EFX_SET_WORD_BIT(_word, _bit)                                   \
1452         do {                                                            \
1453                 (_word).ew_u16[0] |=                                    \
1454                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)));     \
1455         _NOTE(CONSTANTCONDITION)                                        \
1456         } while (B_FALSE)
1457
1458 #define EFX_CLEAR_WORD_BIT(_word, _bit)                                 \
1459         do {                                                            \
1460                 (_word).ew_u32[0] &=                                    \
1461                     __CPU_TO_LE_16(~EFX_SHIFT16(_bit, FIX_LINT(0)));    \
1462         _NOTE(CONSTANTCONDITION)                                        \
1463         } while (B_FALSE)
1464
1465 #define EFX_TEST_WORD_BIT(_word, _bit)                                  \
1466         (((_word).ew_u16[0] &                                           \
1467                     __CPU_TO_LE_16(EFX_SHIFT16(_bit, FIX_LINT(0)))) != 0)
1468
1469
1470 #define EFX_SET_BYTE_BIT(_byte, _bit)                                   \
1471         do {                                                            \
1472                 (_byte).eb_u8[0] |=                                     \
1473                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)));          \
1474         _NOTE(CONSTANTCONDITION)                                        \
1475         } while (B_FALSE)
1476
1477 #define EFX_CLEAR_BYTE_BIT(_byte, _bit)                                 \
1478         do {                                                            \
1479                 (_byte).eb_u8[0] &=                                     \
1480                     __NATIVE_8(~EFX_SHIFT8(_bit, FIX_LINT(0)));         \
1481         _NOTE(CONSTANTCONDITION)                                        \
1482         } while (B_FALSE)
1483
1484 #define EFX_TEST_BYTE_BIT(_byte, _bit)                                  \
1485         (((_byte).eb_u8[0] &                                            \
1486                     __NATIVE_8(EFX_SHIFT8(_bit, FIX_LINT(0)))) != 0)
1487
1488
1489 #define EFX_OR_OWORD64(_oword1, _oword2)                                \
1490         do {                                                            \
1491                 (_oword1).eo_u64[0] |= (_oword2).eo_u64[0];             \
1492                 (_oword1).eo_u64[1] |= (_oword2).eo_u64[1];             \
1493         _NOTE(CONSTANTCONDITION)                                        \
1494         } while (B_FALSE)
1495
1496 #define EFX_OR_OWORD32(_oword1, _oword2)                                \
1497         do {                                                            \
1498                 (_oword1).eo_u32[0] |= (_oword2).eo_u32[0];             \
1499                 (_oword1).eo_u32[1] |= (_oword2).eo_u32[1];             \
1500                 (_oword1).eo_u32[2] |= (_oword2).eo_u32[2];             \
1501                 (_oword1).eo_u32[3] |= (_oword2).eo_u32[3];             \
1502         _NOTE(CONSTANTCONDITION)                                        \
1503         } while (B_FALSE)
1504
1505 #define EFX_AND_OWORD64(_oword1, _oword2)                               \
1506         do {                                                            \
1507                 (_oword1).eo_u64[0] &= (_oword2).eo_u64[0];             \
1508                 (_oword1).eo_u64[1] &= (_oword2).eo_u64[1];             \
1509         _NOTE(CONSTANTCONDITION)                                        \
1510         } while (B_FALSE)
1511
1512 #define EFX_AND_OWORD32(_oword1, _oword2)                               \
1513         do {                                                            \
1514                 (_oword1).eo_u32[0] &= (_oword2).eo_u32[0];             \
1515                 (_oword1).eo_u32[1] &= (_oword2).eo_u32[1];             \
1516                 (_oword1).eo_u32[2] &= (_oword2).eo_u32[2];             \
1517                 (_oword1).eo_u32[3] &= (_oword2).eo_u32[3];             \
1518         _NOTE(CONSTANTCONDITION)                                        \
1519         } while (B_FALSE)
1520
1521 #define EFX_OR_QWORD64(_qword1, _qword2)                                \
1522         do {                                                            \
1523                 (_qword1).eq_u64[0] |= (_qword2).eq_u64[0];             \
1524         _NOTE(CONSTANTCONDITION)                                        \
1525         } while (B_FALSE)
1526
1527 #define EFX_OR_QWORD32(_qword1, _qword2)                                \
1528         do {                                                            \
1529                 (_qword1).eq_u32[0] |= (_qword2).eq_u32[0];             \
1530                 (_qword1).eq_u32[1] |= (_qword2).eq_u32[1];             \
1531         _NOTE(CONSTANTCONDITION)                                        \
1532         } while (B_FALSE)
1533
1534 #define EFX_AND_QWORD64(_qword1, _qword2)                               \
1535         do {                                                            \
1536                 (_qword1).eq_u64[0] &= (_qword2).eq_u64[0];             \
1537         _NOTE(CONSTANTCONDITION)                                        \
1538         } while (B_FALSE)
1539
1540 #define EFX_AND_QWORD32(_qword1, _qword2)                               \
1541         do {                                                            \
1542                 (_qword1).eq_u32[0] &= (_qword2).eq_u32[0];             \
1543                 (_qword1).eq_u32[1] &= (_qword2).eq_u32[1];             \
1544         _NOTE(CONSTANTCONDITION)                                        \
1545         } while (B_FALSE)
1546
1547 #define EFX_OR_DWORD(_dword1, _dword2)                                  \
1548         do {                                                            \
1549                 (_dword1).ed_u32[0] |= (_dword2).ed_u32[0];             \
1550         _NOTE(CONSTANTCONDITION)                                        \
1551         } while (B_FALSE)
1552
1553 #define EFX_AND_DWORD(_dword1, _dword2)                                 \
1554         do {                                                            \
1555                 (_dword1).ed_u32[0] &= (_dword2).ed_u32[0];             \
1556         _NOTE(CONSTANTCONDITION)                                        \
1557         } while (B_FALSE)
1558
1559 #define EFX_OR_WORD(_word1, _word2)                                     \
1560         do {                                                            \
1561                 (_word1).ew_u16[0] |= (_word2).ew_u16[0];               \
1562         _NOTE(CONSTANTCONDITION)                                        \
1563         } while (B_FALSE)
1564
1565 #define EFX_AND_WORD(_word1, _word2)                                    \
1566         do {                                                            \
1567                 (_word1).ew_u16[0] &= (_word2).ew_u16[0];               \
1568         _NOTE(CONSTANTCONDITION)                                        \
1569         } while (B_FALSE)
1570
1571 #define EFX_OR_BYTE(_byte1, _byte2)                                     \
1572         do {                                                            \
1573                 (_byte1).eb_u8[0] |= (_byte2).eb_u8[0];                 \
1574         _NOTE(CONSTANTCONDITION)                                        \
1575         } while (B_FALSE)
1576
1577 #define EFX_AND_BYTE(_byte1, _byte2)                                    \
1578         do {                                                            \
1579                 (_byte1).eb_u8[0] &= (_byte2).eb_u8[0];                 \
1580         _NOTE(CONSTANTCONDITION)                                        \
1581         } while (B_FALSE)
1582
1583 #if EFSYS_USE_UINT64
1584 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD64
1585 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD64
1586 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL64
1587 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL64
1588 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO64
1589 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO64
1590 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET64
1591 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET64
1592 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD64
1593 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD64
1594 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD64
1595 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD64
1596 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT64
1597 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT64
1598 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT64
1599 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT64
1600 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT64
1601 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT64
1602 #define EFX_OR_OWORD            EFX_OR_OWORD64
1603 #define EFX_AND_OWORD           EFX_AND_OWORD64
1604 #define EFX_OR_QWORD            EFX_OR_QWORD64
1605 #define EFX_AND_QWORD           EFX_AND_QWORD64
1606 #else
1607 #define EFX_OWORD_FIELD         EFX_OWORD_FIELD32
1608 #define EFX_QWORD_FIELD         EFX_QWORD_FIELD32
1609 #define EFX_OWORD_IS_EQUAL      EFX_OWORD_IS_EQUAL32
1610 #define EFX_QWORD_IS_EQUAL      EFX_QWORD_IS_EQUAL32
1611 #define EFX_OWORD_IS_ZERO       EFX_OWORD_IS_ZERO32
1612 #define EFX_QWORD_IS_ZERO       EFX_QWORD_IS_ZERO32
1613 #define EFX_OWORD_IS_SET        EFX_OWORD_IS_SET32
1614 #define EFX_QWORD_IS_SET        EFX_QWORD_IS_SET32
1615 #define EFX_POPULATE_OWORD      EFX_POPULATE_OWORD32
1616 #define EFX_POPULATE_QWORD      EFX_POPULATE_QWORD32
1617 #define EFX_SET_OWORD_FIELD     EFX_SET_OWORD_FIELD32
1618 #define EFX_SET_QWORD_FIELD     EFX_SET_QWORD_FIELD32
1619 #define EFX_SET_OWORD_BIT       EFX_SET_OWORD_BIT32
1620 #define EFX_CLEAR_OWORD_BIT     EFX_CLEAR_OWORD_BIT32
1621 #define EFX_TEST_OWORD_BIT      EFX_TEST_OWORD_BIT32
1622 #define EFX_SET_QWORD_BIT       EFX_SET_QWORD_BIT32
1623 #define EFX_CLEAR_QWORD_BIT     EFX_CLEAR_QWORD_BIT32
1624 #define EFX_TEST_QWORD_BIT      EFX_TEST_QWORD_BIT32
1625 #define EFX_OR_OWORD            EFX_OR_OWORD32
1626 #define EFX_AND_OWORD           EFX_AND_OWORD32
1627 #define EFX_OR_QWORD            EFX_OR_QWORD32
1628 #define EFX_AND_QWORD           EFX_AND_QWORD32
1629 #endif
1630
1631
1632 #ifdef  __cplusplus
1633 }
1634 #endif
1635
1636 #endif  /* _SYS_EFX_TYPES_H */