drivers/common/sfc_efx/efsys.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2019-2020 Xilinx, Inc.
   4  * Copyright(c) 2016-2019 Solarflare Communications Inc.
   5  *
   6  * This software was jointly developed between OKTET Labs (under contract
   7  * for Solarflare) and Solarflare Communications, Inc.
   8  */
   9
  10 #ifndef _SFC_COMMON_EFSYS_H
  11 #define _SFC_COMMON_EFSYS_H
  12
  13 #include <stdbool.h>
  14
  15 #include <rte_spinlock.h>
  16 #include <rte_byteorder.h>
  17 #include <rte_debug.h>
  18 #include <rte_memzone.h>
  19 #include <rte_memory.h>
  20 #include <rte_memcpy.h>
  21 #include <rte_cycles.h>
  22 #include <rte_prefetch.h>
  23 #include <rte_common.h>
  24 #include <rte_malloc.h>
  25 #include <rte_log.h>
  26 #include <rte_io.h>
  27
  28 #include "sfc_efx_debug.h"
  29 #include "sfc_efx_log.h"
  30
  31 #ifdef __cplusplus
  32 extern "C" {
  33 #endif
  34
  35 #define LIBEFX_API              __rte_internal
  36
  37 /* No specific decorations required since functions are local by default */
  38 #define LIBEFX_INTERNAL
  39
  40 #define EFSYS_HAS_UINT64 1
  41 #define EFSYS_USE_UINT64 1
  42 #define EFSYS_HAS_SSE2_M128 1
  43
  44 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
  45 #define EFSYS_IS_BIG_ENDIAN 1
  46 #define EFSYS_IS_LITTLE_ENDIAN 0
  47 #elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
  48 #define EFSYS_IS_BIG_ENDIAN 0
  49 #define EFSYS_IS_LITTLE_ENDIAN 1
  50 #else
  51 #error "Cannot determine system endianness"
  52 #endif
  53 #include "efx_types.h"
  54
  55
  56 typedef bool boolean_t;
  57
  58 #ifndef B_FALSE
  59 #define B_FALSE false
  60 #endif
  61 #ifndef B_TRUE
  62 #define B_TRUE  true
  63 #endif
  64
  65 /*
  66  * RTE_MAX() and RTE_MIN() cannot be used since braced-group within
  67  * expression allowed only inside a function, but MAX() is used as
  68  * a number of elements in array.
  69  */
  70 #ifndef MAX
  71 #define MAX(v1, v2)     ((v1) > (v2) ? (v1) : (v2))
  72 #endif
  73 #ifndef MIN
  74 #define MIN(v1, v2)     ((v1) < (v2) ? (v1) : (v2))
  75 #endif
  76
  77 #ifndef ISP2
  78 #define ISP2(x)                 rte_is_power_of_2(x)
  79 #endif
  80
  81 #define ENOTACTIVE      ENOTCONN
  82
  83 static inline void
  84 prefetch_read_many(const volatile void *addr)
  85 {
  86         rte_prefetch0(addr);
  87 }
  88
  89 static inline void
  90 prefetch_read_once(const volatile void *addr)
  91 {
  92         rte_prefetch_non_temporal(addr);
  93 }
  94
  95 /* Code inclusion options */
  96
  97
  98 #define EFSYS_OPT_NAMES 1
  99
 100 /* Disable SFN5xxx/SFN6xxx since it requires specific support in the PMD */
 101 #define EFSYS_OPT_SIENA 0
 102 /* Enable SFN7xxx support */
 103 #define EFSYS_OPT_HUNTINGTON 1
 104 /* Enable SFN8xxx support */
 105 #define EFSYS_OPT_MEDFORD 1
 106 /* Enable SFN2xxx support */
 107 #define EFSYS_OPT_MEDFORD2 1
 108 /* Disable Riverhead support */
 109 #define EFSYS_OPT_RIVERHEAD 0
 110
 111 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
 112 #define EFSYS_OPT_CHECK_REG 1
 113 #else
 114 #define EFSYS_OPT_CHECK_REG 0
 115 #endif
 116
 117 /* MCDI is required for SFN7xxx and SFN8xx */
 118 #define EFSYS_OPT_MCDI 1
 119 #define EFSYS_OPT_MCDI_LOGGING 1
 120 #define EFSYS_OPT_MCDI_PROXY_AUTH 1
 121
 122 #define EFSYS_OPT_MAC_STATS 1
 123
 124 #define EFSYS_OPT_LOOPBACK 1
 125
 126 #define EFSYS_OPT_MON_MCDI 0
 127 #define EFSYS_OPT_MON_STATS 0
 128
 129 #define EFSYS_OPT_PHY_STATS 0
 130 #define EFSYS_OPT_BIST 0
 131 #define EFSYS_OPT_PHY_LED_CONTROL 0
 132 #define EFSYS_OPT_PHY_FLAGS 0
 133
 134 #define EFSYS_OPT_VPD 0
 135 #define EFSYS_OPT_NVRAM 0
 136 #define EFSYS_OPT_BOOTCFG 0
 137 #define EFSYS_OPT_IMAGE_LAYOUT 0
 138
 139 #define EFSYS_OPT_DIAG 0
 140 #define EFSYS_OPT_RX_SCALE 1
 141 #define EFSYS_OPT_QSTATS 0
 142 /* Filters support is required for SFN7xxx and SFN8xx */
 143 #define EFSYS_OPT_FILTER 1
 144 #define EFSYS_OPT_RX_SCATTER 0
 145
 146 #define EFSYS_OPT_EV_PREFETCH 0
 147
 148 #define EFSYS_OPT_DECODE_INTR_FATAL 0
 149
 150 #define EFSYS_OPT_LICENSING 0
 151
 152 #define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0
 153
 154 #define EFSYS_OPT_RX_PACKED_STREAM 0
 155
 156 #define EFSYS_OPT_RX_ES_SUPER_BUFFER 1
 157
 158 #define EFSYS_OPT_TUNNEL 1
 159
 160 #define EFSYS_OPT_FW_SUBVARIANT_AWARE 1
 161
 162 #define EFSYS_OPT_EVB 0
 163
 164 #define EFSYS_OPT_MCDI_PROXY_AUTH_SERVER 0
 165
 166 /* ID */
 167
 168 typedef struct __efsys_identifier_s efsys_identifier_t;
 169
 170
 171 #define EFSYS_PROBE(_name)                                              \
 172         do { } while (0)
 173
 174 #define EFSYS_PROBE1(_name, _type1, _arg1)                              \
 175         do { } while (0)
 176
 177 #define EFSYS_PROBE2(_name, _type1, _arg1, _type2, _arg2)               \
 178         do { } while (0)
 179
 180 #define EFSYS_PROBE3(_name, _type1, _arg1, _type2, _arg2,               \
 181                      _type3, _arg3)                                     \
 182         do { } while (0)
 183
 184 #define EFSYS_PROBE4(_name, _type1, _arg1, _type2, _arg2,               \
 185                      _type3, _arg3, _type4, _arg4)                      \
 186         do { } while (0)
 187
 188 #define EFSYS_PROBE5(_name, _type1, _arg1, _type2, _arg2,               \
 189                      _type3, _arg3, _type4, _arg4, _type5, _arg5)       \
 190         do { } while (0)
 191
 192 #define EFSYS_PROBE6(_name, _type1, _arg1, _type2, _arg2,               \
 193                      _type3, _arg3, _type4, _arg4, _type5, _arg5,       \
 194                      _type6, _arg6)                                     \
 195         do { } while (0)
 196
 197 #define EFSYS_PROBE7(_name, _type1, _arg1, _type2, _arg2,               \
 198                      _type3, _arg3, _type4, _arg4, _type5, _arg5,       \
 199                      _type6, _arg6, _type7, _arg7)                      \
 200         do { } while (0)
 201
 202
 203 /* DMA */
 204
 205 typedef rte_iova_t efsys_dma_addr_t;
 206
 207 typedef struct efsys_mem_s {
 208         const struct rte_memzone        *esm_mz;
 209         /*
 210          * Ideally it should have volatile qualifier to denote that
 211          * the memory may be updated by someone else. However, it adds
 212          * qualifier discard warnings when the pointer or its derivative
 213          * is passed to memset() or rte_mov16().
 214          * So, skip the qualifier here, but make sure that it is added
 215          * below in access macros.
 216          */
 217         void                            *esm_base;
 218         efsys_dma_addr_t                esm_addr;
 219 } efsys_mem_t;
 220
 221
 222 #define EFSYS_MEM_ZERO(_esmp, _size)                                    \
 223         do {                                                            \
 224                 (void)memset((void *)(_esmp)->esm_base, 0, (_size));    \
 225                                                                         \
 226                 _NOTE(CONSTANTCONDITION);                               \
 227         } while (B_FALSE)
 228
 229 #define EFSYS_MEM_READD(_esmp, _offset, _edp)                           \
 230         do {                                                            \
 231                 volatile uint8_t  *_base = (_esmp)->esm_base;           \
 232                 volatile uint32_t *_addr;                               \
 233                                                                         \
 234                 _NOTE(CONSTANTCONDITION);                               \
 235                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 236                                                 sizeof(efx_dword_t)));  \
 237                                                                         \
 238                 _addr = (volatile uint32_t *)(_base + (_offset));       \
 239                 (_edp)->ed_u32[0] = _addr[0];                           \
 240                                                                         \
 241                 EFSYS_PROBE2(mem_readl, unsigned int, (_offset),        \
 242                                          uint32_t, (_edp)->ed_u32[0]);  \
 243                                                                         \
 244                 _NOTE(CONSTANTCONDITION);                               \
 245         } while (B_FALSE)
 246
 247 #define EFSYS_MEM_READQ(_esmp, _offset, _eqp)                           \
 248         do {                                                            \
 249                 volatile uint8_t  *_base = (_esmp)->esm_base;           \
 250                 volatile uint64_t *_addr;                               \
 251                                                                         \
 252                 _NOTE(CONSTANTCONDITION);                               \
 253                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 254                                                 sizeof(efx_qword_t)));  \
 255                                                                         \
 256                 _addr = (volatile uint64_t *)(_base + (_offset));       \
 257                 (_eqp)->eq_u64[0] = _addr[0];                           \
 258                                                                         \
 259                 EFSYS_PROBE3(mem_readq, unsigned int, (_offset),        \
 260                                          uint32_t, (_eqp)->eq_u32[1],   \
 261                                          uint32_t, (_eqp)->eq_u32[0]);  \
 262                                                                         \
 263                 _NOTE(CONSTANTCONDITION);                               \
 264         } while (B_FALSE)
 265
 266 #define EFSYS_MEM_READO(_esmp, _offset, _eop)                           \
 267         do {                                                            \
 268                 volatile uint8_t *_base = (_esmp)->esm_base;            \
 269                 volatile __m128i *_addr;                                \
 270                                                                         \
 271                 _NOTE(CONSTANTCONDITION);                               \
 272                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 273                                                 sizeof(efx_oword_t)));  \
 274                                                                         \
 275                 _addr = (volatile __m128i *)(_base + (_offset));        \
 276                 (_eop)->eo_u128[0] = _addr[0];                          \
 277                                                                         \
 278                 EFSYS_PROBE5(mem_reado, unsigned int, (_offset),        \
 279                                          uint32_t, (_eop)->eo_u32[3],   \
 280                                          uint32_t, (_eop)->eo_u32[2],   \
 281                                          uint32_t, (_eop)->eo_u32[1],   \
 282                                          uint32_t, (_eop)->eo_u32[0]);  \
 283                                                                         \
 284                 _NOTE(CONSTANTCONDITION);                               \
 285         } while (B_FALSE)
 286
 287
 288 #define EFSYS_MEM_WRITED(_esmp, _offset, _edp)                          \
 289         do {                                                            \
 290                 volatile uint8_t  *_base = (_esmp)->esm_base;           \
 291                 volatile uint32_t *_addr;                               \
 292                                                                         \
 293                 _NOTE(CONSTANTCONDITION);                               \
 294                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 295                                                 sizeof(efx_dword_t)));  \
 296                                                                         \
 297                 EFSYS_PROBE2(mem_writed, unsigned int, (_offset),       \
 298                                          uint32_t, (_edp)->ed_u32[0]);  \
 299                                                                         \
 300                 _addr = (volatile uint32_t *)(_base + (_offset));       \
 301                 _addr[0] = (_edp)->ed_u32[0];                           \
 302                                                                         \
 303                 _NOTE(CONSTANTCONDITION);                               \
 304         } while (B_FALSE)
 305
 306 #define EFSYS_MEM_WRITEQ(_esmp, _offset, _eqp)                          \
 307         do {                                                            \
 308                 volatile uint8_t  *_base = (_esmp)->esm_base;           \
 309                 volatile uint64_t *_addr;                               \
 310                                                                         \
 311                 _NOTE(CONSTANTCONDITION);                               \
 312                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 313                                                 sizeof(efx_qword_t)));  \
 314                                                                         \
 315                 EFSYS_PROBE3(mem_writeq, unsigned int, (_offset),       \
 316                                          uint32_t, (_eqp)->eq_u32[1],   \
 317                                          uint32_t, (_eqp)->eq_u32[0]);  \
 318                                                                         \
 319                 _addr = (volatile uint64_t *)(_base + (_offset));       \
 320                 _addr[0] = (_eqp)->eq_u64[0];                           \
 321                                                                         \
 322                 _NOTE(CONSTANTCONDITION);                               \
 323         } while (B_FALSE)
 324
 325 #define EFSYS_MEM_WRITEO(_esmp, _offset, _eop)                          \
 326         do {                                                            \
 327                 volatile uint8_t *_base = (_esmp)->esm_base;            \
 328                 volatile __m128i *_addr;                                \
 329                                                                         \
 330                 _NOTE(CONSTANTCONDITION);                               \
 331                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 332                                                 sizeof(efx_oword_t)));  \
 333                                                                         \
 334                                                                         \
 335                 EFSYS_PROBE5(mem_writeo, unsigned int, (_offset),       \
 336                                          uint32_t, (_eop)->eo_u32[3],   \
 337                                          uint32_t, (_eop)->eo_u32[2],   \
 338                                          uint32_t, (_eop)->eo_u32[1],   \
 339                                          uint32_t, (_eop)->eo_u32[0]);  \
 340                                                                         \
 341                 _addr = (volatile __m128i *)(_base + (_offset));        \
 342                 _addr[0] = (_eop)->eo_u128[0];                          \
 343                                                                         \
 344                 _NOTE(CONSTANTCONDITION);                               \
 345         } while (B_FALSE)
 346
 347
 348 #define EFSYS_MEM_SIZE(_esmp)                                           \
 349         ((_esmp)->esm_mz->len)
 350
 351 #define EFSYS_MEM_ADDR(_esmp)                                           \
 352         ((_esmp)->esm_addr)
 353
 354 #define EFSYS_MEM_IS_NULL(_esmp)                                        \
 355         ((_esmp)->esm_base == NULL)
 356
 357 #define EFSYS_MEM_PREFETCH(_esmp, _offset)                              \
 358         do {                                                            \
 359                 volatile uint8_t *_base = (_esmp)->esm_base;            \
 360                                                                         \
 361                 rte_prefetch0(_base + (_offset));                       \
 362         } while (0)
 363
 364
 365 /* BAR */
 366
 367 typedef struct efsys_bar_s {
 368         rte_spinlock_t          esb_lock;
 369         int                     esb_rid;
 370         struct rte_pci_device   *esb_dev;
 371         /*
 372          * Ideally it should have volatile qualifier to denote that
 373          * the memory may be updated by someone else. However, it adds
 374          * qualifier discard warnings when the pointer or its derivative
 375          * is passed to memset() or rte_mov16().
 376          * So, skip the qualifier here, but make sure that it is added
 377          * below in access macros.
 378          */
 379         void                    *esb_base;
 380 } efsys_bar_t;
 381
 382 #define SFC_BAR_LOCK_INIT(_esbp, _ifname)                               \
 383         do {                                                            \
 384                 rte_spinlock_init(&(_esbp)->esb_lock);                  \
 385                 _NOTE(CONSTANTCONDITION);                               \
 386         } while (B_FALSE)
 387 #define SFC_BAR_LOCK_DESTROY(_esbp)     ((void)0)
 388 #define SFC_BAR_LOCK(_esbp)             rte_spinlock_lock(&(_esbp)->esb_lock)
 389 #define SFC_BAR_UNLOCK(_esbp)           rte_spinlock_unlock(&(_esbp)->esb_lock)
 390
 391 #define EFSYS_BAR_READD(_esbp, _offset, _edp, _lock)                    \
 392         do {                                                            \
 393                 volatile uint8_t  *_base = (_esbp)->esb_base;           \
 394                 volatile uint32_t *_addr;                               \
 395                                                                         \
 396                 _NOTE(CONSTANTCONDITION);                               \
 397                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 398                                                 sizeof(efx_dword_t)));  \
 399                 _NOTE(CONSTANTCONDITION);                               \
 400                 if (_lock)                                              \
 401                         SFC_BAR_LOCK(_esbp);                            \
 402                                                                         \
 403                 _addr = (volatile uint32_t *)(_base + (_offset));       \
 404                 rte_rmb();                                              \
 405                 (_edp)->ed_u32[0] = rte_read32_relaxed(_addr);          \
 406                                                                         \
 407                 EFSYS_PROBE2(bar_readd, unsigned int, (_offset),        \
 408                                          uint32_t, (_edp)->ed_u32[0]);  \
 409                                                                         \
 410                 _NOTE(CONSTANTCONDITION);                               \
 411                 if (_lock)                                              \
 412                         SFC_BAR_UNLOCK(_esbp);                          \
 413                 _NOTE(CONSTANTCONDITION);                               \
 414         } while (B_FALSE)
 415
 416 #define EFSYS_BAR_READQ(_esbp, _offset, _eqp)                           \
 417         do {                                                            \
 418                 volatile uint8_t  *_base = (_esbp)->esb_base;           \
 419                 volatile uint64_t *_addr;                               \
 420                                                                         \
 421                 _NOTE(CONSTANTCONDITION);                               \
 422                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 423                                                 sizeof(efx_qword_t)));  \
 424                                                                         \
 425                 SFC_BAR_LOCK(_esbp);                                    \
 426                                                                         \
 427                 _addr = (volatile uint64_t *)(_base + (_offset));       \
 428                 rte_rmb();                                              \
 429                 (_eqp)->eq_u64[0] = rte_read64_relaxed(_addr);          \
 430                                                                         \
 431                 EFSYS_PROBE3(bar_readq, unsigned int, (_offset),        \
 432                                          uint32_t, (_eqp)->eq_u32[1],   \
 433                                          uint32_t, (_eqp)->eq_u32[0]);  \
 434                                                                         \
 435                 SFC_BAR_UNLOCK(_esbp);                                  \
 436                 _NOTE(CONSTANTCONDITION);                               \
 437         } while (B_FALSE)
 438
 439 #define EFSYS_BAR_READO(_esbp, _offset, _eop, _lock)                    \
 440         do {                                                            \
 441                 volatile uint8_t *_base = (_esbp)->esb_base;            \
 442                 volatile __m128i *_addr;                                \
 443                                                                         \
 444                 _NOTE(CONSTANTCONDITION);                               \
 445                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 446                                                 sizeof(efx_oword_t)));  \
 447                                                                         \
 448                 _NOTE(CONSTANTCONDITION);                               \
 449                 if (_lock)                                              \
 450                         SFC_BAR_LOCK(_esbp);                            \
 451                                                                         \
 452                 _addr = (volatile __m128i *)(_base + (_offset));        \
 453                 rte_rmb();                                              \
 454                 /* There is no rte_read128_relaxed() yet */             \
 455                 (_eop)->eo_u128[0] = _addr[0];                          \
 456                                                                         \
 457                 EFSYS_PROBE5(bar_reado, unsigned int, (_offset),        \
 458                                          uint32_t, (_eop)->eo_u32[3],   \
 459                                          uint32_t, (_eop)->eo_u32[2],   \
 460                                          uint32_t, (_eop)->eo_u32[1],   \
 461                                          uint32_t, (_eop)->eo_u32[0]);  \
 462                                                                         \
 463                 _NOTE(CONSTANTCONDITION);                               \
 464                 if (_lock)                                              \
 465                         SFC_BAR_UNLOCK(_esbp);                          \
 466                 _NOTE(CONSTANTCONDITION);                               \
 467         } while (B_FALSE)
 468
 469
 470 #define EFSYS_BAR_WRITED(_esbp, _offset, _edp, _lock)                   \
 471         do {                                                            \
 472                 volatile uint8_t  *_base = (_esbp)->esb_base;           \
 473                 volatile uint32_t *_addr;                               \
 474                                                                         \
 475                 _NOTE(CONSTANTCONDITION);                               \
 476                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 477                                                 sizeof(efx_dword_t)));  \
 478                                                                         \
 479                 _NOTE(CONSTANTCONDITION);                               \
 480                 if (_lock)                                              \
 481                         SFC_BAR_LOCK(_esbp);                            \
 482                                                                         \
 483                 EFSYS_PROBE2(bar_writed, unsigned int, (_offset),       \
 484                                          uint32_t, (_edp)->ed_u32[0]);  \
 485                                                                         \
 486                 _addr = (volatile uint32_t *)(_base + (_offset));       \
 487                 rte_write32_relaxed((_edp)->ed_u32[0], _addr);          \
 488                 rte_wmb();                                              \
 489                                                                         \
 490                 _NOTE(CONSTANTCONDITION);                               \
 491                 if (_lock)                                              \
 492                         SFC_BAR_UNLOCK(_esbp);                          \
 493                 _NOTE(CONSTANTCONDITION);                               \
 494         } while (B_FALSE)
 495
 496 #define EFSYS_BAR_WRITEQ(_esbp, _offset, _eqp)                          \
 497         do {                                                            \
 498                 volatile uint8_t  *_base = (_esbp)->esb_base;           \
 499                 volatile uint64_t *_addr;                               \
 500                                                                         \
 501                 _NOTE(CONSTANTCONDITION);                               \
 502                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 503                                                 sizeof(efx_qword_t)));  \
 504                                                                         \
 505                 SFC_BAR_LOCK(_esbp);                                    \
 506                                                                         \
 507                 EFSYS_PROBE3(bar_writeq, unsigned int, (_offset),       \
 508                                          uint32_t, (_eqp)->eq_u32[1],   \
 509                                          uint32_t, (_eqp)->eq_u32[0]);  \
 510                                                                         \
 511                 _addr = (volatile uint64_t *)(_base + (_offset));       \
 512                 rte_write64_relaxed((_eqp)->eq_u64[0], _addr);          \
 513                 rte_wmb();                                              \
 514                                                                         \
 515                 SFC_BAR_UNLOCK(_esbp);                                  \
 516                 _NOTE(CONSTANTCONDITION);                               \
 517         } while (B_FALSE)
 518
 519 /*
 520  * Guarantees 64bit aligned 64bit writes to write combined BAR mapping
 521  * (required by PIO hardware).
 522  *
 523  * Neither VFIO, nor UIO, nor NIC UIO (on FreeBSD) support
 524  * write-combined memory mapped to user-land, so just abort if used.
 525  */
 526 #define EFSYS_BAR_WC_WRITEQ(_esbp, _offset, _eqp)                       \
 527         do {                                                            \
 528                 rte_panic("Write-combined BAR access not supported");   \
 529         } while (B_FALSE)
 530
 531 #define EFSYS_BAR_WRITEO(_esbp, _offset, _eop, _lock)                   \
 532         do {                                                            \
 533                 volatile uint8_t *_base = (_esbp)->esb_base;            \
 534                 volatile __m128i *_addr;                                \
 535                                                                         \
 536                 _NOTE(CONSTANTCONDITION);                               \
 537                 SFC_EFX_ASSERT(EFX_IS_P2ALIGNED(size_t, _offset,        \
 538                                                 sizeof(efx_oword_t)));  \
 539                                                                         \
 540                 _NOTE(CONSTANTCONDITION);                               \
 541                 if (_lock)                                              \
 542                         SFC_BAR_LOCK(_esbp);                            \
 543                                                                         \
 544                 EFSYS_PROBE5(bar_writeo, unsigned int, (_offset),       \
 545                                          uint32_t, (_eop)->eo_u32[3],   \
 546                                          uint32_t, (_eop)->eo_u32[2],   \
 547                                          uint32_t, (_eop)->eo_u32[1],   \
 548                                          uint32_t, (_eop)->eo_u32[0]);  \
 549                                                                         \
 550                 _addr = (volatile __m128i *)(_base + (_offset));        \
 551                 /* There is no rte_write128_relaxed() yet */            \
 552                 _addr[0] = (_eop)->eo_u128[0];                          \
 553                 rte_wmb();                                              \
 554                                                                         \
 555                 _NOTE(CONSTANTCONDITION);                               \
 556                 if (_lock)                                              \
 557                         SFC_BAR_UNLOCK(_esbp);                          \
 558                 _NOTE(CONSTANTCONDITION);                               \
 559         } while (B_FALSE)
 560
 561 /* Use the standard octo-word write for doorbell writes */
 562 #define EFSYS_BAR_DOORBELL_WRITEO(_esbp, _offset, _eop)                 \
 563         do {                                                            \
 564                 EFSYS_BAR_WRITEO((_esbp), (_offset), (_eop), B_FALSE);  \
 565                 _NOTE(CONSTANTCONDITION);                               \
 566         } while (B_FALSE)
 567
 568 /* SPIN */
 569
 570 #define EFSYS_SPIN(_us)                                                 \
 571         do {                                                            \
 572                 rte_delay_us(_us);                                      \
 573                 _NOTE(CONSTANTCONDITION);                               \
 574         } while (B_FALSE)
 575
 576 #define EFSYS_SLEEP EFSYS_SPIN
 577
 578 /* BARRIERS */
 579
 580 #define EFSYS_MEM_READ_BARRIER()        rte_rmb()
 581 #define EFSYS_PIO_WRITE_BARRIER()       rte_io_wmb()
 582
 583 /* DMA SYNC */
 584
 585 /*
 586  * DPDK does not provide any DMA syncing API, and no PMD drivers
 587  * have any traces of explicit DMA syncing.
 588  * DMA mapping is assumed to be coherent.
 589  */
 590
 591 #define EFSYS_DMA_SYNC_FOR_KERNEL(_esmp, _offset, _size)        ((void)0)
 592
 593 /* Just avoid store and compiler (impliciltly) reordering */
 594 #define EFSYS_DMA_SYNC_FOR_DEVICE(_esmp, _offset, _size)        rte_wmb()
 595
 596 /* TIMESTAMP */
 597
 598 typedef uint64_t efsys_timestamp_t;
 599
 600 #define EFSYS_TIMESTAMP(_usp)                                           \
 601         do {                                                            \
 602                 *(_usp) = rte_get_timer_cycles() * 1000000 /            \
 603                         rte_get_timer_hz();                             \
 604                 _NOTE(CONSTANTCONDITION);                               \
 605         } while (B_FALSE)
 606
 607 /* KMEM */
 608
 609 #define EFSYS_KMEM_ALLOC(_esip, _size, _p)                              \
 610         do {                                                            \
 611                 (_esip) = (_esip);                                      \
 612                 (_p) = rte_zmalloc("sfc", (_size), 0);                  \
 613                 _NOTE(CONSTANTCONDITION);                               \
 614         } while (B_FALSE)
 615
 616 #define EFSYS_KMEM_FREE(_esip, _size, _p)                               \
 617         do {                                                            \
 618                 (void)(_esip);                                          \
 619                 (void)(_size);                                          \
 620                 rte_free((_p));                                         \
 621                 _NOTE(CONSTANTCONDITION);                               \
 622         } while (B_FALSE)
 623
 624 /* LOCK */
 625
 626 typedef rte_spinlock_t efsys_lock_t;
 627
 628 #define SFC_EFSYS_LOCK_INIT(_eslp, _ifname, _label)     \
 629         rte_spinlock_init((_eslp))
 630 #define SFC_EFSYS_LOCK_DESTROY(_eslp) ((void)0)
 631 #define SFC_EFSYS_LOCK(_eslp)                           \
 632         rte_spinlock_lock((_eslp))
 633 #define SFC_EFSYS_UNLOCK(_eslp)                         \
 634         rte_spinlock_unlock((_eslp))
 635 #define SFC_EFSYS_LOCK_ASSERT_OWNED(_eslp)              \
 636         SFC_EFX_ASSERT(rte_spinlock_is_locked((_eslp)))
 637
 638 typedef int efsys_lock_state_t;
 639
 640 #define EFSYS_LOCK_MAGIC        0x000010c4
 641
 642 #define EFSYS_LOCK(_lockp, _state)                              \
 643         do {                                                    \
 644                 SFC_EFSYS_LOCK(_lockp);                         \
 645                 (_state) = EFSYS_LOCK_MAGIC;                    \
 646                 _NOTE(CONSTANTCONDITION);                       \
 647         } while (B_FALSE)
 648
 649 #define EFSYS_UNLOCK(_lockp, _state)                            \
 650         do {                                                    \
 651                 SFC_EFX_ASSERT((_state) == EFSYS_LOCK_MAGIC);   \
 652                 SFC_EFSYS_UNLOCK(_lockp);                       \
 653                 _NOTE(CONSTANTCONDITION);                       \
 654         } while (B_FALSE)
 655
 656 /* STAT */
 657
 658 typedef uint64_t        efsys_stat_t;
 659
 660 #define EFSYS_STAT_INCR(_knp, _delta)                           \
 661         do {                                                    \
 662                 *(_knp) += (_delta);                            \
 663                 _NOTE(CONSTANTCONDITION);                       \
 664         } while (B_FALSE)
 665
 666 #define EFSYS_STAT_DECR(_knp, _delta)                           \
 667         do {                                                    \
 668                 *(_knp) -= (_delta);                            \
 669                 _NOTE(CONSTANTCONDITION);                       \
 670         } while (B_FALSE)
 671
 672 #define EFSYS_STAT_SET(_knp, _val)                              \
 673         do {                                                    \
 674                 *(_knp) = (_val);                               \
 675                 _NOTE(CONSTANTCONDITION);                       \
 676         } while (B_FALSE)
 677
 678 #define EFSYS_STAT_SET_QWORD(_knp, _valp)                       \
 679         do {                                                    \
 680                 *(_knp) = rte_le_to_cpu_64((_valp)->eq_u64[0]); \
 681                 _NOTE(CONSTANTCONDITION);                       \
 682         } while (B_FALSE)
 683
 684 #define EFSYS_STAT_SET_DWORD(_knp, _valp)                       \
 685         do {                                                    \
 686                 *(_knp) = rte_le_to_cpu_32((_valp)->ed_u32[0]); \
 687                 _NOTE(CONSTANTCONDITION);                       \
 688         } while (B_FALSE)
 689
 690 #define EFSYS_STAT_INCR_QWORD(_knp, _valp)                              \
 691         do {                                                            \
 692                 *(_knp) += rte_le_to_cpu_64((_valp)->eq_u64[0]);        \
 693                 _NOTE(CONSTANTCONDITION);                               \
 694         } while (B_FALSE)
 695
 696 #define EFSYS_STAT_SUBR_QWORD(_knp, _valp)                              \
 697         do {                                                            \
 698                 *(_knp) -= rte_le_to_cpu_64((_valp)->eq_u64[0]);        \
 699                 _NOTE(CONSTANTCONDITION);                               \
 700         } while (B_FALSE)
 701
 702 /* ERR */
 703
 704 #if EFSYS_OPT_DECODE_INTR_FATAL
 705 #define EFSYS_ERR(_esip, _code, _dword0, _dword1)                       \
 706         do {                                                            \
 707                 (void)(_esip);                                          \
 708                 SFC_EFX_LOG(ERR, "FATAL ERROR #%u (0x%08x%08x)",        \
 709                         (_code), (_dword0), (_dword1));                 \
 710                 _NOTE(CONSTANTCONDITION);                               \
 711         } while (B_FALSE)
 712 #endif
 713
 714 /* ASSERT */
 715
 716 /* RTE_VERIFY from DPDK treats expressions with % operator incorrectly,
 717  * so we re-implement it here
 718  */
 719 #ifdef RTE_LIBRTE_SFC_EFX_DEBUG
 720 #define EFSYS_ASSERT(_exp)                                              \
 721         do {                                                            \
 722                 if (unlikely(!(_exp)))                                  \
 723                         rte_panic("line %d\tassert \"%s\" failed\n",    \
 724                                   __LINE__, (#_exp));                   \
 725         } while (0)
 726 #else
 727 #define EFSYS_ASSERT(_exp)              (void)(_exp)
 728 #endif
 729
 730 #define EFSYS_ASSERT3(_x, _op, _y, _t)  EFSYS_ASSERT((_t)(_x) _op (_t)(_y))
 731
 732 #define EFSYS_ASSERT3U(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, uint64_t)
 733 #define EFSYS_ASSERT3S(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, int64_t)
 734 #define EFSYS_ASSERT3P(_x, _op, _y)     EFSYS_ASSERT3(_x, _op, _y, uintptr_t)
 735
 736 /* ROTATE */
 737
 738 #define EFSYS_HAS_ROTL_DWORD    0
 739
 740 #ifdef __cplusplus
 741 }
 742 #endif
 743
 744 #endif  /* _SFC_COMMON_EFSYS_H */