drivers/common/sfc_efx/base/efx_filter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2019-2020 Xilinx, Inc.
   4  * Copyright(c) 2007-2019 Solarflare Communications Inc.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_FILTER
  12
  13 #if EFSYS_OPT_SIENA
  14
  15 static  __checkReturn   efx_rc_t
  16 siena_filter_init(
  17         __in            efx_nic_t *enp);
  18
  19 static                  void
  20 siena_filter_fini(
  21         __in            efx_nic_t *enp);
  22
  23 static  __checkReturn   efx_rc_t
  24 siena_filter_restore(
  25         __in            efx_nic_t *enp);
  26
  27 static  __checkReturn   efx_rc_t
  28 siena_filter_add(
  29         __in            efx_nic_t *enp,
  30         __inout         efx_filter_spec_t *spec,
  31         __in            efx_filter_replacement_policy_t policy);
  32
  33 static  __checkReturn   efx_rc_t
  34 siena_filter_delete(
  35         __in            efx_nic_t *enp,
  36         __inout         efx_filter_spec_t *spec);
  37
  38 static  __checkReturn   efx_rc_t
  39 siena_filter_supported_filters(
  40         __in                            efx_nic_t *enp,
  41         __out_ecount(buffer_length)     uint32_t *buffer,
  42         __in                            size_t buffer_length,
  43         __out                           size_t *list_lengthp);
  44
  45 #endif /* EFSYS_OPT_SIENA */
  46
  47 #if EFSYS_OPT_SIENA
  48 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  49         siena_filter_init,              /* efo_init */
  50         siena_filter_fini,              /* efo_fini */
  51         siena_filter_restore,           /* efo_restore */
  52         siena_filter_add,               /* efo_add */
  53         siena_filter_delete,            /* efo_delete */
  54         siena_filter_supported_filters, /* efo_supported_filters */
  55         NULL,                           /* efo_reconfigure */
  56 };
  57 #endif /* EFSYS_OPT_SIENA */
  58
  59 #if EFX_OPTS_EF10()
  60 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  61         ef10_filter_init,               /* efo_init */
  62         ef10_filter_fini,               /* efo_fini */
  63         ef10_filter_restore,            /* efo_restore */
  64         ef10_filter_add,                /* efo_add */
  65         ef10_filter_delete,             /* efo_delete */
  66         ef10_filter_supported_filters,  /* efo_supported_filters */
  67         ef10_filter_reconfigure,        /* efo_reconfigure */
  68 };
  69 #endif /* EFX_OPTS_EF10() */
  70
  71 #if EFSYS_OPT_RIVERHEAD
  72 static const efx_filter_ops_t   __efx_filter_rhead_ops = {
  73         ef10_filter_init,               /* efo_init */
  74         ef10_filter_fini,               /* efo_fini */
  75         ef10_filter_restore,            /* efo_restore */
  76         ef10_filter_add,                /* efo_add */
  77         ef10_filter_delete,             /* efo_delete */
  78         ef10_filter_supported_filters,  /* efo_supported_filters */
  79         ef10_filter_reconfigure,        /* efo_reconfigure */
  80 };
  81 #endif /* EFSYS_OPT_RIVERHEAD */
  82
  83         __checkReturn   efx_rc_t
  84 efx_filter_insert(
  85         __in            efx_nic_t *enp,
  86         __inout         efx_filter_spec_t *spec)
  87 {
  88         const efx_filter_ops_t *efop = enp->en_efop;
  89         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
  90         efx_rc_t rc;
  91
  92         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
  93         EFSYS_ASSERT3P(spec, !=, NULL);
  94         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
  95
  96         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) &&
  97             !encp->enc_filter_action_mark_supported) {
  98                 rc = ENOTSUP;
  99                 goto fail1;
 100         }
 101
 102         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG) &&
 103             !encp->enc_filter_action_flag_supported) {
 104                 rc = ENOTSUP;
 105                 goto fail2;
 106         }
 107
 108         if (spec->efs_priority == EFX_FILTER_PRI_AUTO) {
 109                 rc = EINVAL;
 110                 goto fail3;
 111         }
 112
 113         return (efop->efo_add(enp, spec,
 114             EFX_FILTER_REPLACEMENT_HIGHER_PRIORITY));
 115
 116 fail3:
 117         EFSYS_PROBE(fail3);
 118 fail2:
 119         EFSYS_PROBE(fail2);
 120 fail1:
 121         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 122
 123         return (rc);
 124 }
 125
 126         __checkReturn   efx_rc_t
 127 efx_filter_remove(
 128         __in            efx_nic_t *enp,
 129         __inout         efx_filter_spec_t *spec)
 130 {
 131         const efx_filter_ops_t *efop = enp->en_efop;
 132
 133         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 134         EFSYS_ASSERT3P(spec, !=, NULL);
 135         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 136
 137         return (efop->efo_delete(enp, spec));
 138 }
 139
 140         __checkReturn   efx_rc_t
 141 efx_filter_restore(
 142         __in            efx_nic_t *enp)
 143 {
 144         efx_rc_t rc;
 145
 146         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 147
 148         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 149                 goto fail1;
 150
 151         return (0);
 152
 153 fail1:
 154         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 155
 156         return (rc);
 157 }
 158
 159         __checkReturn   efx_rc_t
 160 efx_filter_init(
 161         __in            efx_nic_t *enp)
 162 {
 163         const efx_filter_ops_t *efop;
 164         efx_rc_t rc;
 165
 166         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 167         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 168         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 169
 170         switch (enp->en_family) {
 171 #if EFSYS_OPT_SIENA
 172         case EFX_FAMILY_SIENA:
 173                 efop = &__efx_filter_siena_ops;
 174                 break;
 175 #endif /* EFSYS_OPT_SIENA */
 176
 177 #if EFSYS_OPT_HUNTINGTON
 178         case EFX_FAMILY_HUNTINGTON:
 179                 efop = &__efx_filter_ef10_ops;
 180                 break;
 181 #endif /* EFSYS_OPT_HUNTINGTON */
 182
 183 #if EFSYS_OPT_MEDFORD
 184         case EFX_FAMILY_MEDFORD:
 185                 efop = &__efx_filter_ef10_ops;
 186                 break;
 187 #endif /* EFSYS_OPT_MEDFORD */
 188
 189 #if EFSYS_OPT_MEDFORD2
 190         case EFX_FAMILY_MEDFORD2:
 191                 efop = &__efx_filter_ef10_ops;
 192                 break;
 193 #endif /* EFSYS_OPT_MEDFORD2 */
 194
 195 #if EFSYS_OPT_RIVERHEAD
 196         case EFX_FAMILY_RIVERHEAD:
 197                 efop = &__efx_filter_rhead_ops;
 198                 break;
 199 #endif /* EFSYS_OPT_RIVERHEAD */
 200
 201         default:
 202                 EFSYS_ASSERT(0);
 203                 rc = ENOTSUP;
 204                 goto fail1;
 205         }
 206
 207         if ((rc = efop->efo_init(enp)) != 0)
 208                 goto fail2;
 209
 210         enp->en_efop = efop;
 211         enp->en_mod_flags |= EFX_MOD_FILTER;
 212         return (0);
 213
 214 fail2:
 215         EFSYS_PROBE(fail2);
 216 fail1:
 217         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 218
 219         enp->en_efop = NULL;
 220         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 221         return (rc);
 222 }
 223
 224                         void
 225 efx_filter_fini(
 226         __in            efx_nic_t *enp)
 227 {
 228         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 229         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 230         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 231
 232         enp->en_efop->efo_fini(enp);
 233
 234         enp->en_efop = NULL;
 235         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 236 }
 237
 238 /*
 239  * Query the possible combinations of match flags which can be filtered on.
 240  * These are returned as a list, of which each 32 bit element is a bitmask
 241  * formed of EFX_FILTER_MATCH flags.
 242  *
 243  * The combinations are ordered in priority from highest to lowest.
 244  *
 245  * If the provided buffer is too short to hold the list, the call with fail with
 246  * ENOSPC and *list_lengthp will be set to the buffer length required.
 247  */
 248         __checkReturn   efx_rc_t
 249 efx_filter_supported_filters(
 250         __in                            efx_nic_t *enp,
 251         __out_ecount(buffer_length)     uint32_t *buffer,
 252         __in                            size_t buffer_length,
 253         __out                           size_t *list_lengthp)
 254 {
 255         efx_rc_t rc;
 256
 257         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 258         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 259         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 260         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 261
 262         if (buffer == NULL) {
 263                 rc = EINVAL;
 264                 goto fail1;
 265         }
 266
 267         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 268                                                     list_lengthp);
 269         if (rc != 0)
 270                 goto fail2;
 271
 272         return (0);
 273
 274 fail2:
 275         EFSYS_PROBE(fail2);
 276 fail1:
 277         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 278
 279         return (rc);
 280 }
 281
 282         __checkReturn   efx_rc_t
 283 efx_filter_reconfigure(
 284         __in                            efx_nic_t *enp,
 285         __in_ecount(6)                  uint8_t const *mac_addr,
 286         __in                            boolean_t all_unicst,
 287         __in                            boolean_t mulcst,
 288         __in                            boolean_t all_mulcst,
 289         __in                            boolean_t brdcst,
 290         __in_ecount(6*count)            uint8_t const *addrs,
 291         __in                            uint32_t count)
 292 {
 293         efx_rc_t rc;
 294
 295         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 296         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 297         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 298
 299         if (enp->en_efop->efo_reconfigure != NULL) {
 300                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 301                                                         all_unicst, mulcst,
 302                                                         all_mulcst, brdcst,
 303                                                         addrs, count)) != 0)
 304                         goto fail1;
 305         }
 306
 307         return (0);
 308
 309 fail1:
 310         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 311
 312         return (rc);
 313 }
 314
 315                 void
 316 efx_filter_spec_init_rx(
 317         __out           efx_filter_spec_t *spec,
 318         __in            efx_filter_priority_t priority,
 319         __in            efx_filter_flags_t flags,
 320         __in            efx_rxq_t *erp)
 321 {
 322         EFSYS_ASSERT3P(spec, !=, NULL);
 323         EFSYS_ASSERT3P(erp, !=, NULL);
 324         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 325                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 326
 327         memset(spec, 0, sizeof (*spec));
 328         spec->efs_priority = priority;
 329         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 330         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 331         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 332 }
 333
 334                 void
 335 efx_filter_spec_init_tx(
 336         __out           efx_filter_spec_t *spec,
 337         __in            efx_txq_t *etp)
 338 {
 339         EFSYS_ASSERT3P(spec, !=, NULL);
 340         EFSYS_ASSERT3P(etp, !=, NULL);
 341
 342         memset(spec, 0, sizeof (*spec));
 343         spec->efs_priority = EFX_FILTER_PRI_MANUAL;
 344         spec->efs_flags = EFX_FILTER_FLAG_TX;
 345         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 346 }
 347
 348
 349 /*
 350  *  Specify IPv4 host, transport protocol and port in a filter specification
 351  */
 352 __checkReturn           efx_rc_t
 353 efx_filter_spec_set_ipv4_local(
 354         __inout         efx_filter_spec_t *spec,
 355         __in            uint8_t proto,
 356         __in            uint32_t host,
 357         __in            uint16_t port)
 358 {
 359         EFSYS_ASSERT3P(spec, !=, NULL);
 360
 361         spec->efs_match_flags |=
 362                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 363                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 364         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 365         spec->efs_ip_proto = proto;
 366         spec->efs_loc_host.eo_u32[0] = host;
 367         spec->efs_loc_port = port;
 368         return (0);
 369 }
 370
 371 /*
 372  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 373  */
 374 __checkReturn           efx_rc_t
 375 efx_filter_spec_set_ipv4_full(
 376         __inout         efx_filter_spec_t *spec,
 377         __in            uint8_t proto,
 378         __in            uint32_t lhost,
 379         __in            uint16_t lport,
 380         __in            uint32_t rhost,
 381         __in            uint16_t rport)
 382 {
 383         EFSYS_ASSERT3P(spec, !=, NULL);
 384
 385         spec->efs_match_flags |=
 386                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 387                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 388                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 389         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 390         spec->efs_ip_proto = proto;
 391         spec->efs_loc_host.eo_u32[0] = lhost;
 392         spec->efs_loc_port = lport;
 393         spec->efs_rem_host.eo_u32[0] = rhost;
 394         spec->efs_rem_port = rport;
 395         return (0);
 396 }
 397
 398 /*
 399  * Specify local Ethernet address and/or VID in filter specification
 400  */
 401 __checkReturn           efx_rc_t
 402 efx_filter_spec_set_eth_local(
 403         __inout         efx_filter_spec_t *spec,
 404         __in            uint16_t vid,
 405         __in            const uint8_t *addr)
 406 {
 407         EFSYS_ASSERT3P(spec, !=, NULL);
 408         EFSYS_ASSERT3P(addr, !=, NULL);
 409
 410         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 411                 return (EINVAL);
 412
 413         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 414                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 415                 spec->efs_outer_vid = vid;
 416         }
 417         if (addr != NULL) {
 418                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 419                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 420         }
 421         return (0);
 422 }
 423
 424                         void
 425 efx_filter_spec_set_ether_type(
 426         __inout         efx_filter_spec_t *spec,
 427         __in            uint16_t ether_type)
 428 {
 429         EFSYS_ASSERT3P(spec, !=, NULL);
 430
 431         spec->efs_ether_type = ether_type;
 432         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 433 }
 434
 435 /*
 436  * Specify matching otherwise-unmatched unicast in a filter specification
 437  */
 438 __checkReturn           efx_rc_t
 439 efx_filter_spec_set_uc_def(
 440         __inout         efx_filter_spec_t *spec)
 441 {
 442         EFSYS_ASSERT3P(spec, !=, NULL);
 443
 444         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 445         return (0);
 446 }
 447
 448 /*
 449  * Specify matching otherwise-unmatched multicast in a filter specification
 450  */
 451 __checkReturn           efx_rc_t
 452 efx_filter_spec_set_mc_def(
 453         __inout         efx_filter_spec_t *spec)
 454 {
 455         EFSYS_ASSERT3P(spec, !=, NULL);
 456
 457         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 458         return (0);
 459 }
 460
 461
 462 __checkReturn           efx_rc_t
 463 efx_filter_spec_set_encap_type(
 464         __inout         efx_filter_spec_t *spec,
 465         __in            efx_tunnel_protocol_t encap_type,
 466         __in            efx_filter_inner_frame_match_t inner_frame_match)
 467 {
 468         uint32_t match_flags = EFX_FILTER_MATCH_ENCAP_TYPE;
 469         uint8_t ip_proto;
 470         efx_rc_t rc;
 471
 472         EFSYS_ASSERT3P(spec, !=, NULL);
 473
 474         switch (encap_type) {
 475         case EFX_TUNNEL_PROTOCOL_VXLAN:
 476         case EFX_TUNNEL_PROTOCOL_GENEVE:
 477                 ip_proto = EFX_IPPROTO_UDP;
 478                 break;
 479         case EFX_TUNNEL_PROTOCOL_NVGRE:
 480                 ip_proto = EFX_IPPROTO_GRE;
 481                 break;
 482         default:
 483                 EFSYS_ASSERT(0);
 484                 rc = EINVAL;
 485                 goto fail1;
 486         }
 487
 488         switch (inner_frame_match) {
 489         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 490                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 491                 break;
 492         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 493                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 494                 break;
 495         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 496                 /* This is for when specific inner frames are to be matched. */
 497                 break;
 498         default:
 499                 EFSYS_ASSERT(0);
 500                 rc = EINVAL;
 501                 goto fail2;
 502         }
 503
 504         spec->efs_encap_type = encap_type;
 505         spec->efs_ip_proto = ip_proto;
 506         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 507
 508         return (0);
 509
 510 fail2:
 511         EFSYS_PROBE(fail2);
 512 fail1:
 513         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 514
 515         return (rc);
 516 }
 517
 518 /*
 519  * Specify inner and outer Ethernet address and VNI or VSID in tunnel filter
 520  * specification.
 521  */
 522 static  __checkReturn   efx_rc_t
 523 efx_filter_spec_set_tunnel(
 524         __inout efx_filter_spec_t *spec,
 525         __in            efx_tunnel_protocol_t encap_type,
 526         __in            const uint8_t *vni_or_vsid,
 527         __in            const uint8_t *inner_addr,
 528         __in            const uint8_t *outer_addr)
 529 {
 530         efx_rc_t rc;
 531
 532         EFSYS_ASSERT3P(spec, !=, NULL);
 533         EFSYS_ASSERT3P(vni_or_vsid, !=, NULL);
 534         EFSYS_ASSERT3P(inner_addr, !=, NULL);
 535         EFSYS_ASSERT3P(outer_addr, !=, NULL);
 536
 537         switch (encap_type) {
 538         case EFX_TUNNEL_PROTOCOL_VXLAN:
 539         case EFX_TUNNEL_PROTOCOL_GENEVE:
 540         case EFX_TUNNEL_PROTOCOL_NVGRE:
 541                 break;
 542         default:
 543                 rc = EINVAL;
 544                 goto fail1;
 545         }
 546
 547         if ((inner_addr == NULL) && (outer_addr == NULL)) {
 548                 rc = EINVAL;
 549                 goto fail2;
 550         }
 551
 552         if (vni_or_vsid != NULL) {
 553                 spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 554                 memcpy(spec->efs_vni_or_vsid, vni_or_vsid, EFX_VNI_OR_VSID_LEN);
 555         }
 556         if (outer_addr != NULL) {
 557                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 558                 memcpy(spec->efs_loc_mac, outer_addr, EFX_MAC_ADDR_LEN);
 559         }
 560         if (inner_addr != NULL) {
 561                 spec->efs_match_flags |= EFX_FILTER_MATCH_IFRM_LOC_MAC;
 562                 memcpy(spec->efs_ifrm_loc_mac, inner_addr, EFX_MAC_ADDR_LEN);
 563         }
 564
 565         spec->efs_match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE;
 566         spec->efs_encap_type = encap_type;
 567
 568         return (0);
 569
 570 fail2:
 571         EFSYS_PROBE(fail2);
 572 fail1:
 573         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 574
 575         return (rc);
 576 }
 577
 578 /*
 579  * Specify inner and outer Ethernet address and VNI in VXLAN filter
 580  * specification.
 581  */
 582 __checkReturn           efx_rc_t
 583 efx_filter_spec_set_vxlan(
 584         __inout         efx_filter_spec_t *spec,
 585         __in            const uint8_t *vni,
 586         __in            const uint8_t *inner_addr,
 587         __in            const uint8_t *outer_addr)
 588 {
 589         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_VXLAN,
 590             vni, inner_addr, outer_addr);
 591 }
 592
 593 /*
 594  * Specify inner and outer Ethernet address and VNI in Geneve filter
 595  * specification.
 596  */
 597 __checkReturn           efx_rc_t
 598 efx_filter_spec_set_geneve(
 599         __inout         efx_filter_spec_t *spec,
 600         __in            const uint8_t *vni,
 601         __in            const uint8_t *inner_addr,
 602         __in            const uint8_t *outer_addr)
 603 {
 604         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_GENEVE,
 605             vni, inner_addr, outer_addr);
 606 }
 607
 608 /*
 609  * Specify inner and outer Ethernet address and vsid in NVGRE filter
 610  * specification.
 611  */
 612 __checkReturn           efx_rc_t
 613 efx_filter_spec_set_nvgre(
 614         __inout         efx_filter_spec_t *spec,
 615         __in            const uint8_t *vsid,
 616         __in            const uint8_t *inner_addr,
 617         __in            const uint8_t *outer_addr)
 618 {
 619         return efx_filter_spec_set_tunnel(spec, EFX_TUNNEL_PROTOCOL_NVGRE,
 620             vsid, inner_addr, outer_addr);
 621 }
 622
 623 #if EFSYS_OPT_RX_SCALE
 624         __checkReturn   efx_rc_t
 625 efx_filter_spec_set_rss_context(
 626         __inout         efx_filter_spec_t *spec,
 627         __in            uint32_t rss_context)
 628 {
 629         efx_rc_t rc;
 630
 631         EFSYS_ASSERT3P(spec, !=, NULL);
 632
 633         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 634         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 635                 rc = EINVAL;
 636                 goto fail1;
 637         }
 638
 639         spec->efs_rss_context = rss_context;
 640
 641         return (0);
 642
 643 fail1:
 644         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 645
 646         return (rc);
 647 }
 648 #endif
 649
 650 #if EFSYS_OPT_SIENA
 651
 652 /*
 653  * "Fudge factors" - difference between programmed value and actual depth.
 654  * Due to pipelined implementation we need to program H/W with a value that
 655  * is larger than the hop limit we want.
 656  */
 657 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 658 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 659
 660 /*
 661  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 662  * We also need to avoid infinite loops in efx_filter_search() when the
 663  * table is full.
 664  */
 665 #define FILTER_CTL_SRCH_MAX 200
 666
 667 static  __checkReturn   efx_rc_t
 668 siena_filter_spec_from_gen_spec(
 669         __out           siena_filter_spec_t *sf_spec,
 670         __in            efx_filter_spec_t *gen_spec)
 671 {
 672         efx_rc_t rc;
 673         boolean_t is_full = B_FALSE;
 674
 675         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 676                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 677         else
 678                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 679
 680         /* Siena only has one RSS context */
 681         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 682             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 683                 rc = EINVAL;
 684                 goto fail1;
 685         }
 686
 687         sf_spec->sfs_flags = gen_spec->efs_flags;
 688         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 689
 690         switch (gen_spec->efs_match_flags) {
 691         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 692             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 693             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 694                 is_full = B_TRUE;
 695                 /* Fall through */
 696         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 697             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 698                 uint32_t rhost, host1, host2;
 699                 uint16_t rport, port1, port2;
 700
 701                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 702                         rc = ENOTSUP;
 703                         goto fail2;
 704                 }
 705                 if (gen_spec->efs_loc_port == 0 ||
 706                     (is_full && gen_spec->efs_rem_port == 0)) {
 707                         rc = EINVAL;
 708                         goto fail3;
 709                 }
 710                 switch (gen_spec->efs_ip_proto) {
 711                 case EFX_IPPROTO_TCP:
 712                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 713                                 sf_spec->sfs_type = (is_full ?
 714                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 715                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 716                         } else {
 717                                 sf_spec->sfs_type = (is_full ?
 718                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 719                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 720                         }
 721                         break;
 722                 case EFX_IPPROTO_UDP:
 723                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 724                                 sf_spec->sfs_type = (is_full ?
 725                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 726                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 727                         } else {
 728                                 sf_spec->sfs_type = (is_full ?
 729                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 730                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 731                         }
 732                         break;
 733                 default:
 734                         rc = ENOTSUP;
 735                         goto fail4;
 736                 }
 737                 /*
 738                  * The filter is constructed in terms of source and destination,
 739                  * with the odd wrinkle that the ports are swapped in a UDP
 740                  * wildcard filter. We need to convert from local and remote
 741                  * addresses (zero for a wildcard).
 742                  */
 743                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 744                 rport = is_full ? gen_spec->efs_rem_port : 0;
 745                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 746                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 747                         host2 = rhost;
 748                 } else {
 749                         host1 = rhost;
 750                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 751                 }
 752                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 753                         if (sf_spec->sfs_type ==
 754                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 755                                 port1 = rport;
 756                                 port2 = gen_spec->efs_loc_port;
 757                         } else {
 758                                 port1 = gen_spec->efs_loc_port;
 759                                 port2 = rport;
 760                         }
 761                 } else {
 762                         if (sf_spec->sfs_type ==
 763                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 764                                 port1 = gen_spec->efs_loc_port;
 765                                 port2 = rport;
 766                         } else {
 767                                 port1 = rport;
 768                                 port2 = gen_spec->efs_loc_port;
 769                         }
 770                 }
 771                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 772                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 773                 sf_spec->sfs_dword[2] = host2;
 774                 break;
 775         }
 776
 777         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 778                 is_full = B_TRUE;
 779                 /* Fall through */
 780         case EFX_FILTER_MATCH_LOC_MAC:
 781                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 782                         sf_spec->sfs_type = (is_full ?
 783                             EFX_SIENA_FILTER_TX_MAC_FULL :
 784                             EFX_SIENA_FILTER_TX_MAC_WILD);
 785                 } else {
 786                         sf_spec->sfs_type = (is_full ?
 787                             EFX_SIENA_FILTER_RX_MAC_FULL :
 788                             EFX_SIENA_FILTER_RX_MAC_WILD);
 789                 }
 790                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 791                 sf_spec->sfs_dword[1] =
 792                     gen_spec->efs_loc_mac[2] << 24 |
 793                     gen_spec->efs_loc_mac[3] << 16 |
 794                     gen_spec->efs_loc_mac[4] <<  8 |
 795                     gen_spec->efs_loc_mac[5];
 796                 sf_spec->sfs_dword[2] =
 797                     gen_spec->efs_loc_mac[0] << 8 |
 798                     gen_spec->efs_loc_mac[1];
 799                 break;
 800
 801         default:
 802                 EFSYS_ASSERT(B_FALSE);
 803                 rc = ENOTSUP;
 804                 goto fail5;
 805         }
 806
 807         return (0);
 808
 809 fail5:
 810         EFSYS_PROBE(fail5);
 811 fail4:
 812         EFSYS_PROBE(fail4);
 813 fail3:
 814         EFSYS_PROBE(fail3);
 815 fail2:
 816         EFSYS_PROBE(fail2);
 817 fail1:
 818         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 819
 820         return (rc);
 821 }
 822
 823 /*
 824  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 825  * key derived from the n-tuple.
 826  */
 827 static                  uint16_t
 828 siena_filter_tbl_hash(
 829         __in            uint32_t key)
 830 {
 831         uint16_t tmp;
 832
 833         /* First 16 rounds */
 834         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 835         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 836         tmp = tmp ^ tmp >> 9;
 837
 838         /* Last 16 rounds */
 839         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 840         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 841         tmp = tmp ^ tmp >> 9;
 842
 843         return (tmp);
 844 }
 845
 846 /*
 847  * To allow for hash collisions, filter search continues at these
 848  * increments from the first possible entry selected by the hash.
 849  */
 850 static                  uint16_t
 851 siena_filter_tbl_increment(
 852         __in            uint32_t key)
 853 {
 854         return ((uint16_t)(key * 2 - 1));
 855 }
 856
 857 static  __checkReturn   boolean_t
 858 siena_filter_test_used(
 859         __in            siena_filter_tbl_t *sftp,
 860         __in            unsigned int index)
 861 {
 862         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 863         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 864 }
 865
 866 static                  void
 867 siena_filter_set_used(
 868         __in            siena_filter_tbl_t *sftp,
 869         __in            unsigned int index)
 870 {
 871         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 872         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 873         ++sftp->sft_used;
 874 }
 875
 876 static                  void
 877 siena_filter_clear_used(
 878         __in            siena_filter_tbl_t *sftp,
 879         __in            unsigned int index)
 880 {
 881         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 882         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 883
 884         --sftp->sft_used;
 885         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 886 }
 887
 888
 889 static                  siena_filter_tbl_id_t
 890 siena_filter_tbl_id(
 891         __in            siena_filter_type_t type)
 892 {
 893         siena_filter_tbl_id_t tbl_id;
 894
 895         switch (type) {
 896         case EFX_SIENA_FILTER_RX_TCP_FULL:
 897         case EFX_SIENA_FILTER_RX_TCP_WILD:
 898         case EFX_SIENA_FILTER_RX_UDP_FULL:
 899         case EFX_SIENA_FILTER_RX_UDP_WILD:
 900                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 901                 break;
 902
 903         case EFX_SIENA_FILTER_RX_MAC_FULL:
 904         case EFX_SIENA_FILTER_RX_MAC_WILD:
 905                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 906                 break;
 907
 908         case EFX_SIENA_FILTER_TX_TCP_FULL:
 909         case EFX_SIENA_FILTER_TX_TCP_WILD:
 910         case EFX_SIENA_FILTER_TX_UDP_FULL:
 911         case EFX_SIENA_FILTER_TX_UDP_WILD:
 912                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 913                 break;
 914
 915         case EFX_SIENA_FILTER_TX_MAC_FULL:
 916         case EFX_SIENA_FILTER_TX_MAC_WILD:
 917                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 918                 break;
 919
 920         default:
 921                 EFSYS_ASSERT(B_FALSE);
 922                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 923                 break;
 924         }
 925         return (tbl_id);
 926 }
 927
 928 static                  void
 929 siena_filter_reset_search_depth(
 930         __inout         siena_filter_t *sfp,
 931         __in            siena_filter_tbl_id_t tbl_id)
 932 {
 933         switch (tbl_id) {
 934         case EFX_SIENA_FILTER_TBL_RX_IP:
 935                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 936                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 937                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 938                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 939                 break;
 940
 941         case EFX_SIENA_FILTER_TBL_RX_MAC:
 942                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 943                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 944                 break;
 945
 946         case EFX_SIENA_FILTER_TBL_TX_IP:
 947                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 948                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 949                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 950                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 951                 break;
 952
 953         case EFX_SIENA_FILTER_TBL_TX_MAC:
 954                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 955                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 956                 break;
 957
 958         default:
 959                 EFSYS_ASSERT(B_FALSE);
 960                 break;
 961         }
 962 }
 963
 964 static                  void
 965 siena_filter_push_rx_limits(
 966         __in            efx_nic_t *enp)
 967 {
 968         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 969         efx_oword_t oword;
 970
 971         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 972
 973         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 974             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 975             FILTER_CTL_SRCH_FUDGE_FULL);
 976         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 977             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 978             FILTER_CTL_SRCH_FUDGE_WILD);
 979         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 980             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 981             FILTER_CTL_SRCH_FUDGE_FULL);
 982         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 983             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 984             FILTER_CTL_SRCH_FUDGE_WILD);
 985
 986         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 987                 EFX_SET_OWORD_FIELD(oword,
 988                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 989                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 990                     FILTER_CTL_SRCH_FUDGE_FULL);
 991                 EFX_SET_OWORD_FIELD(oword,
 992                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 993                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 994                     FILTER_CTL_SRCH_FUDGE_WILD);
 995         }
 996
 997         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 998 }
 999
1000 static                  void
1001 siena_filter_push_tx_limits(
1002         __in            efx_nic_t *enp)
1003 {
1004         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1005         efx_oword_t oword;
1006
1007         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
1008
1009         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
1010                 EFX_SET_OWORD_FIELD(oword,
1011                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
1012                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
1013                     FILTER_CTL_SRCH_FUDGE_FULL);
1014                 EFX_SET_OWORD_FIELD(oword,
1015                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
1016                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
1017                     FILTER_CTL_SRCH_FUDGE_WILD);
1018                 EFX_SET_OWORD_FIELD(oword,
1019                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
1020                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
1021                     FILTER_CTL_SRCH_FUDGE_FULL);
1022                 EFX_SET_OWORD_FIELD(oword,
1023                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
1024                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
1025                     FILTER_CTL_SRCH_FUDGE_WILD);
1026         }
1027
1028         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
1029                 EFX_SET_OWORD_FIELD(
1030                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
1031                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
1032                         FILTER_CTL_SRCH_FUDGE_FULL);
1033                 EFX_SET_OWORD_FIELD(
1034                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
1035                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
1036                         FILTER_CTL_SRCH_FUDGE_WILD);
1037         }
1038
1039         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
1040 }
1041
1042 /* Build a filter entry and return its n-tuple key. */
1043 static  __checkReturn   uint32_t
1044 siena_filter_build(
1045         __out           efx_oword_t *filter,
1046         __in            siena_filter_spec_t *spec)
1047 {
1048         uint32_t dword3;
1049         uint32_t key;
1050         uint8_t  type  = spec->sfs_type;
1051         uint32_t flags = spec->sfs_flags;
1052
1053         switch (siena_filter_tbl_id(type)) {
1054         case EFX_SIENA_FILTER_TBL_RX_IP: {
1055                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
1056                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
1057                 EFX_POPULATE_OWORD_7(*filter,
1058                     FRF_BZ_RSS_EN,
1059                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1060                     FRF_BZ_SCATTER_EN,
1061                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1062                     FRF_AZ_TCP_UDP, is_udp,
1063                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
1064                     EFX_DWORD_2, spec->sfs_dword[2],
1065                     EFX_DWORD_1, spec->sfs_dword[1],
1066                     EFX_DWORD_0, spec->sfs_dword[0]);
1067                 dword3 = is_udp;
1068                 break;
1069         }
1070
1071         case EFX_SIENA_FILTER_TBL_RX_MAC: {
1072                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
1073                 EFX_POPULATE_OWORD_7(*filter,
1074                     FRF_CZ_RMFT_RSS_EN,
1075                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
1076                     FRF_CZ_RMFT_SCATTER_EN,
1077                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
1078                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
1079                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
1080                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
1081                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
1082                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
1083                 dword3 = is_wild;
1084                 break;
1085         }
1086
1087         case EFX_SIENA_FILTER_TBL_TX_IP: {
1088                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
1089                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
1090                 EFX_POPULATE_OWORD_5(*filter,
1091                     FRF_CZ_TIFT_TCP_UDP, is_udp,
1092                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
1093                     EFX_DWORD_2, spec->sfs_dword[2],
1094                     EFX_DWORD_1, spec->sfs_dword[1],
1095                     EFX_DWORD_0, spec->sfs_dword[0]);
1096                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
1097                 break;
1098         }
1099
1100         case EFX_SIENA_FILTER_TBL_TX_MAC: {
1101                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
1102                 EFX_POPULATE_OWORD_5(*filter,
1103                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
1104                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
1105                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
1106                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
1107                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
1108                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
1109                 break;
1110         }
1111
1112         default:
1113                 EFSYS_ASSERT(B_FALSE);
1114                 EFX_ZERO_OWORD(*filter);
1115                 return (0);
1116         }
1117
1118         key =
1119             spec->sfs_dword[0] ^
1120             spec->sfs_dword[1] ^
1121             spec->sfs_dword[2] ^
1122             dword3;
1123
1124         return (key);
1125 }
1126
1127 static  __checkReturn           efx_rc_t
1128 siena_filter_push_entry(
1129         __inout                 efx_nic_t *enp,
1130         __in                    siena_filter_type_t type,
1131         __in                    int index,
1132         __in                    efx_oword_t *eop)
1133 {
1134         efx_rc_t rc;
1135
1136         switch (type) {
1137         case EFX_SIENA_FILTER_RX_TCP_FULL:
1138         case EFX_SIENA_FILTER_RX_TCP_WILD:
1139         case EFX_SIENA_FILTER_RX_UDP_FULL:
1140         case EFX_SIENA_FILTER_RX_UDP_WILD:
1141                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1142                     eop, B_TRUE);
1143                 break;
1144
1145         case EFX_SIENA_FILTER_RX_MAC_FULL:
1146         case EFX_SIENA_FILTER_RX_MAC_WILD:
1147                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1148                     eop, B_TRUE);
1149                 break;
1150
1151         case EFX_SIENA_FILTER_TX_TCP_FULL:
1152         case EFX_SIENA_FILTER_TX_TCP_WILD:
1153         case EFX_SIENA_FILTER_TX_UDP_FULL:
1154         case EFX_SIENA_FILTER_TX_UDP_WILD:
1155                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1156                     eop, B_TRUE);
1157                 break;
1158
1159         case EFX_SIENA_FILTER_TX_MAC_FULL:
1160         case EFX_SIENA_FILTER_TX_MAC_WILD:
1161                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1162                     eop, B_TRUE);
1163                 break;
1164
1165         default:
1166                 EFSYS_ASSERT(B_FALSE);
1167                 rc = ENOTSUP;
1168                 goto fail1;
1169         }
1170         return (0);
1171
1172 fail1:
1173         return (rc);
1174 }
1175
1176
1177 static  __checkReturn   boolean_t
1178 siena_filter_equal(
1179         __in            const siena_filter_spec_t *left,
1180         __in            const siena_filter_spec_t *right)
1181 {
1182         siena_filter_tbl_id_t tbl_id;
1183
1184         tbl_id = siena_filter_tbl_id(left->sfs_type);
1185
1186
1187         if (left->sfs_type != right->sfs_type)
1188                 return (B_FALSE);
1189
1190         if (memcmp(left->sfs_dword, right->sfs_dword,
1191                 sizeof (left->sfs_dword)))
1192                 return (B_FALSE);
1193
1194         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1195                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1196             left->sfs_dmaq_id != right->sfs_dmaq_id)
1197                 return (B_FALSE);
1198
1199         return (B_TRUE);
1200 }
1201
1202 static  __checkReturn   efx_rc_t
1203 siena_filter_search(
1204         __in            siena_filter_tbl_t *sftp,
1205         __in            siena_filter_spec_t *spec,
1206         __in            uint32_t key,
1207         __in            boolean_t for_insert,
1208         __out           int *filter_index,
1209         __out           unsigned int *depth_required)
1210 {
1211         unsigned int hash, incr, filter_idx, depth;
1212
1213         hash = siena_filter_tbl_hash(key);
1214         incr = siena_filter_tbl_increment(key);
1215
1216         filter_idx = hash & (sftp->sft_size - 1);
1217         depth = 1;
1218
1219         for (;;) {
1220                 /*
1221                  * Return success if entry is used and matches this spec
1222                  * or entry is unused and we are trying to insert.
1223                  */
1224                 if (siena_filter_test_used(sftp, filter_idx) ?
1225                     siena_filter_equal(spec,
1226                     &sftp->sft_spec[filter_idx]) :
1227                     for_insert) {
1228                         *filter_index = filter_idx;
1229                         *depth_required = depth;
1230                         return (0);
1231                 }
1232
1233                 /* Return failure if we reached the maximum search depth */
1234                 if (depth == FILTER_CTL_SRCH_MAX)
1235                         return (for_insert ? EBUSY : ENOENT);
1236
1237                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1238                 ++depth;
1239         }
1240 }
1241
1242 static                  void
1243 siena_filter_clear_entry(
1244         __in            efx_nic_t *enp,
1245         __in            siena_filter_tbl_t *sftp,
1246         __in            int index)
1247 {
1248         efx_oword_t filter;
1249
1250         if (siena_filter_test_used(sftp, index)) {
1251                 siena_filter_clear_used(sftp, index);
1252
1253                 EFX_ZERO_OWORD(filter);
1254                 siena_filter_push_entry(enp,
1255                     sftp->sft_spec[index].sfs_type,
1256                     index, &filter);
1257
1258                 memset(&sftp->sft_spec[index],
1259                     0, sizeof (sftp->sft_spec[0]));
1260         }
1261 }
1262
1263                         void
1264 siena_filter_tbl_clear(
1265         __in            efx_nic_t *enp,
1266         __in            siena_filter_tbl_id_t tbl_id)
1267 {
1268         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1269         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1270         int index;
1271         efsys_lock_state_t state;
1272
1273         EFSYS_LOCK(enp->en_eslp, state);
1274
1275         for (index = 0; index < sftp->sft_size; ++index) {
1276                 siena_filter_clear_entry(enp, sftp, index);
1277         }
1278
1279         if (sftp->sft_used == 0)
1280                 siena_filter_reset_search_depth(sfp, tbl_id);
1281
1282         EFSYS_UNLOCK(enp->en_eslp, state);
1283 }
1284
1285 static  __checkReturn   efx_rc_t
1286 siena_filter_init(
1287         __in            efx_nic_t *enp)
1288 {
1289         siena_filter_t *sfp;
1290         siena_filter_tbl_t *sftp;
1291         int tbl_id;
1292         efx_rc_t rc;
1293
1294         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1295
1296         if (!sfp) {
1297                 rc = ENOMEM;
1298                 goto fail1;
1299         }
1300
1301         enp->en_filter.ef_siena_filter = sfp;
1302
1303         switch (enp->en_family) {
1304         case EFX_FAMILY_SIENA:
1305                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1306                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1307
1308                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1309                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1310
1311                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1312                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1313
1314                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1315                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1316                 break;
1317
1318         default:
1319                 rc = ENOTSUP;
1320                 goto fail2;
1321         }
1322
1323         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1324                 unsigned int bitmap_size;
1325
1326                 sftp = &sfp->sf_tbl[tbl_id];
1327                 if (sftp->sft_size == 0)
1328                         continue;
1329
1330                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1331                     sizeof (uint32_t));
1332                 bitmap_size =
1333                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1334
1335                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1336                 if (!sftp->sft_bitmap) {
1337                         rc = ENOMEM;
1338                         goto fail3;
1339                 }
1340
1341                 EFSYS_KMEM_ALLOC(enp->en_esip,
1342                     sftp->sft_size * sizeof (*sftp->sft_spec),
1343                     sftp->sft_spec);
1344                 if (!sftp->sft_spec) {
1345                         rc = ENOMEM;
1346                         goto fail4;
1347                 }
1348                 memset(sftp->sft_spec, 0,
1349                     sftp->sft_size * sizeof (*sftp->sft_spec));
1350         }
1351
1352         return (0);
1353
1354 fail4:
1355         EFSYS_PROBE(fail4);
1356
1357 fail3:
1358         EFSYS_PROBE(fail3);
1359
1360 fail2:
1361         EFSYS_PROBE(fail2);
1362         siena_filter_fini(enp);
1363
1364 fail1:
1365         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1366         return (rc);
1367 }
1368
1369 static                  void
1370 siena_filter_fini(
1371         __in            efx_nic_t *enp)
1372 {
1373         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1374         siena_filter_tbl_id_t tbl_id;
1375
1376         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1377         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1378
1379         if (sfp == NULL)
1380                 return;
1381
1382         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1383                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1384                 unsigned int bitmap_size;
1385
1386                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1387                     sizeof (uint32_t));
1388                 bitmap_size =
1389                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1390
1391                 if (sftp->sft_bitmap != NULL) {
1392                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1393                             sftp->sft_bitmap);
1394                         sftp->sft_bitmap = NULL;
1395                 }
1396
1397                 if (sftp->sft_spec != NULL) {
1398                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1399                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1400                         sftp->sft_spec = NULL;
1401                 }
1402         }
1403
1404         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1405             enp->en_filter.ef_siena_filter);
1406 }
1407
1408 /* Restore filter state after a reset */
1409 static  __checkReturn   efx_rc_t
1410 siena_filter_restore(
1411         __in            efx_nic_t *enp)
1412 {
1413         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1414         siena_filter_tbl_id_t tbl_id;
1415         siena_filter_tbl_t *sftp;
1416         siena_filter_spec_t *spec;
1417         efx_oword_t filter;
1418         int filter_idx;
1419         efsys_lock_state_t state;
1420         uint32_t key;
1421         efx_rc_t rc;
1422
1423         EFSYS_LOCK(enp->en_eslp, state);
1424
1425         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1426                 sftp = &sfp->sf_tbl[tbl_id];
1427                 for (filter_idx = 0;
1428                         filter_idx < sftp->sft_size;
1429                         filter_idx++) {
1430                         if (!siena_filter_test_used(sftp, filter_idx))
1431                                 continue;
1432
1433                         spec = &sftp->sft_spec[filter_idx];
1434                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1435                                 rc = EINVAL;
1436                                 goto fail1;
1437                         }
1438                         if ((rc = siena_filter_push_entry(enp,
1439                                     spec->sfs_type, filter_idx, &filter)) != 0)
1440                                 goto fail2;
1441                 }
1442         }
1443
1444         siena_filter_push_rx_limits(enp);
1445         siena_filter_push_tx_limits(enp);
1446
1447         EFSYS_UNLOCK(enp->en_eslp, state);
1448
1449         return (0);
1450
1451 fail2:
1452         EFSYS_PROBE(fail2);
1453
1454 fail1:
1455         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1456
1457         EFSYS_UNLOCK(enp->en_eslp, state);
1458
1459         return (rc);
1460 }
1461
1462 static   __checkReturn  efx_rc_t
1463 siena_filter_add(
1464         __in            efx_nic_t *enp,
1465         __inout         efx_filter_spec_t *spec,
1466         __in            efx_filter_replacement_policy_t policy)
1467 {
1468         efx_rc_t rc;
1469         siena_filter_spec_t sf_spec;
1470         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1471         siena_filter_tbl_id_t tbl_id;
1472         siena_filter_tbl_t *sftp;
1473         siena_filter_spec_t *saved_sf_spec;
1474         efx_oword_t filter;
1475         int filter_idx;
1476         unsigned int depth;
1477         efsys_lock_state_t state;
1478         uint32_t key;
1479
1480
1481         EFSYS_ASSERT3P(spec, !=, NULL);
1482
1483         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1484                 goto fail1;
1485
1486         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1487         sftp = &sfp->sf_tbl[tbl_id];
1488
1489         if (sftp->sft_size == 0) {
1490                 rc = EINVAL;
1491                 goto fail2;
1492         }
1493
1494         key = siena_filter_build(&filter, &sf_spec);
1495
1496         EFSYS_LOCK(enp->en_eslp, state);
1497
1498         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1499             &filter_idx, &depth);
1500         if (rc != 0)
1501                 goto fail3;
1502
1503         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1504         saved_sf_spec = &sftp->sft_spec[filter_idx];
1505
1506         if (siena_filter_test_used(sftp, filter_idx)) {
1507                 /* All Siena filter are considered the same priority */
1508                 switch (policy) {
1509                 case EFX_FILTER_REPLACEMENT_NEVER:
1510                 case EFX_FILTER_REPLACEMENT_HIGHER_PRIORITY:
1511                         rc = EEXIST;
1512                         goto fail4;
1513                 case EFX_FILTER_REPLACEMENT_HIGHER_OR_EQUAL_PRIORITY:
1514                         break;
1515                 default:
1516                         EFSYS_ASSERT(0);
1517                         break;
1518                 }
1519         }
1520         siena_filter_set_used(sftp, filter_idx);
1521         *saved_sf_spec = sf_spec;
1522
1523         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1524                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1525                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1526                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1527                         siena_filter_push_tx_limits(enp);
1528                 else
1529                         siena_filter_push_rx_limits(enp);
1530         }
1531
1532         siena_filter_push_entry(enp, sf_spec.sfs_type,
1533             filter_idx, &filter);
1534
1535         EFSYS_UNLOCK(enp->en_eslp, state);
1536         return (0);
1537
1538 fail4:
1539         EFSYS_PROBE(fail4);
1540
1541 fail3:
1542         EFSYS_UNLOCK(enp->en_eslp, state);
1543         EFSYS_PROBE(fail3);
1544
1545 fail2:
1546         EFSYS_PROBE(fail2);
1547
1548 fail1:
1549         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1550         return (rc);
1551 }
1552
1553 static   __checkReturn  efx_rc_t
1554 siena_filter_delete(
1555         __in            efx_nic_t *enp,
1556         __inout         efx_filter_spec_t *spec)
1557 {
1558         efx_rc_t rc;
1559         siena_filter_spec_t sf_spec;
1560         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1561         siena_filter_tbl_id_t tbl_id;
1562         siena_filter_tbl_t *sftp;
1563         efx_oword_t filter;
1564         int filter_idx;
1565         unsigned int depth;
1566         efsys_lock_state_t state;
1567         uint32_t key;
1568
1569         EFSYS_ASSERT3P(spec, !=, NULL);
1570
1571         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1572                 goto fail1;
1573
1574         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1575         sftp = &sfp->sf_tbl[tbl_id];
1576
1577         key = siena_filter_build(&filter, &sf_spec);
1578
1579         EFSYS_LOCK(enp->en_eslp, state);
1580
1581         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1582             &filter_idx, &depth);
1583         if (rc != 0)
1584                 goto fail2;
1585
1586         siena_filter_clear_entry(enp, sftp, filter_idx);
1587         if (sftp->sft_used == 0)
1588                 siena_filter_reset_search_depth(sfp, tbl_id);
1589
1590         EFSYS_UNLOCK(enp->en_eslp, state);
1591         return (0);
1592
1593 fail2:
1594         EFSYS_UNLOCK(enp->en_eslp, state);
1595         EFSYS_PROBE(fail2);
1596
1597 fail1:
1598         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1599         return (rc);
1600 }
1601
1602 #define SIENA_MAX_SUPPORTED_MATCHES 4
1603
1604 static  __checkReturn   efx_rc_t
1605 siena_filter_supported_filters(
1606         __in                            efx_nic_t *enp,
1607         __out_ecount(buffer_length)     uint32_t *buffer,
1608         __in                            size_t buffer_length,
1609         __out                           size_t *list_lengthp)
1610 {
1611         uint32_t index = 0;
1612         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1613         size_t list_length;
1614         efx_rc_t rc;
1615
1616         rx_matches[index++] =
1617             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1618             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1619             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1620
1621         rx_matches[index++] =
1622             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1623             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1624
1625         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1626                 rx_matches[index++] =
1627                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1628
1629                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1630         }
1631
1632         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1633         list_length = index;
1634
1635         *list_lengthp = list_length;
1636
1637         if (buffer_length < list_length) {
1638                 rc = ENOSPC;
1639                 goto fail1;
1640         }
1641
1642         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1643
1644         return (0);
1645
1646 fail1:
1647         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1648
1649         return (rc);
1650 }
1651
1652 #undef MAX_SUPPORTED
1653
1654 #endif /* EFSYS_OPT_SIENA */
1655
1656 #endif /* EFSYS_OPT_FILTER */