drivers/net/sfc/base/efx_filter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2007-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10
  11 #if EFSYS_OPT_FILTER
  12
  13 #if EFSYS_OPT_SIENA
  14
  15 static  __checkReturn   efx_rc_t
  16 siena_filter_init(
  17         __in            efx_nic_t *enp);
  18
  19 static                  void
  20 siena_filter_fini(
  21         __in            efx_nic_t *enp);
  22
  23 static  __checkReturn   efx_rc_t
  24 siena_filter_restore(
  25         __in            efx_nic_t *enp);
  26
  27 static  __checkReturn   efx_rc_t
  28 siena_filter_add(
  29         __in            efx_nic_t *enp,
  30         __inout         efx_filter_spec_t *spec,
  31         __in            boolean_t may_replace);
  32
  33 static  __checkReturn   efx_rc_t
  34 siena_filter_delete(
  35         __in            efx_nic_t *enp,
  36         __inout         efx_filter_spec_t *spec);
  37
  38 static  __checkReturn   efx_rc_t
  39 siena_filter_supported_filters(
  40         __in                            efx_nic_t *enp,
  41         __out_ecount(buffer_length)     uint32_t *buffer,
  42         __in                            size_t buffer_length,
  43         __out                           size_t *list_lengthp);
  44
  45 #endif /* EFSYS_OPT_SIENA */
  46
  47 #if EFSYS_OPT_SIENA
  48 static const efx_filter_ops_t   __efx_filter_siena_ops = {
  49         siena_filter_init,              /* efo_init */
  50         siena_filter_fini,              /* efo_fini */
  51         siena_filter_restore,           /* efo_restore */
  52         siena_filter_add,               /* efo_add */
  53         siena_filter_delete,            /* efo_delete */
  54         siena_filter_supported_filters, /* efo_supported_filters */
  55         NULL,                           /* efo_reconfigure */
  56 };
  57 #endif /* EFSYS_OPT_SIENA */
  58
  59 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2
  60 static const efx_filter_ops_t   __efx_filter_ef10_ops = {
  61         ef10_filter_init,               /* efo_init */
  62         ef10_filter_fini,               /* efo_fini */
  63         ef10_filter_restore,            /* efo_restore */
  64         ef10_filter_add,                /* efo_add */
  65         ef10_filter_delete,             /* efo_delete */
  66         ef10_filter_supported_filters,  /* efo_supported_filters */
  67         ef10_filter_reconfigure,        /* efo_reconfigure */
  68 };
  69 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD || EFSYS_OPT_MEDFORD2 */
  70
  71         __checkReturn   efx_rc_t
  72 efx_filter_insert(
  73         __in            efx_nic_t *enp,
  74         __inout         efx_filter_spec_t *spec)
  75 {
  76         const efx_filter_ops_t *efop = enp->en_efop;
  77
  78         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
  79         EFSYS_ASSERT3P(spec, !=, NULL);
  80         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
  81
  82         return (efop->efo_add(enp, spec, B_FALSE));
  83 }
  84
  85         __checkReturn   efx_rc_t
  86 efx_filter_remove(
  87         __in            efx_nic_t *enp,
  88         __inout         efx_filter_spec_t *spec)
  89 {
  90         const efx_filter_ops_t *efop = enp->en_efop;
  91
  92         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
  93         EFSYS_ASSERT3P(spec, !=, NULL);
  94         EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX);
  95
  96         return (efop->efo_delete(enp, spec));
  97 }
  98
  99         __checkReturn   efx_rc_t
 100 efx_filter_restore(
 101         __in            efx_nic_t *enp)
 102 {
 103         efx_rc_t rc;
 104
 105         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 106
 107         if ((rc = enp->en_efop->efo_restore(enp)) != 0)
 108                 goto fail1;
 109
 110         return (0);
 111
 112 fail1:
 113         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 114
 115         return (rc);
 116 }
 117
 118         __checkReturn   efx_rc_t
 119 efx_filter_init(
 120         __in            efx_nic_t *enp)
 121 {
 122         const efx_filter_ops_t *efop;
 123         efx_rc_t rc;
 124
 125         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 126         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 127         EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_FILTER));
 128
 129         switch (enp->en_family) {
 130 #if EFSYS_OPT_SIENA
 131         case EFX_FAMILY_SIENA:
 132                 efop = &__efx_filter_siena_ops;
 133                 break;
 134 #endif /* EFSYS_OPT_SIENA */
 135
 136 #if EFSYS_OPT_HUNTINGTON
 137         case EFX_FAMILY_HUNTINGTON:
 138                 efop = &__efx_filter_ef10_ops;
 139                 break;
 140 #endif /* EFSYS_OPT_HUNTINGTON */
 141
 142 #if EFSYS_OPT_MEDFORD
 143         case EFX_FAMILY_MEDFORD:
 144                 efop = &__efx_filter_ef10_ops;
 145                 break;
 146 #endif /* EFSYS_OPT_MEDFORD */
 147
 148 #if EFSYS_OPT_MEDFORD2
 149         case EFX_FAMILY_MEDFORD2:
 150                 efop = &__efx_filter_ef10_ops;
 151                 break;
 152 #endif /* EFSYS_OPT_MEDFORD2 */
 153
 154         default:
 155                 EFSYS_ASSERT(0);
 156                 rc = ENOTSUP;
 157                 goto fail1;
 158         }
 159
 160         if ((rc = efop->efo_init(enp)) != 0)
 161                 goto fail2;
 162
 163         enp->en_efop = efop;
 164         enp->en_mod_flags |= EFX_MOD_FILTER;
 165         return (0);
 166
 167 fail2:
 168         EFSYS_PROBE(fail2);
 169 fail1:
 170         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 171
 172         enp->en_efop = NULL;
 173         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 174         return (rc);
 175 }
 176
 177                         void
 178 efx_filter_fini(
 179         __in            efx_nic_t *enp)
 180 {
 181         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 182         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 183         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 184
 185         enp->en_efop->efo_fini(enp);
 186
 187         enp->en_efop = NULL;
 188         enp->en_mod_flags &= ~EFX_MOD_FILTER;
 189 }
 190
 191 /*
 192  * Query the possible combinations of match flags which can be filtered on.
 193  * These are returned as a list, of which each 32 bit element is a bitmask
 194  * formed of EFX_FILTER_MATCH flags.
 195  *
 196  * The combinations are ordered in priority from highest to lowest.
 197  *
 198  * If the provided buffer is too short to hold the list, the call with fail with
 199  * ENOSPC and *list_lengthp will be set to the buffer length required.
 200  */
 201         __checkReturn   efx_rc_t
 202 efx_filter_supported_filters(
 203         __in                            efx_nic_t *enp,
 204         __out_ecount(buffer_length)     uint32_t *buffer,
 205         __in                            size_t buffer_length,
 206         __out                           size_t *list_lengthp)
 207 {
 208         efx_rc_t rc;
 209
 210         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 211         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 212         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 213         EFSYS_ASSERT(enp->en_efop->efo_supported_filters != NULL);
 214
 215         if (buffer == NULL) {
 216                 rc = EINVAL;
 217                 goto fail1;
 218         }
 219
 220         rc = enp->en_efop->efo_supported_filters(enp, buffer, buffer_length,
 221                                                     list_lengthp);
 222         if (rc != 0)
 223                 goto fail2;
 224
 225         return (0);
 226
 227 fail2:
 228         EFSYS_PROBE(fail2);
 229 fail1:
 230         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 231
 232         return (rc);
 233 }
 234
 235         __checkReturn   efx_rc_t
 236 efx_filter_reconfigure(
 237         __in                            efx_nic_t *enp,
 238         __in_ecount(6)                  uint8_t const *mac_addr,
 239         __in                            boolean_t all_unicst,
 240         __in                            boolean_t mulcst,
 241         __in                            boolean_t all_mulcst,
 242         __in                            boolean_t brdcst,
 243         __in_ecount(6*count)            uint8_t const *addrs,
 244         __in                            uint32_t count)
 245 {
 246         efx_rc_t rc;
 247
 248         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
 249         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
 250         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_FILTER);
 251
 252         if (enp->en_efop->efo_reconfigure != NULL) {
 253                 if ((rc = enp->en_efop->efo_reconfigure(enp, mac_addr,
 254                                                         all_unicst, mulcst,
 255                                                         all_mulcst, brdcst,
 256                                                         addrs, count)) != 0)
 257                         goto fail1;
 258         }
 259
 260         return (0);
 261
 262 fail1:
 263         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 264
 265         return (rc);
 266 }
 267
 268                 void
 269 efx_filter_spec_init_rx(
 270         __out           efx_filter_spec_t *spec,
 271         __in            efx_filter_priority_t priority,
 272         __in            efx_filter_flags_t flags,
 273         __in            efx_rxq_t *erp)
 274 {
 275         EFSYS_ASSERT3P(spec, !=, NULL);
 276         EFSYS_ASSERT3P(erp, !=, NULL);
 277         EFSYS_ASSERT((flags & ~(EFX_FILTER_FLAG_RX_RSS |
 278                                 EFX_FILTER_FLAG_RX_SCATTER)) == 0);
 279
 280         memset(spec, 0, sizeof (*spec));
 281         spec->efs_priority = priority;
 282         spec->efs_flags = EFX_FILTER_FLAG_RX | flags;
 283         spec->efs_rss_context = EFX_RSS_CONTEXT_DEFAULT;
 284         spec->efs_dmaq_id = (uint16_t)erp->er_index;
 285 }
 286
 287                 void
 288 efx_filter_spec_init_tx(
 289         __out           efx_filter_spec_t *spec,
 290         __in            efx_txq_t *etp)
 291 {
 292         EFSYS_ASSERT3P(spec, !=, NULL);
 293         EFSYS_ASSERT3P(etp, !=, NULL);
 294
 295         memset(spec, 0, sizeof (*spec));
 296         spec->efs_priority = EFX_FILTER_PRI_REQUIRED;
 297         spec->efs_flags = EFX_FILTER_FLAG_TX;
 298         spec->efs_dmaq_id = (uint16_t)etp->et_index;
 299 }
 300
 301
 302 /*
 303  *  Specify IPv4 host, transport protocol and port in a filter specification
 304  */
 305 __checkReturn           efx_rc_t
 306 efx_filter_spec_set_ipv4_local(
 307         __inout         efx_filter_spec_t *spec,
 308         __in            uint8_t proto,
 309         __in            uint32_t host,
 310         __in            uint16_t port)
 311 {
 312         EFSYS_ASSERT3P(spec, !=, NULL);
 313
 314         spec->efs_match_flags |=
 315                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 316                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
 317         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 318         spec->efs_ip_proto = proto;
 319         spec->efs_loc_host.eo_u32[0] = host;
 320         spec->efs_loc_port = port;
 321         return (0);
 322 }
 323
 324 /*
 325  * Specify IPv4 hosts, transport protocol and ports in a filter specification
 326  */
 327 __checkReturn           efx_rc_t
 328 efx_filter_spec_set_ipv4_full(
 329         __inout         efx_filter_spec_t *spec,
 330         __in            uint8_t proto,
 331         __in            uint32_t lhost,
 332         __in            uint16_t lport,
 333         __in            uint32_t rhost,
 334         __in            uint16_t rport)
 335 {
 336         EFSYS_ASSERT3P(spec, !=, NULL);
 337
 338         spec->efs_match_flags |=
 339                 EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 340                 EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 341                 EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
 342         spec->efs_ether_type = EFX_ETHER_TYPE_IPV4;
 343         spec->efs_ip_proto = proto;
 344         spec->efs_loc_host.eo_u32[0] = lhost;
 345         spec->efs_loc_port = lport;
 346         spec->efs_rem_host.eo_u32[0] = rhost;
 347         spec->efs_rem_port = rport;
 348         return (0);
 349 }
 350
 351 /*
 352  * Specify local Ethernet address and/or VID in filter specification
 353  */
 354 __checkReturn           efx_rc_t
 355 efx_filter_spec_set_eth_local(
 356         __inout         efx_filter_spec_t *spec,
 357         __in            uint16_t vid,
 358         __in            const uint8_t *addr)
 359 {
 360         EFSYS_ASSERT3P(spec, !=, NULL);
 361         EFSYS_ASSERT3P(addr, !=, NULL);
 362
 363         if (vid == EFX_FILTER_SPEC_VID_UNSPEC && addr == NULL)
 364                 return (EINVAL);
 365
 366         if (vid != EFX_FILTER_SPEC_VID_UNSPEC) {
 367                 spec->efs_match_flags |= EFX_FILTER_MATCH_OUTER_VID;
 368                 spec->efs_outer_vid = vid;
 369         }
 370         if (addr != NULL) {
 371                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 372                 memcpy(spec->efs_loc_mac, addr, EFX_MAC_ADDR_LEN);
 373         }
 374         return (0);
 375 }
 376
 377                         void
 378 efx_filter_spec_set_ether_type(
 379         __inout         efx_filter_spec_t *spec,
 380         __in            uint16_t ether_type)
 381 {
 382         EFSYS_ASSERT3P(spec, !=, NULL);
 383
 384         spec->efs_ether_type = ether_type;
 385         spec->efs_match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
 386 }
 387
 388 /*
 389  * Specify matching otherwise-unmatched unicast in a filter specification
 390  */
 391 __checkReturn           efx_rc_t
 392 efx_filter_spec_set_uc_def(
 393         __inout         efx_filter_spec_t *spec)
 394 {
 395         EFSYS_ASSERT3P(spec, !=, NULL);
 396
 397         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_UCAST_DST;
 398         return (0);
 399 }
 400
 401 /*
 402  * Specify matching otherwise-unmatched multicast in a filter specification
 403  */
 404 __checkReturn           efx_rc_t
 405 efx_filter_spec_set_mc_def(
 406         __inout         efx_filter_spec_t *spec)
 407 {
 408         EFSYS_ASSERT3P(spec, !=, NULL);
 409
 410         spec->efs_match_flags |= EFX_FILTER_MATCH_UNKNOWN_MCAST_DST;
 411         return (0);
 412 }
 413
 414
 415 __checkReturn           efx_rc_t
 416 efx_filter_spec_set_encap_type(
 417         __inout         efx_filter_spec_t *spec,
 418         __in            efx_tunnel_protocol_t encap_type,
 419         __in            efx_filter_inner_frame_match_t inner_frame_match)
 420 {
 421         uint32_t match_flags = 0;
 422         uint8_t ip_proto;
 423         efx_rc_t rc;
 424
 425         EFSYS_ASSERT3P(spec, !=, NULL);
 426
 427         switch (encap_type) {
 428         case EFX_TUNNEL_PROTOCOL_VXLAN:
 429         case EFX_TUNNEL_PROTOCOL_GENEVE:
 430                 ip_proto = EFX_IPPROTO_UDP;
 431                 break;
 432         case EFX_TUNNEL_PROTOCOL_NVGRE:
 433                 ip_proto = EFX_IPPROTO_GRE;
 434                 break;
 435         default:
 436                 EFSYS_ASSERT(0);
 437                 rc = EINVAL;
 438                 goto fail1;
 439         }
 440
 441         switch (inner_frame_match) {
 442         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST:
 443                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST;
 444                 break;
 445         case EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_UCAST_DST:
 446                 match_flags |= EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST;
 447                 break;
 448         case EFX_FILTER_INNER_FRAME_MATCH_OTHER:
 449                 /* This is for when specific inner frames are to be matched. */
 450                 break;
 451         default:
 452                 EFSYS_ASSERT(0);
 453                 rc = EINVAL;
 454                 goto fail2;
 455         }
 456
 457         spec->efs_encap_type = encap_type;
 458         spec->efs_ip_proto = ip_proto;
 459         spec->efs_match_flags |= (match_flags | EFX_FILTER_MATCH_IP_PROTO);
 460
 461         return (0);
 462
 463 fail2:
 464         EFSYS_PROBE(fail2);
 465 fail1:
 466         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 467
 468         return (rc);
 469 }
 470
 471 /*
 472  * Specify inner and outer Ethernet address and VXLAN ID in filter
 473  * specification.
 474  */
 475         __checkReturn   efx_rc_t
 476 efx_filter_spec_set_vxlan_full(
 477         __inout         efx_filter_spec_t *spec,
 478         __in            const uint8_t *vxlan_id,
 479         __in            const uint8_t *inner_addr,
 480         __in            const uint8_t *outer_addr)
 481 {
 482         EFSYS_ASSERT3P(spec, !=, NULL);
 483         EFSYS_ASSERT3P(vxlan_id, !=, NULL);
 484         EFSYS_ASSERT3P(inner_addr, !=, NULL);
 485         EFSYS_ASSERT3P(outer_addr, !=, NULL);
 486
 487         if ((inner_addr == NULL) && (outer_addr == NULL))
 488                 return (EINVAL);
 489
 490         if (vxlan_id != NULL) {
 491                 spec->efs_match_flags |= EFX_FILTER_MATCH_VNI_OR_VSID;
 492                 memcpy(spec->efs_vni_or_vsid, vxlan_id, EFX_VNI_OR_VSID_LEN);
 493         }
 494         if (outer_addr != NULL) {
 495                 spec->efs_match_flags |= EFX_FILTER_MATCH_LOC_MAC;
 496                 memcpy(spec->efs_loc_mac, outer_addr, EFX_MAC_ADDR_LEN);
 497         }
 498         if (inner_addr != NULL) {
 499                 spec->efs_match_flags |= EFX_FILTER_MATCH_IFRM_LOC_MAC;
 500                 memcpy(spec->efs_ifrm_loc_mac, inner_addr, EFX_MAC_ADDR_LEN);
 501         }
 502         spec->efs_encap_type = EFX_TUNNEL_PROTOCOL_VXLAN;
 503
 504         return (0);
 505 }
 506
 507 #if EFSYS_OPT_RX_SCALE
 508         __checkReturn   efx_rc_t
 509 efx_filter_spec_set_rss_context(
 510         __inout         efx_filter_spec_t *spec,
 511         __in            uint32_t rss_context)
 512 {
 513         efx_rc_t rc;
 514
 515         EFSYS_ASSERT3P(spec, !=, NULL);
 516
 517         /* The filter must have been created with EFX_FILTER_FLAG_RX_RSS. */
 518         if ((spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) == 0) {
 519                 rc = EINVAL;
 520                 goto fail1;
 521         }
 522
 523         spec->efs_rss_context = rss_context;
 524
 525         return (0);
 526
 527 fail1:
 528         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 529
 530         return (rc);
 531 }
 532 #endif
 533
 534 #if EFSYS_OPT_SIENA
 535
 536 /*
 537  * "Fudge factors" - difference between programmed value and actual depth.
 538  * Due to pipelined implementation we need to program H/W with a value that
 539  * is larger than the hop limit we want.
 540  */
 541 #define FILTER_CTL_SRCH_FUDGE_WILD 3
 542 #define FILTER_CTL_SRCH_FUDGE_FULL 1
 543
 544 /*
 545  * Hard maximum hop limit.  Hardware will time-out beyond 200-something.
 546  * We also need to avoid infinite loops in efx_filter_search() when the
 547  * table is full.
 548  */
 549 #define FILTER_CTL_SRCH_MAX 200
 550
 551 static  __checkReturn   efx_rc_t
 552 siena_filter_spec_from_gen_spec(
 553         __out           siena_filter_spec_t *sf_spec,
 554         __in            efx_filter_spec_t *gen_spec)
 555 {
 556         efx_rc_t rc;
 557         boolean_t is_full = B_FALSE;
 558
 559         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX)
 560                 EFSYS_ASSERT3U(gen_spec->efs_flags, ==, EFX_FILTER_FLAG_TX);
 561         else
 562                 EFSYS_ASSERT3U(gen_spec->efs_flags, &, EFX_FILTER_FLAG_RX);
 563
 564         /* Siena only has one RSS context */
 565         if ((gen_spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 566             gen_spec->efs_rss_context != EFX_RSS_CONTEXT_DEFAULT) {
 567                 rc = EINVAL;
 568                 goto fail1;
 569         }
 570
 571         sf_spec->sfs_flags = gen_spec->efs_flags;
 572         sf_spec->sfs_dmaq_id = gen_spec->efs_dmaq_id;
 573
 574         switch (gen_spec->efs_match_flags) {
 575         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 576             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
 577             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT:
 578                 is_full = B_TRUE;
 579                 /* Fall through */
 580         case EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
 581             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT: {
 582                 uint32_t rhost, host1, host2;
 583                 uint16_t rport, port1, port2;
 584
 585                 if (gen_spec->efs_ether_type != EFX_ETHER_TYPE_IPV4) {
 586                         rc = ENOTSUP;
 587                         goto fail2;
 588                 }
 589                 if (gen_spec->efs_loc_port == 0 ||
 590                     (is_full && gen_spec->efs_rem_port == 0)) {
 591                         rc = EINVAL;
 592                         goto fail3;
 593                 }
 594                 switch (gen_spec->efs_ip_proto) {
 595                 case EFX_IPPROTO_TCP:
 596                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 597                                 sf_spec->sfs_type = (is_full ?
 598                                     EFX_SIENA_FILTER_TX_TCP_FULL :
 599                                     EFX_SIENA_FILTER_TX_TCP_WILD);
 600                         } else {
 601                                 sf_spec->sfs_type = (is_full ?
 602                                     EFX_SIENA_FILTER_RX_TCP_FULL :
 603                                     EFX_SIENA_FILTER_RX_TCP_WILD);
 604                         }
 605                         break;
 606                 case EFX_IPPROTO_UDP:
 607                         if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 608                                 sf_spec->sfs_type = (is_full ?
 609                                     EFX_SIENA_FILTER_TX_UDP_FULL :
 610                                     EFX_SIENA_FILTER_TX_UDP_WILD);
 611                         } else {
 612                                 sf_spec->sfs_type = (is_full ?
 613                                     EFX_SIENA_FILTER_RX_UDP_FULL :
 614                                     EFX_SIENA_FILTER_RX_UDP_WILD);
 615                         }
 616                         break;
 617                 default:
 618                         rc = ENOTSUP;
 619                         goto fail4;
 620                 }
 621                 /*
 622                  * The filter is constructed in terms of source and destination,
 623                  * with the odd wrinkle that the ports are swapped in a UDP
 624                  * wildcard filter. We need to convert from local and remote
 625                  * addresses (zero for a wildcard).
 626                  */
 627                 rhost = is_full ? gen_spec->efs_rem_host.eo_u32[0] : 0;
 628                 rport = is_full ? gen_spec->efs_rem_port : 0;
 629                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 630                         host1 = gen_spec->efs_loc_host.eo_u32[0];
 631                         host2 = rhost;
 632                 } else {
 633                         host1 = rhost;
 634                         host2 = gen_spec->efs_loc_host.eo_u32[0];
 635                 }
 636                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 637                         if (sf_spec->sfs_type ==
 638                             EFX_SIENA_FILTER_TX_UDP_WILD) {
 639                                 port1 = rport;
 640                                 port2 = gen_spec->efs_loc_port;
 641                         } else {
 642                                 port1 = gen_spec->efs_loc_port;
 643                                 port2 = rport;
 644                         }
 645                 } else {
 646                         if (sf_spec->sfs_type ==
 647                             EFX_SIENA_FILTER_RX_UDP_WILD) {
 648                                 port1 = gen_spec->efs_loc_port;
 649                                 port2 = rport;
 650                         } else {
 651                                 port1 = rport;
 652                                 port2 = gen_spec->efs_loc_port;
 653                         }
 654                 }
 655                 sf_spec->sfs_dword[0] = (host1 << 16) | port1;
 656                 sf_spec->sfs_dword[1] = (port2 << 16) | (host1 >> 16);
 657                 sf_spec->sfs_dword[2] = host2;
 658                 break;
 659         }
 660
 661         case EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_OUTER_VID:
 662                 is_full = B_TRUE;
 663                 /* Fall through */
 664         case EFX_FILTER_MATCH_LOC_MAC:
 665                 if (gen_spec->efs_flags & EFX_FILTER_FLAG_TX) {
 666                         sf_spec->sfs_type = (is_full ?
 667                             EFX_SIENA_FILTER_TX_MAC_FULL :
 668                             EFX_SIENA_FILTER_TX_MAC_WILD);
 669                 } else {
 670                         sf_spec->sfs_type = (is_full ?
 671                             EFX_SIENA_FILTER_RX_MAC_FULL :
 672                             EFX_SIENA_FILTER_RX_MAC_WILD);
 673                 }
 674                 sf_spec->sfs_dword[0] = is_full ? gen_spec->efs_outer_vid : 0;
 675                 sf_spec->sfs_dword[1] =
 676                     gen_spec->efs_loc_mac[2] << 24 |
 677                     gen_spec->efs_loc_mac[3] << 16 |
 678                     gen_spec->efs_loc_mac[4] <<  8 |
 679                     gen_spec->efs_loc_mac[5];
 680                 sf_spec->sfs_dword[2] =
 681                     gen_spec->efs_loc_mac[0] << 8 |
 682                     gen_spec->efs_loc_mac[1];
 683                 break;
 684
 685         default:
 686                 EFSYS_ASSERT(B_FALSE);
 687                 rc = ENOTSUP;
 688                 goto fail5;
 689         }
 690
 691         return (0);
 692
 693 fail5:
 694         EFSYS_PROBE(fail5);
 695 fail4:
 696         EFSYS_PROBE(fail4);
 697 fail3:
 698         EFSYS_PROBE(fail3);
 699 fail2:
 700         EFSYS_PROBE(fail2);
 701 fail1:
 702         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 703
 704         return (rc);
 705 }
 706
 707 /*
 708  * The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
 709  * key derived from the n-tuple.
 710  */
 711 static                  uint16_t
 712 siena_filter_tbl_hash(
 713         __in            uint32_t key)
 714 {
 715         uint16_t tmp;
 716
 717         /* First 16 rounds */
 718         tmp = 0x1fff ^ (uint16_t)(key >> 16);
 719         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 720         tmp = tmp ^ tmp >> 9;
 721
 722         /* Last 16 rounds */
 723         tmp = tmp ^ tmp << 13 ^ (uint16_t)(key & 0xffff);
 724         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
 725         tmp = tmp ^ tmp >> 9;
 726
 727         return (tmp);
 728 }
 729
 730 /*
 731  * To allow for hash collisions, filter search continues at these
 732  * increments from the first possible entry selected by the hash.
 733  */
 734 static                  uint16_t
 735 siena_filter_tbl_increment(
 736         __in            uint32_t key)
 737 {
 738         return ((uint16_t)(key * 2 - 1));
 739 }
 740
 741 static  __checkReturn   boolean_t
 742 siena_filter_test_used(
 743         __in            siena_filter_tbl_t *sftp,
 744         __in            unsigned int index)
 745 {
 746         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 747         return ((sftp->sft_bitmap[index / 32] & (1 << (index % 32))) != 0);
 748 }
 749
 750 static                  void
 751 siena_filter_set_used(
 752         __in            siena_filter_tbl_t *sftp,
 753         __in            unsigned int index)
 754 {
 755         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 756         sftp->sft_bitmap[index / 32] |= (1 << (index % 32));
 757         ++sftp->sft_used;
 758 }
 759
 760 static                  void
 761 siena_filter_clear_used(
 762         __in            siena_filter_tbl_t *sftp,
 763         __in            unsigned int index)
 764 {
 765         EFSYS_ASSERT3P(sftp->sft_bitmap, !=, NULL);
 766         sftp->sft_bitmap[index / 32] &= ~(1 << (index % 32));
 767
 768         --sftp->sft_used;
 769         EFSYS_ASSERT3U(sftp->sft_used, >=, 0);
 770 }
 771
 772
 773 static                  siena_filter_tbl_id_t
 774 siena_filter_tbl_id(
 775         __in            siena_filter_type_t type)
 776 {
 777         siena_filter_tbl_id_t tbl_id;
 778
 779         switch (type) {
 780         case EFX_SIENA_FILTER_RX_TCP_FULL:
 781         case EFX_SIENA_FILTER_RX_TCP_WILD:
 782         case EFX_SIENA_FILTER_RX_UDP_FULL:
 783         case EFX_SIENA_FILTER_RX_UDP_WILD:
 784                 tbl_id = EFX_SIENA_FILTER_TBL_RX_IP;
 785                 break;
 786
 787         case EFX_SIENA_FILTER_RX_MAC_FULL:
 788         case EFX_SIENA_FILTER_RX_MAC_WILD:
 789                 tbl_id = EFX_SIENA_FILTER_TBL_RX_MAC;
 790                 break;
 791
 792         case EFX_SIENA_FILTER_TX_TCP_FULL:
 793         case EFX_SIENA_FILTER_TX_TCP_WILD:
 794         case EFX_SIENA_FILTER_TX_UDP_FULL:
 795         case EFX_SIENA_FILTER_TX_UDP_WILD:
 796                 tbl_id = EFX_SIENA_FILTER_TBL_TX_IP;
 797                 break;
 798
 799         case EFX_SIENA_FILTER_TX_MAC_FULL:
 800         case EFX_SIENA_FILTER_TX_MAC_WILD:
 801                 tbl_id = EFX_SIENA_FILTER_TBL_TX_MAC;
 802                 break;
 803
 804         default:
 805                 EFSYS_ASSERT(B_FALSE);
 806                 tbl_id = EFX_SIENA_FILTER_NTBLS;
 807                 break;
 808         }
 809         return (tbl_id);
 810 }
 811
 812 static                  void
 813 siena_filter_reset_search_depth(
 814         __inout         siena_filter_t *sfp,
 815         __in            siena_filter_tbl_id_t tbl_id)
 816 {
 817         switch (tbl_id) {
 818         case EFX_SIENA_FILTER_TBL_RX_IP:
 819                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] = 0;
 820                 sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] = 0;
 821                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] = 0;
 822                 sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] = 0;
 823                 break;
 824
 825         case EFX_SIENA_FILTER_TBL_RX_MAC:
 826                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] = 0;
 827                 sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] = 0;
 828                 break;
 829
 830         case EFX_SIENA_FILTER_TBL_TX_IP:
 831                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] = 0;
 832                 sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] = 0;
 833                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] = 0;
 834                 sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] = 0;
 835                 break;
 836
 837         case EFX_SIENA_FILTER_TBL_TX_MAC:
 838                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] = 0;
 839                 sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] = 0;
 840                 break;
 841
 842         default:
 843                 EFSYS_ASSERT(B_FALSE);
 844                 break;
 845         }
 846 }
 847
 848 static                  void
 849 siena_filter_push_rx_limits(
 850         __in            efx_nic_t *enp)
 851 {
 852         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 853         efx_oword_t oword;
 854
 855         EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 856
 857         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_FULL_SRCH_LIMIT,
 858             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_FULL] +
 859             FILTER_CTL_SRCH_FUDGE_FULL);
 860         EFX_SET_OWORD_FIELD(oword, FRF_AZ_TCP_WILD_SRCH_LIMIT,
 861             sfp->sf_depth[EFX_SIENA_FILTER_RX_TCP_WILD] +
 862             FILTER_CTL_SRCH_FUDGE_WILD);
 863         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_FULL_SRCH_LIMIT,
 864             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_FULL] +
 865             FILTER_CTL_SRCH_FUDGE_FULL);
 866         EFX_SET_OWORD_FIELD(oword, FRF_AZ_UDP_WILD_SRCH_LIMIT,
 867             sfp->sf_depth[EFX_SIENA_FILTER_RX_UDP_WILD] +
 868             FILTER_CTL_SRCH_FUDGE_WILD);
 869
 870         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC].sft_size) {
 871                 EFX_SET_OWORD_FIELD(oword,
 872                     FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 873                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_FULL] +
 874                     FILTER_CTL_SRCH_FUDGE_FULL);
 875                 EFX_SET_OWORD_FIELD(oword,
 876                     FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 877                     sfp->sf_depth[EFX_SIENA_FILTER_RX_MAC_WILD] +
 878                     FILTER_CTL_SRCH_FUDGE_WILD);
 879         }
 880
 881         EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
 882 }
 883
 884 static                  void
 885 siena_filter_push_tx_limits(
 886         __in            efx_nic_t *enp)
 887 {
 888         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
 889         efx_oword_t oword;
 890
 891         EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
 892
 893         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP].sft_size != 0) {
 894                 EFX_SET_OWORD_FIELD(oword,
 895                     FRF_CZ_TX_TCPIP_FILTER_FULL_SEARCH_RANGE,
 896                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_FULL] +
 897                     FILTER_CTL_SRCH_FUDGE_FULL);
 898                 EFX_SET_OWORD_FIELD(oword,
 899                     FRF_CZ_TX_TCPIP_FILTER_WILD_SEARCH_RANGE,
 900                     sfp->sf_depth[EFX_SIENA_FILTER_TX_TCP_WILD] +
 901                     FILTER_CTL_SRCH_FUDGE_WILD);
 902                 EFX_SET_OWORD_FIELD(oword,
 903                     FRF_CZ_TX_UDPIP_FILTER_FULL_SEARCH_RANGE,
 904                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_FULL] +
 905                     FILTER_CTL_SRCH_FUDGE_FULL);
 906                 EFX_SET_OWORD_FIELD(oword,
 907                     FRF_CZ_TX_UDPIP_FILTER_WILD_SEARCH_RANGE,
 908                     sfp->sf_depth[EFX_SIENA_FILTER_TX_UDP_WILD] +
 909                     FILTER_CTL_SRCH_FUDGE_WILD);
 910         }
 911
 912         if (sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC].sft_size != 0) {
 913                 EFX_SET_OWORD_FIELD(
 914                         oword, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 915                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_FULL] +
 916                         FILTER_CTL_SRCH_FUDGE_FULL);
 917                 EFX_SET_OWORD_FIELD(
 918                         oword, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 919                         sfp->sf_depth[EFX_SIENA_FILTER_TX_MAC_WILD] +
 920                         FILTER_CTL_SRCH_FUDGE_WILD);
 921         }
 922
 923         EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
 924 }
 925
 926 /* Build a filter entry and return its n-tuple key. */
 927 static  __checkReturn   uint32_t
 928 siena_filter_build(
 929         __out           efx_oword_t *filter,
 930         __in            siena_filter_spec_t *spec)
 931 {
 932         uint32_t dword3;
 933         uint32_t key;
 934         uint8_t  type  = spec->sfs_type;
 935         uint32_t flags = spec->sfs_flags;
 936
 937         switch (siena_filter_tbl_id(type)) {
 938         case EFX_SIENA_FILTER_TBL_RX_IP: {
 939                 boolean_t is_udp = (type == EFX_SIENA_FILTER_RX_UDP_FULL ||
 940                     type == EFX_SIENA_FILTER_RX_UDP_WILD);
 941                 EFX_POPULATE_OWORD_7(*filter,
 942                     FRF_BZ_RSS_EN,
 943                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 944                     FRF_BZ_SCATTER_EN,
 945                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 946                     FRF_AZ_TCP_UDP, is_udp,
 947                     FRF_AZ_RXQ_ID, spec->sfs_dmaq_id,
 948                     EFX_DWORD_2, spec->sfs_dword[2],
 949                     EFX_DWORD_1, spec->sfs_dword[1],
 950                     EFX_DWORD_0, spec->sfs_dword[0]);
 951                 dword3 = is_udp;
 952                 break;
 953         }
 954
 955         case EFX_SIENA_FILTER_TBL_RX_MAC: {
 956                 boolean_t is_wild = (type == EFX_SIENA_FILTER_RX_MAC_WILD);
 957                 EFX_POPULATE_OWORD_7(*filter,
 958                     FRF_CZ_RMFT_RSS_EN,
 959                     (flags & EFX_FILTER_FLAG_RX_RSS) ? 1 : 0,
 960                     FRF_CZ_RMFT_SCATTER_EN,
 961                     (flags & EFX_FILTER_FLAG_RX_SCATTER) ? 1 : 0,
 962                     FRF_CZ_RMFT_RXQ_ID, spec->sfs_dmaq_id,
 963                     FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 964                     FRF_CZ_RMFT_DEST_MAC_DW1, spec->sfs_dword[2],
 965                     FRF_CZ_RMFT_DEST_MAC_DW0, spec->sfs_dword[1],
 966                     FRF_CZ_RMFT_VLAN_ID, spec->sfs_dword[0]);
 967                 dword3 = is_wild;
 968                 break;
 969         }
 970
 971         case EFX_SIENA_FILTER_TBL_TX_IP: {
 972                 boolean_t is_udp = (type == EFX_SIENA_FILTER_TX_UDP_FULL ||
 973                     type == EFX_SIENA_FILTER_TX_UDP_WILD);
 974                 EFX_POPULATE_OWORD_5(*filter,
 975                     FRF_CZ_TIFT_TCP_UDP, is_udp,
 976                     FRF_CZ_TIFT_TXQ_ID, spec->sfs_dmaq_id,
 977                     EFX_DWORD_2, spec->sfs_dword[2],
 978                     EFX_DWORD_1, spec->sfs_dword[1],
 979                     EFX_DWORD_0, spec->sfs_dword[0]);
 980                 dword3 = is_udp | spec->sfs_dmaq_id << 1;
 981                 break;
 982         }
 983
 984         case EFX_SIENA_FILTER_TBL_TX_MAC: {
 985                 boolean_t is_wild = (type == EFX_SIENA_FILTER_TX_MAC_WILD);
 986                 EFX_POPULATE_OWORD_5(*filter,
 987                     FRF_CZ_TMFT_TXQ_ID, spec->sfs_dmaq_id,
 988                     FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 989                     FRF_CZ_TMFT_SRC_MAC_DW1, spec->sfs_dword[2],
 990                     FRF_CZ_TMFT_SRC_MAC_DW0, spec->sfs_dword[1],
 991                     FRF_CZ_TMFT_VLAN_ID, spec->sfs_dword[0]);
 992                 dword3 = is_wild | spec->sfs_dmaq_id << 1;
 993                 break;
 994         }
 995
 996         default:
 997                 EFSYS_ASSERT(B_FALSE);
 998                 EFX_ZERO_OWORD(*filter);
 999                 return (0);
1000         }
1001
1002         key =
1003             spec->sfs_dword[0] ^
1004             spec->sfs_dword[1] ^
1005             spec->sfs_dword[2] ^
1006             dword3;
1007
1008         return (key);
1009 }
1010
1011 static  __checkReturn           efx_rc_t
1012 siena_filter_push_entry(
1013         __inout                 efx_nic_t *enp,
1014         __in                    siena_filter_type_t type,
1015         __in                    int index,
1016         __in                    efx_oword_t *eop)
1017 {
1018         efx_rc_t rc;
1019
1020         switch (type) {
1021         case EFX_SIENA_FILTER_RX_TCP_FULL:
1022         case EFX_SIENA_FILTER_RX_TCP_WILD:
1023         case EFX_SIENA_FILTER_RX_UDP_FULL:
1024         case EFX_SIENA_FILTER_RX_UDP_WILD:
1025                 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_FILTER_TBL0, index,
1026                     eop, B_TRUE);
1027                 break;
1028
1029         case EFX_SIENA_FILTER_RX_MAC_FULL:
1030         case EFX_SIENA_FILTER_RX_MAC_WILD:
1031                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_RX_MAC_FILTER_TBL0, index,
1032                     eop, B_TRUE);
1033                 break;
1034
1035         case EFX_SIENA_FILTER_TX_TCP_FULL:
1036         case EFX_SIENA_FILTER_TX_TCP_WILD:
1037         case EFX_SIENA_FILTER_TX_UDP_FULL:
1038         case EFX_SIENA_FILTER_TX_UDP_WILD:
1039                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_FILTER_TBL0, index,
1040                     eop, B_TRUE);
1041                 break;
1042
1043         case EFX_SIENA_FILTER_TX_MAC_FULL:
1044         case EFX_SIENA_FILTER_TX_MAC_WILD:
1045                 EFX_BAR_TBL_WRITEO(enp, FR_CZ_TX_MAC_FILTER_TBL0, index,
1046                     eop, B_TRUE);
1047                 break;
1048
1049         default:
1050                 EFSYS_ASSERT(B_FALSE);
1051                 rc = ENOTSUP;
1052                 goto fail1;
1053         }
1054         return (0);
1055
1056 fail1:
1057         return (rc);
1058 }
1059
1060
1061 static  __checkReturn   boolean_t
1062 siena_filter_equal(
1063         __in            const siena_filter_spec_t *left,
1064         __in            const siena_filter_spec_t *right)
1065 {
1066         siena_filter_tbl_id_t tbl_id;
1067
1068         tbl_id = siena_filter_tbl_id(left->sfs_type);
1069
1070
1071         if (left->sfs_type != right->sfs_type)
1072                 return (B_FALSE);
1073
1074         if (memcmp(left->sfs_dword, right->sfs_dword,
1075                 sizeof (left->sfs_dword)))
1076                 return (B_FALSE);
1077
1078         if ((tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1079                 tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC) &&
1080             left->sfs_dmaq_id != right->sfs_dmaq_id)
1081                 return (B_FALSE);
1082
1083         return (B_TRUE);
1084 }
1085
1086 static  __checkReturn   efx_rc_t
1087 siena_filter_search(
1088         __in            siena_filter_tbl_t *sftp,
1089         __in            siena_filter_spec_t *spec,
1090         __in            uint32_t key,
1091         __in            boolean_t for_insert,
1092         __out           int *filter_index,
1093         __out           unsigned int *depth_required)
1094 {
1095         unsigned int hash, incr, filter_idx, depth;
1096
1097         hash = siena_filter_tbl_hash(key);
1098         incr = siena_filter_tbl_increment(key);
1099
1100         filter_idx = hash & (sftp->sft_size - 1);
1101         depth = 1;
1102
1103         for (;;) {
1104                 /*
1105                  * Return success if entry is used and matches this spec
1106                  * or entry is unused and we are trying to insert.
1107                  */
1108                 if (siena_filter_test_used(sftp, filter_idx) ?
1109                     siena_filter_equal(spec,
1110                     &sftp->sft_spec[filter_idx]) :
1111                     for_insert) {
1112                         *filter_index = filter_idx;
1113                         *depth_required = depth;
1114                         return (0);
1115                 }
1116
1117                 /* Return failure if we reached the maximum search depth */
1118                 if (depth == FILTER_CTL_SRCH_MAX)
1119                         return (for_insert ? EBUSY : ENOENT);
1120
1121                 filter_idx = (filter_idx + incr) & (sftp->sft_size - 1);
1122                 ++depth;
1123         }
1124 }
1125
1126 static                  void
1127 siena_filter_clear_entry(
1128         __in            efx_nic_t *enp,
1129         __in            siena_filter_tbl_t *sftp,
1130         __in            int index)
1131 {
1132         efx_oword_t filter;
1133
1134         if (siena_filter_test_used(sftp, index)) {
1135                 siena_filter_clear_used(sftp, index);
1136
1137                 EFX_ZERO_OWORD(filter);
1138                 siena_filter_push_entry(enp,
1139                     sftp->sft_spec[index].sfs_type,
1140                     index, &filter);
1141
1142                 memset(&sftp->sft_spec[index],
1143                     0, sizeof (sftp->sft_spec[0]));
1144         }
1145 }
1146
1147                         void
1148 siena_filter_tbl_clear(
1149         __in            efx_nic_t *enp,
1150         __in            siena_filter_tbl_id_t tbl_id)
1151 {
1152         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1153         siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1154         int index;
1155         efsys_lock_state_t state;
1156
1157         EFSYS_LOCK(enp->en_eslp, state);
1158
1159         for (index = 0; index < sftp->sft_size; ++index) {
1160                 siena_filter_clear_entry(enp, sftp, index);
1161         }
1162
1163         if (sftp->sft_used == 0)
1164                 siena_filter_reset_search_depth(sfp, tbl_id);
1165
1166         EFSYS_UNLOCK(enp->en_eslp, state);
1167 }
1168
1169 static  __checkReturn   efx_rc_t
1170 siena_filter_init(
1171         __in            efx_nic_t *enp)
1172 {
1173         siena_filter_t *sfp;
1174         siena_filter_tbl_t *sftp;
1175         int tbl_id;
1176         efx_rc_t rc;
1177
1178         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (siena_filter_t), sfp);
1179
1180         if (!sfp) {
1181                 rc = ENOMEM;
1182                 goto fail1;
1183         }
1184
1185         enp->en_filter.ef_siena_filter = sfp;
1186
1187         switch (enp->en_family) {
1188         case EFX_FAMILY_SIENA:
1189                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_IP];
1190                 sftp->sft_size = FR_AZ_RX_FILTER_TBL0_ROWS;
1191
1192                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_RX_MAC];
1193                 sftp->sft_size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1194
1195                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_IP];
1196                 sftp->sft_size = FR_CZ_TX_FILTER_TBL0_ROWS;
1197
1198                 sftp = &sfp->sf_tbl[EFX_SIENA_FILTER_TBL_TX_MAC];
1199                 sftp->sft_size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1200                 break;
1201
1202         default:
1203                 rc = ENOTSUP;
1204                 goto fail2;
1205         }
1206
1207         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1208                 unsigned int bitmap_size;
1209
1210                 sftp = &sfp->sf_tbl[tbl_id];
1211                 if (sftp->sft_size == 0)
1212                         continue;
1213
1214                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1215                     sizeof (uint32_t));
1216                 bitmap_size =
1217                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1218
1219                 EFSYS_KMEM_ALLOC(enp->en_esip, bitmap_size, sftp->sft_bitmap);
1220                 if (!sftp->sft_bitmap) {
1221                         rc = ENOMEM;
1222                         goto fail3;
1223                 }
1224
1225                 EFSYS_KMEM_ALLOC(enp->en_esip,
1226                     sftp->sft_size * sizeof (*sftp->sft_spec),
1227                     sftp->sft_spec);
1228                 if (!sftp->sft_spec) {
1229                         rc = ENOMEM;
1230                         goto fail4;
1231                 }
1232                 memset(sftp->sft_spec, 0,
1233                     sftp->sft_size * sizeof (*sftp->sft_spec));
1234         }
1235
1236         return (0);
1237
1238 fail4:
1239         EFSYS_PROBE(fail4);
1240
1241 fail3:
1242         EFSYS_PROBE(fail3);
1243
1244 fail2:
1245         EFSYS_PROBE(fail2);
1246         siena_filter_fini(enp);
1247
1248 fail1:
1249         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1250         return (rc);
1251 }
1252
1253 static                  void
1254 siena_filter_fini(
1255         __in            efx_nic_t *enp)
1256 {
1257         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1258         siena_filter_tbl_id_t tbl_id;
1259
1260         EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1261         EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_PROBE);
1262
1263         if (sfp == NULL)
1264                 return;
1265
1266         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1267                 siena_filter_tbl_t *sftp = &sfp->sf_tbl[tbl_id];
1268                 unsigned int bitmap_size;
1269
1270                 EFX_STATIC_ASSERT(sizeof (sftp->sft_bitmap[0]) ==
1271                     sizeof (uint32_t));
1272                 bitmap_size =
1273                     (sftp->sft_size + (sizeof (uint32_t) * 8) - 1) / 8;
1274
1275                 if (sftp->sft_bitmap != NULL) {
1276                         EFSYS_KMEM_FREE(enp->en_esip, bitmap_size,
1277                             sftp->sft_bitmap);
1278                         sftp->sft_bitmap = NULL;
1279                 }
1280
1281                 if (sftp->sft_spec != NULL) {
1282                         EFSYS_KMEM_FREE(enp->en_esip, sftp->sft_size *
1283                             sizeof (*sftp->sft_spec), sftp->sft_spec);
1284                         sftp->sft_spec = NULL;
1285                 }
1286         }
1287
1288         EFSYS_KMEM_FREE(enp->en_esip, sizeof (siena_filter_t),
1289             enp->en_filter.ef_siena_filter);
1290 }
1291
1292 /* Restore filter state after a reset */
1293 static  __checkReturn   efx_rc_t
1294 siena_filter_restore(
1295         __in            efx_nic_t *enp)
1296 {
1297         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1298         siena_filter_tbl_id_t tbl_id;
1299         siena_filter_tbl_t *sftp;
1300         siena_filter_spec_t *spec;
1301         efx_oword_t filter;
1302         int filter_idx;
1303         efsys_lock_state_t state;
1304         uint32_t key;
1305         efx_rc_t rc;
1306
1307         EFSYS_LOCK(enp->en_eslp, state);
1308
1309         for (tbl_id = 0; tbl_id < EFX_SIENA_FILTER_NTBLS; tbl_id++) {
1310                 sftp = &sfp->sf_tbl[tbl_id];
1311                 for (filter_idx = 0;
1312                         filter_idx < sftp->sft_size;
1313                         filter_idx++) {
1314                         if (!siena_filter_test_used(sftp, filter_idx))
1315                                 continue;
1316
1317                         spec = &sftp->sft_spec[filter_idx];
1318                         if ((key = siena_filter_build(&filter, spec)) == 0) {
1319                                 rc = EINVAL;
1320                                 goto fail1;
1321                         }
1322                         if ((rc = siena_filter_push_entry(enp,
1323                                     spec->sfs_type, filter_idx, &filter)) != 0)
1324                                 goto fail2;
1325                 }
1326         }
1327
1328         siena_filter_push_rx_limits(enp);
1329         siena_filter_push_tx_limits(enp);
1330
1331         EFSYS_UNLOCK(enp->en_eslp, state);
1332
1333         return (0);
1334
1335 fail2:
1336         EFSYS_PROBE(fail2);
1337
1338 fail1:
1339         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1340
1341         EFSYS_UNLOCK(enp->en_eslp, state);
1342
1343         return (rc);
1344 }
1345
1346 static   __checkReturn  efx_rc_t
1347 siena_filter_add(
1348         __in            efx_nic_t *enp,
1349         __inout         efx_filter_spec_t *spec,
1350         __in            boolean_t may_replace)
1351 {
1352         efx_rc_t rc;
1353         siena_filter_spec_t sf_spec;
1354         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1355         siena_filter_tbl_id_t tbl_id;
1356         siena_filter_tbl_t *sftp;
1357         siena_filter_spec_t *saved_sf_spec;
1358         efx_oword_t filter;
1359         int filter_idx;
1360         unsigned int depth;
1361         efsys_lock_state_t state;
1362         uint32_t key;
1363
1364
1365         EFSYS_ASSERT3P(spec, !=, NULL);
1366
1367         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1368                 goto fail1;
1369
1370         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1371         sftp = &sfp->sf_tbl[tbl_id];
1372
1373         if (sftp->sft_size == 0) {
1374                 rc = EINVAL;
1375                 goto fail2;
1376         }
1377
1378         key = siena_filter_build(&filter, &sf_spec);
1379
1380         EFSYS_LOCK(enp->en_eslp, state);
1381
1382         rc = siena_filter_search(sftp, &sf_spec, key, B_TRUE,
1383             &filter_idx, &depth);
1384         if (rc != 0)
1385                 goto fail3;
1386
1387         EFSYS_ASSERT3U(filter_idx, <, sftp->sft_size);
1388         saved_sf_spec = &sftp->sft_spec[filter_idx];
1389
1390         if (siena_filter_test_used(sftp, filter_idx)) {
1391                 if (may_replace == B_FALSE) {
1392                         rc = EEXIST;
1393                         goto fail4;
1394                 }
1395         }
1396         siena_filter_set_used(sftp, filter_idx);
1397         *saved_sf_spec = sf_spec;
1398
1399         if (sfp->sf_depth[sf_spec.sfs_type] < depth) {
1400                 sfp->sf_depth[sf_spec.sfs_type] = depth;
1401                 if (tbl_id == EFX_SIENA_FILTER_TBL_TX_IP ||
1402                     tbl_id == EFX_SIENA_FILTER_TBL_TX_MAC)
1403                         siena_filter_push_tx_limits(enp);
1404                 else
1405                         siena_filter_push_rx_limits(enp);
1406         }
1407
1408         siena_filter_push_entry(enp, sf_spec.sfs_type,
1409             filter_idx, &filter);
1410
1411         EFSYS_UNLOCK(enp->en_eslp, state);
1412         return (0);
1413
1414 fail4:
1415         EFSYS_PROBE(fail4);
1416
1417 fail3:
1418         EFSYS_UNLOCK(enp->en_eslp, state);
1419         EFSYS_PROBE(fail3);
1420
1421 fail2:
1422         EFSYS_PROBE(fail2);
1423
1424 fail1:
1425         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1426         return (rc);
1427 }
1428
1429 static   __checkReturn  efx_rc_t
1430 siena_filter_delete(
1431         __in            efx_nic_t *enp,
1432         __inout         efx_filter_spec_t *spec)
1433 {
1434         efx_rc_t rc;
1435         siena_filter_spec_t sf_spec;
1436         siena_filter_t *sfp = enp->en_filter.ef_siena_filter;
1437         siena_filter_tbl_id_t tbl_id;
1438         siena_filter_tbl_t *sftp;
1439         efx_oword_t filter;
1440         int filter_idx;
1441         unsigned int depth;
1442         efsys_lock_state_t state;
1443         uint32_t key;
1444
1445         EFSYS_ASSERT3P(spec, !=, NULL);
1446
1447         if ((rc = siena_filter_spec_from_gen_spec(&sf_spec, spec)) != 0)
1448                 goto fail1;
1449
1450         tbl_id = siena_filter_tbl_id(sf_spec.sfs_type);
1451         sftp = &sfp->sf_tbl[tbl_id];
1452
1453         key = siena_filter_build(&filter, &sf_spec);
1454
1455         EFSYS_LOCK(enp->en_eslp, state);
1456
1457         rc = siena_filter_search(sftp, &sf_spec, key, B_FALSE,
1458             &filter_idx, &depth);
1459         if (rc != 0)
1460                 goto fail2;
1461
1462         siena_filter_clear_entry(enp, sftp, filter_idx);
1463         if (sftp->sft_used == 0)
1464                 siena_filter_reset_search_depth(sfp, tbl_id);
1465
1466         EFSYS_UNLOCK(enp->en_eslp, state);
1467         return (0);
1468
1469 fail2:
1470         EFSYS_UNLOCK(enp->en_eslp, state);
1471         EFSYS_PROBE(fail2);
1472
1473 fail1:
1474         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1475         return (rc);
1476 }
1477
1478 #define SIENA_MAX_SUPPORTED_MATCHES 4
1479
1480 static  __checkReturn   efx_rc_t
1481 siena_filter_supported_filters(
1482         __in                            efx_nic_t *enp,
1483         __out_ecount(buffer_length)     uint32_t *buffer,
1484         __in                            size_t buffer_length,
1485         __out                           size_t *list_lengthp)
1486 {
1487         uint32_t index = 0;
1488         uint32_t rx_matches[SIENA_MAX_SUPPORTED_MATCHES];
1489         size_t list_length;
1490         efx_rc_t rc;
1491
1492         rx_matches[index++] =
1493             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1494             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT |
1495             EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT;
1496
1497         rx_matches[index++] =
1498             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
1499             EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
1500
1501         if (enp->en_features & EFX_FEATURE_MAC_HEADER_FILTERS) {
1502                 rx_matches[index++] =
1503                     EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC;
1504
1505                 rx_matches[index++] = EFX_FILTER_MATCH_LOC_MAC;
1506         }
1507
1508         EFSYS_ASSERT3U(index, <=, SIENA_MAX_SUPPORTED_MATCHES);
1509         list_length = index;
1510
1511         *list_lengthp = list_length;
1512
1513         if (buffer_length < list_length) {
1514                 rc = ENOSPC;
1515                 goto fail1;
1516         }
1517
1518         memcpy(buffer, rx_matches, list_length * sizeof (rx_matches[0]));
1519
1520         return (0);
1521
1522 fail1:
1523         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1524
1525         return (rc);
1526 }
1527
1528 #undef MAX_SUPPORTED
1529
1530 #endif /* EFSYS_OPT_SIENA */
1531
1532 #endif /* EFSYS_OPT_FILTER */