drivers/common/sfc_efx/base/ef10_filter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2019-2021 Xilinx, Inc.
   4  * Copyright(c) 2007-2019 Solarflare Communications Inc.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10 #if EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10()
  11
  12 #if EFSYS_OPT_FILTER
  13
  14 #define EFE_SPEC(eftp, index)   ((eftp)->eft_entry[(index)].efe_spec)
  15
  16 static                  efx_filter_spec_t *
  17 ef10_filter_entry_spec(
  18         __in            const ef10_filter_table_t *eftp,
  19         __in            unsigned int index)
  20 {
  21         return ((efx_filter_spec_t *)(EFE_SPEC(eftp, index) &
  22                 ~(uintptr_t)EFX_EF10_FILTER_FLAGS));
  23 }
  24
  25 static                  boolean_t
  26 ef10_filter_entry_is_busy(
  27         __in            const ef10_filter_table_t *eftp,
  28         __in            unsigned int index)
  29 {
  30         if (EFE_SPEC(eftp, index) & EFX_EF10_FILTER_FLAG_BUSY)
  31                 return (B_TRUE);
  32         else
  33                 return (B_FALSE);
  34 }
  35
  36 static                  boolean_t
  37 ef10_filter_entry_is_auto_old(
  38         __in            const ef10_filter_table_t *eftp,
  39         __in            unsigned int index)
  40 {
  41         if (EFE_SPEC(eftp, index) & EFX_EF10_FILTER_FLAG_AUTO_OLD)
  42                 return (B_TRUE);
  43         else
  44                 return (B_FALSE);
  45 }
  46
  47 static                  void
  48 ef10_filter_set_entry(
  49         __inout         ef10_filter_table_t *eftp,
  50         __in            unsigned int index,
  51         __in_opt        const efx_filter_spec_t *efsp)
  52 {
  53         EFE_SPEC(eftp, index) = (uintptr_t)efsp;
  54 }
  55
  56 static                  void
  57 ef10_filter_set_entry_busy(
  58         __inout         ef10_filter_table_t *eftp,
  59         __in            unsigned int index)
  60 {
  61         EFE_SPEC(eftp, index) |= (uintptr_t)EFX_EF10_FILTER_FLAG_BUSY;
  62 }
  63
  64 static                  void
  65 ef10_filter_set_entry_not_busy(
  66         __inout         ef10_filter_table_t *eftp,
  67         __in            unsigned int index)
  68 {
  69         EFE_SPEC(eftp, index) &= ~(uintptr_t)EFX_EF10_FILTER_FLAG_BUSY;
  70 }
  71
  72 static                  void
  73 ef10_filter_set_entry_auto_old(
  74         __inout         ef10_filter_table_t *eftp,
  75         __in            unsigned int index)
  76 {
  77         EFSYS_ASSERT(ef10_filter_entry_spec(eftp, index) != NULL);
  78         EFE_SPEC(eftp, index) |= (uintptr_t)EFX_EF10_FILTER_FLAG_AUTO_OLD;
  79 }
  80
  81 static                  void
  82 ef10_filter_set_entry_not_auto_old(
  83         __inout         ef10_filter_table_t *eftp,
  84         __in            unsigned int index)
  85 {
  86         EFE_SPEC(eftp, index) &= ~(uintptr_t)EFX_EF10_FILTER_FLAG_AUTO_OLD;
  87         EFSYS_ASSERT(ef10_filter_entry_spec(eftp, index) != NULL);
  88 }
  89
  90         __checkReturn   efx_rc_t
  91 ef10_filter_init(
  92         __in            efx_nic_t *enp)
  93 {
  94         efx_rc_t rc;
  95         ef10_filter_table_t *eftp;
  96
  97         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
  98
  99 #define MATCH_MASK(match) (EFX_MASK32(match) << EFX_LOW_BIT(match))
 100         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_HOST ==
 101             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_IP));
 102         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_HOST ==
 103             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_IP));
 104         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_MAC ==
 105             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_MAC));
 106         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_PORT ==
 107             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_PORT));
 108         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_MAC ==
 109             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_MAC));
 110         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_PORT ==
 111             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_PORT));
 112         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_ETHER_TYPE ==
 113             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE));
 114         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_INNER_VID ==
 115             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_INNER_VLAN));
 116         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_OUTER_VID ==
 117             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_OUTER_VLAN));
 118         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IP_PROTO ==
 119             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO));
 120         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_VNI_OR_VSID ==
 121             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_VNI_OR_VSID));
 122         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_LOC_MAC ==
 123             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_DST_MAC));
 124         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST ==
 125             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST));
 126         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST ==
 127             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST));
 128         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_UNKNOWN_MCAST_DST ==
 129             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST));
 130         EFX_STATIC_ASSERT((uint32_t)EFX_FILTER_MATCH_UNKNOWN_UCAST_DST ==
 131             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST));
 132 #undef MATCH_MASK
 133
 134         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (ef10_filter_table_t), eftp);
 135
 136         if (!eftp) {
 137                 rc = ENOMEM;
 138                 goto fail1;
 139         }
 140
 141         enp->en_filter.ef_ef10_filter_table = eftp;
 142
 143         return (0);
 144
 145 fail1:
 146         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 147
 148         return (rc);
 149 }
 150
 151                         void
 152 ef10_filter_fini(
 153         __in            efx_nic_t *enp)
 154 {
 155         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
 156
 157         if (enp->en_filter.ef_ef10_filter_table != NULL) {
 158                 EFSYS_KMEM_FREE(enp->en_esip, sizeof (ef10_filter_table_t),
 159                     enp->en_filter.ef_ef10_filter_table);
 160         }
 161 }
 162
 163 static  __checkReturn   efx_rc_t
 164 efx_mcdi_filter_op_add(
 165         __in            efx_nic_t *enp,
 166         __in            efx_filter_spec_t *spec,
 167         __in            unsigned int filter_op,
 168         __inout         ef10_filter_handle_t *handle)
 169 {
 170         efx_mcdi_req_t req;
 171         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FILTER_OP_V3_IN_LEN,
 172                 MC_CMD_FILTER_OP_EXT_OUT_LEN);
 173         efx_filter_match_flags_t match_flags;
 174         uint32_t port_id;
 175         efx_rc_t rc;
 176
 177         req.emr_cmd = MC_CMD_FILTER_OP;
 178         req.emr_in_buf = payload;
 179         req.emr_in_length = MC_CMD_FILTER_OP_V3_IN_LEN;
 180         req.emr_out_buf = payload;
 181         req.emr_out_length = MC_CMD_FILTER_OP_EXT_OUT_LEN;
 182
 183         /*
 184          * Remove EFX match flags that do not correspond
 185          * to the MCDI match flags
 186          */
 187         match_flags = spec->efs_match_flags & ~EFX_FILTER_MATCH_ENCAP_TYPE;
 188         match_flags &= ~EFX_FILTER_MATCH_MPORT;
 189
 190         switch (filter_op) {
 191         case MC_CMD_FILTER_OP_IN_OP_REPLACE:
 192                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_LO,
 193                     handle->efh_lo);
 194                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_HI,
 195                     handle->efh_hi);
 196                 /* Fall through */
 197         case MC_CMD_FILTER_OP_IN_OP_INSERT:
 198         case MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE:
 199                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP, filter_op);
 200                 break;
 201         default:
 202                 EFSYS_ASSERT(0);
 203                 rc = EINVAL;
 204                 goto fail1;
 205         }
 206
 207         if (spec->efs_match_flags & EFX_FILTER_MATCH_MPORT)
 208                 port_id = spec->efs_ingress_mport;
 209         else
 210                 port_id = enp->en_vport_id;
 211
 212         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_PORT_ID, port_id);
 213         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_MATCH_FIELDS,
 214             match_flags);
 215         if (spec->efs_dmaq_id == EFX_FILTER_SPEC_RX_DMAQ_ID_DROP) {
 216                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_DEST,
 217                     MC_CMD_FILTER_OP_EXT_IN_RX_DEST_DROP);
 218         } else {
 219                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_DEST,
 220                     MC_CMD_FILTER_OP_EXT_IN_RX_DEST_HOST);
 221                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_QUEUE,
 222                     spec->efs_dmaq_id);
 223         }
 224
 225 #if EFSYS_OPT_RX_SCALE
 226         if (spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) {
 227                 uint32_t rss_context;
 228
 229                 if (spec->efs_rss_context == EFX_RSS_CONTEXT_DEFAULT)
 230                         rss_context = enp->en_rss_context;
 231                 else
 232                         rss_context = spec->efs_rss_context;
 233                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_CONTEXT,
 234                     rss_context);
 235         }
 236 #endif
 237
 238         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_MODE,
 239             spec->efs_flags & EFX_FILTER_FLAG_RX_RSS ?
 240             MC_CMD_FILTER_OP_EXT_IN_RX_MODE_RSS :
 241             MC_CMD_FILTER_OP_EXT_IN_RX_MODE_SIMPLE);
 242         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_TX_DEST,
 243             MC_CMD_FILTER_OP_EXT_IN_TX_DEST_DEFAULT);
 244
 245         if (filter_op != MC_CMD_FILTER_OP_IN_OP_REPLACE) {
 246                 /*
 247                  * NOTE: Unlike most MCDI requests, the filter fields
 248                  * are presented in network (big endian) byte order.
 249                  */
 250                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_SRC_MAC),
 251                     spec->efs_rem_mac, EFX_MAC_ADDR_LEN);
 252                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_DST_MAC),
 253                     spec->efs_loc_mac, EFX_MAC_ADDR_LEN);
 254
 255                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_SRC_PORT,
 256                     __CPU_TO_BE_16(spec->efs_rem_port));
 257                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_DST_PORT,
 258                     __CPU_TO_BE_16(spec->efs_loc_port));
 259
 260                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_ETHER_TYPE,
 261                     __CPU_TO_BE_16(spec->efs_ether_type));
 262
 263                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_INNER_VLAN,
 264                     __CPU_TO_BE_16(spec->efs_inner_vid));
 265                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_OUTER_VLAN,
 266                     __CPU_TO_BE_16(spec->efs_outer_vid));
 267
 268                 /* IP protocol (in low byte, high byte is zero) */
 269                 MCDI_IN_SET_BYTE(req, FILTER_OP_EXT_IN_IP_PROTO,
 270                     spec->efs_ip_proto);
 271
 272                 EFX_STATIC_ASSERT(sizeof (spec->efs_rem_host) ==
 273                     MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN);
 274                 EFX_STATIC_ASSERT(sizeof (spec->efs_loc_host) ==
 275                     MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN);
 276
 277                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_SRC_IP),
 278                     &spec->efs_rem_host.eo_byte[0],
 279                     MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN);
 280                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_DST_IP),
 281                     &spec->efs_loc_host.eo_byte[0],
 282                     MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN);
 283
 284                 /*
 285                  * On Medford, filters for encapsulated packets match based on
 286                  * the ether type and IP protocol in the outer frame.  In
 287                  * addition we need to fill in the VNI or VSID type field.
 288                  */
 289                 switch (spec->efs_encap_type) {
 290                 case EFX_TUNNEL_PROTOCOL_NONE:
 291                         break;
 292                 case EFX_TUNNEL_PROTOCOL_VXLAN:
 293                 case EFX_TUNNEL_PROTOCOL_GENEVE:
 294                         MCDI_IN_POPULATE_DWORD_1(req,
 295                             FILTER_OP_EXT_IN_VNI_OR_VSID,
 296                             FILTER_OP_EXT_IN_VNI_TYPE,
 297                             spec->efs_encap_type == EFX_TUNNEL_PROTOCOL_VXLAN ?
 298                                     MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN :
 299                                     MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE);
 300                         break;
 301                 case EFX_TUNNEL_PROTOCOL_NVGRE:
 302                         MCDI_IN_POPULATE_DWORD_1(req,
 303                             FILTER_OP_EXT_IN_VNI_OR_VSID,
 304                             FILTER_OP_EXT_IN_VSID_TYPE,
 305                             MC_CMD_FILTER_OP_EXT_IN_VSID_TYPE_NVGRE);
 306                         break;
 307                 default:
 308                         EFSYS_ASSERT(0);
 309                         rc = EINVAL;
 310                         goto fail2;
 311                 }
 312
 313                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_VNI_OR_VSID),
 314                     spec->efs_vni_or_vsid, EFX_VNI_OR_VSID_LEN);
 315
 316                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_IFRM_DST_MAC),
 317                     spec->efs_ifrm_loc_mac, EFX_MAC_ADDR_LEN);
 318         }
 319
 320         /*
 321          * Set the "MARK" or "FLAG" action for all packets matching this filter
 322          * if necessary (only useful with equal stride packed stream Rx mode
 323          * which provide the information in pseudo-header).
 324          * These actions require MC_CMD_FILTER_OP_V3_IN msgrequest.
 325          */
 326         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) &&
 327             (spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG)) {
 328                 rc = EINVAL;
 329                 goto fail3;
 330         }
 331         if (spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) {
 332                 MCDI_IN_SET_DWORD(req, FILTER_OP_V3_IN_MATCH_ACTION,
 333                     MC_CMD_FILTER_OP_V3_IN_MATCH_ACTION_MARK);
 334                 MCDI_IN_SET_DWORD(req, FILTER_OP_V3_IN_MATCH_MARK_VALUE,
 335                     spec->efs_mark);
 336         } else if (spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG) {
 337                 MCDI_IN_SET_DWORD(req, FILTER_OP_V3_IN_MATCH_ACTION,
 338                     MC_CMD_FILTER_OP_V3_IN_MATCH_ACTION_FLAG);
 339         }
 340
 341         efx_mcdi_execute(enp, &req);
 342
 343         if (req.emr_rc != 0) {
 344                 rc = req.emr_rc;
 345                 goto fail4;
 346         }
 347
 348         if (req.emr_out_length_used < MC_CMD_FILTER_OP_EXT_OUT_LEN) {
 349                 rc = EMSGSIZE;
 350                 goto fail5;
 351         }
 352
 353         handle->efh_lo = MCDI_OUT_DWORD(req, FILTER_OP_EXT_OUT_HANDLE_LO);
 354         handle->efh_hi = MCDI_OUT_DWORD(req, FILTER_OP_EXT_OUT_HANDLE_HI);
 355
 356         return (0);
 357
 358 fail5:
 359         EFSYS_PROBE(fail5);
 360 fail4:
 361         EFSYS_PROBE(fail4);
 362 fail3:
 363         EFSYS_PROBE(fail3);
 364 fail2:
 365         EFSYS_PROBE(fail2);
 366 fail1:
 367         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 368
 369         return (rc);
 370
 371 }
 372
 373 static  __checkReturn   efx_rc_t
 374 efx_mcdi_filter_op_delete(
 375         __in            efx_nic_t *enp,
 376         __in            unsigned int filter_op,
 377         __inout         ef10_filter_handle_t *handle)
 378 {
 379         efx_mcdi_req_t req;
 380         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FILTER_OP_EXT_IN_LEN,
 381                 MC_CMD_FILTER_OP_EXT_OUT_LEN);
 382         efx_rc_t rc;
 383
 384         req.emr_cmd = MC_CMD_FILTER_OP;
 385         req.emr_in_buf = payload;
 386         req.emr_in_length = MC_CMD_FILTER_OP_EXT_IN_LEN;
 387         req.emr_out_buf = payload;
 388         req.emr_out_length = MC_CMD_FILTER_OP_EXT_OUT_LEN;
 389
 390         switch (filter_op) {
 391         case MC_CMD_FILTER_OP_IN_OP_REMOVE:
 392                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP,
 393                     MC_CMD_FILTER_OP_IN_OP_REMOVE);
 394                 break;
 395         case MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE:
 396                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP,
 397                     MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
 398                 break;
 399         default:
 400                 EFSYS_ASSERT(0);
 401                 rc = EINVAL;
 402                 goto fail1;
 403         }
 404
 405         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_LO, handle->efh_lo);
 406         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_HI, handle->efh_hi);
 407
 408         efx_mcdi_execute_quiet(enp, &req);
 409
 410         if (req.emr_rc != 0) {
 411                 rc = req.emr_rc;
 412                 goto fail2;
 413         }
 414
 415         if (req.emr_out_length_used < MC_CMD_FILTER_OP_EXT_OUT_LEN) {
 416                 rc = EMSGSIZE;
 417                 goto fail3;
 418         }
 419
 420         return (0);
 421
 422 fail3:
 423         EFSYS_PROBE(fail3);
 424
 425 fail2:
 426         EFSYS_PROBE(fail2);
 427 fail1:
 428         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 429
 430         return (rc);
 431 }
 432
 433 static  __checkReturn   boolean_t
 434 ef10_filter_equal(
 435         __in            const efx_filter_spec_t *left,
 436         __in            const efx_filter_spec_t *right)
 437 {
 438         /* FIXME: Consider rx vs tx filters (look at efs_flags) */
 439         if (left->efs_match_flags != right->efs_match_flags)
 440                 return (B_FALSE);
 441         if (!EFX_OWORD_IS_EQUAL(left->efs_rem_host, right->efs_rem_host))
 442                 return (B_FALSE);
 443         if (!EFX_OWORD_IS_EQUAL(left->efs_loc_host, right->efs_loc_host))
 444                 return (B_FALSE);
 445         if (memcmp(left->efs_rem_mac, right->efs_rem_mac, EFX_MAC_ADDR_LEN))
 446                 return (B_FALSE);
 447         if (memcmp(left->efs_loc_mac, right->efs_loc_mac, EFX_MAC_ADDR_LEN))
 448                 return (B_FALSE);
 449         if (left->efs_rem_port != right->efs_rem_port)
 450                 return (B_FALSE);
 451         if (left->efs_loc_port != right->efs_loc_port)
 452                 return (B_FALSE);
 453         if (left->efs_inner_vid != right->efs_inner_vid)
 454                 return (B_FALSE);
 455         if (left->efs_outer_vid != right->efs_outer_vid)
 456                 return (B_FALSE);
 457         if (left->efs_ether_type != right->efs_ether_type)
 458                 return (B_FALSE);
 459         if (left->efs_ip_proto != right->efs_ip_proto)
 460                 return (B_FALSE);
 461         if (left->efs_encap_type != right->efs_encap_type)
 462                 return (B_FALSE);
 463         if (memcmp(left->efs_vni_or_vsid, right->efs_vni_or_vsid,
 464             EFX_VNI_OR_VSID_LEN))
 465                 return (B_FALSE);
 466         if (memcmp(left->efs_ifrm_loc_mac, right->efs_ifrm_loc_mac,
 467             EFX_MAC_ADDR_LEN))
 468                 return (B_FALSE);
 469
 470         return (B_TRUE);
 471
 472 }
 473
 474 static  __checkReturn   boolean_t
 475 ef10_filter_same_dest(
 476         __in            const efx_filter_spec_t *left,
 477         __in            const efx_filter_spec_t *right)
 478 {
 479         if ((left->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 480             (right->efs_flags & EFX_FILTER_FLAG_RX_RSS)) {
 481                 if (left->efs_rss_context == right->efs_rss_context)
 482                         return (B_TRUE);
 483         } else if ((~(left->efs_flags) & EFX_FILTER_FLAG_RX_RSS) &&
 484             (~(right->efs_flags) & EFX_FILTER_FLAG_RX_RSS)) {
 485                 if (left->efs_dmaq_id == right->efs_dmaq_id)
 486                         return (B_TRUE);
 487         }
 488         return (B_FALSE);
 489 }
 490
 491 static  __checkReturn   uint32_t
 492 ef10_filter_hash(
 493         __in            efx_filter_spec_t *spec)
 494 {
 495         EFX_STATIC_ASSERT((sizeof (efx_filter_spec_t) % sizeof (uint32_t))
 496                             == 0);
 497         EFX_STATIC_ASSERT((EFX_FIELD_OFFSET(efx_filter_spec_t, efs_outer_vid) %
 498                             sizeof (uint32_t)) == 0);
 499
 500         /*
 501          * As the area of the efx_filter_spec_t we need to hash is DWORD
 502          * aligned and an exact number of DWORDs in size we can use the
 503          * optimised efx_hash_dwords() rather than efx_hash_bytes()
 504          */
 505         return (efx_hash_dwords((const uint32_t *)&spec->efs_outer_vid,
 506                         (sizeof (efx_filter_spec_t) -
 507                         EFX_FIELD_OFFSET(efx_filter_spec_t, efs_outer_vid)) /
 508                         sizeof (uint32_t), 0));
 509 }
 510
 511 /*
 512  * Decide whether a filter should be exclusive or else should allow
 513  * delivery to additional recipients.  Currently we decide that
 514  * filters for specific local unicast MAC and IP addresses are
 515  * exclusive.
 516  */
 517 static  __checkReturn   boolean_t
 518 ef10_filter_is_exclusive(
 519         __in            efx_filter_spec_t *spec)
 520 {
 521         if ((spec->efs_match_flags & EFX_FILTER_MATCH_LOC_MAC) &&
 522             !EFX_MAC_ADDR_IS_MULTICAST(spec->efs_loc_mac))
 523                 return (B_TRUE);
 524
 525         if ((spec->efs_match_flags &
 526                 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
 527             (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
 528                 if ((spec->efs_ether_type == EFX_ETHER_TYPE_IPV4) &&
 529                     ((spec->efs_loc_host.eo_u8[0] & 0xf) != 0xe))
 530                         return (B_TRUE);
 531                 if ((spec->efs_ether_type == EFX_ETHER_TYPE_IPV6) &&
 532                     (spec->efs_loc_host.eo_u8[0] != 0xff))
 533                         return (B_TRUE);
 534         }
 535
 536         return (B_FALSE);
 537 }
 538
 539         __checkReturn   efx_rc_t
 540 ef10_filter_restore(
 541         __in            efx_nic_t *enp)
 542 {
 543         int tbl_id;
 544         efx_filter_spec_t *spec;
 545         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 546         boolean_t restoring;
 547         efsys_lock_state_t state;
 548         efx_rc_t rc;
 549
 550         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
 551
 552         for (tbl_id = 0; tbl_id < EFX_EF10_FILTER_TBL_ROWS; tbl_id++) {
 553
 554                 EFSYS_LOCK(enp->en_eslp, state);
 555
 556                 spec = ef10_filter_entry_spec(eftp, tbl_id);
 557                 if (spec == NULL) {
 558                         restoring = B_FALSE;
 559                 } else if (ef10_filter_entry_is_busy(eftp, tbl_id)) {
 560                         /* Ignore busy entries. */
 561                         restoring = B_FALSE;
 562                 } else {
 563                         ef10_filter_set_entry_busy(eftp, tbl_id);
 564                         restoring = B_TRUE;
 565                 }
 566
 567                 EFSYS_UNLOCK(enp->en_eslp, state);
 568
 569                 if (restoring == B_FALSE)
 570                         continue;
 571
 572                 if (ef10_filter_is_exclusive(spec)) {
 573                         rc = efx_mcdi_filter_op_add(enp, spec,
 574                             MC_CMD_FILTER_OP_IN_OP_INSERT,
 575                             &eftp->eft_entry[tbl_id].efe_handle);
 576                 } else {
 577                         rc = efx_mcdi_filter_op_add(enp, spec,
 578                             MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE,
 579                             &eftp->eft_entry[tbl_id].efe_handle);
 580                 }
 581
 582                 if (rc != 0)
 583                         goto fail1;
 584
 585                 EFSYS_LOCK(enp->en_eslp, state);
 586
 587                 ef10_filter_set_entry_not_busy(eftp, tbl_id);
 588
 589                 EFSYS_UNLOCK(enp->en_eslp, state);
 590         }
 591
 592         return (0);
 593
 594 fail1:
 595         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 596
 597         return (rc);
 598 }
 599
 600 enum ef10_filter_add_action_e {
 601         /* Insert a new filter */
 602         EF10_FILTER_ADD_NEW,
 603         /*
 604          * Replace old filter with a new, overriding the old one
 605          * if it has lower priority.
 606          */
 607         EF10_FILTER_ADD_REPLACE,
 608         /* Store new, lower priority filter as overridden by old filter */
 609         EF10_FILTER_ADD_STORE,
 610         /* Special case for AUTO filters, remove AUTO_OLD flag */
 611         EF10_FILTER_ADD_REFRESH,
 612 };
 613
 614 static  __checkReturn   efx_rc_t
 615 ef10_filter_add_lookup_equal_spec(
 616         __in            efx_filter_spec_t *spec,
 617         __in            efx_filter_spec_t *probe_spec,
 618         __in            efx_filter_replacement_policy_t policy,
 619         __out           boolean_t *found)
 620 {
 621         efx_rc_t rc;
 622
 623         /* Refreshing AUTO filter */
 624         if (spec->efs_priority == EFX_FILTER_PRI_AUTO &&
 625             probe_spec->efs_priority == EFX_FILTER_PRI_AUTO) {
 626                 *found = B_TRUE;
 627                 return (0);
 628         }
 629
 630         /*
 631          * With exclusive filters, higher priority ones
 632          * override lower priority ones, and lower priority
 633          * ones are stored in case the higher priority one
 634          * is removed.
 635          */
 636         if (ef10_filter_is_exclusive(spec)) {
 637                 switch (policy) {
 638                 case EFX_FILTER_REPLACEMENT_HIGHER_OR_EQUAL_PRIORITY:
 639                         if (spec->efs_priority == probe_spec->efs_priority) {
 640                                 *found = B_TRUE;
 641                                 break;
 642                         }
 643                         /* Fall-through */
 644                 case EFX_FILTER_REPLACEMENT_HIGHER_PRIORITY:
 645                         if (spec->efs_priority > probe_spec->efs_priority) {
 646                                 *found = B_TRUE;
 647                                 break;
 648                         }
 649                         /* Fall-through */
 650                 case EFX_FILTER_REPLACEMENT_NEVER:
 651                         /*
 652                          * Lower priority filter needs to be
 653                          * stored. It does *not* replace the
 654                          * old one. That is why EEXIST is not
 655                          * returned in that case.
 656                          */
 657                         if (spec->efs_priority < probe_spec->efs_priority) {
 658                                 *found = B_TRUE;
 659                                 break;
 660                         } else {
 661                                 rc = EEXIST;
 662                                 goto fail1;
 663                         }
 664                 default:
 665                         EFSYS_ASSERT(0);
 666                         rc = EEXIST;
 667                         goto fail2;
 668                 }
 669         } else {
 670                 *found = B_FALSE;
 671         }
 672
 673         return (0);
 674
 675 fail2:
 676         EFSYS_PROBE(fail2);
 677
 678 fail1:
 679         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 680
 681         return (rc);
 682 }
 683
 684
 685 static                  void
 686 ef10_filter_add_select_action(
 687         __in            efx_filter_spec_t *saved_spec,
 688         __in            efx_filter_spec_t *spec,
 689         __out           enum ef10_filter_add_action_e *action,
 690         __out           efx_filter_spec_t **overridden_spec)
 691 {
 692         efx_filter_spec_t *overridden = NULL;
 693
 694         if (saved_spec == NULL) {
 695                 *action = EF10_FILTER_ADD_NEW;
 696         } else if (ef10_filter_is_exclusive(spec) == B_FALSE) {
 697                 /*
 698                  * Non-exclusive filters are always stored in separate entries
 699                  * in the table. The only case involving a saved spec is
 700                  * refreshing an AUTO filter.
 701                  */
 702                 EFSYS_ASSERT(saved_spec->efs_overridden_spec == NULL);
 703                 EFSYS_ASSERT(spec->efs_priority == EFX_FILTER_PRI_AUTO);
 704                 EFSYS_ASSERT(saved_spec->efs_priority == EFX_FILTER_PRI_AUTO);
 705                 *action = EF10_FILTER_ADD_REFRESH;
 706         } else {
 707                 /* Exclusive filters stored in the same entry */
 708                 if (spec->efs_priority > saved_spec->efs_priority) {
 709                         /*
 710                          * Insert a high priority filter over a lower priority
 711                          * one. Only two priority levels are implemented, so
 712                          * there must not already be an overridden filter.
 713                          */
 714                         EFX_STATIC_ASSERT(EFX_FILTER_NPRI == 2);
 715                         EFSYS_ASSERT(saved_spec->efs_overridden_spec == NULL);
 716                         overridden = saved_spec;
 717                         *action = EF10_FILTER_ADD_REPLACE;
 718                 } else if (spec->efs_priority == saved_spec->efs_priority) {
 719                         /* Replace in-place or refresh an existing filter */
 720                         if (spec->efs_priority == EFX_FILTER_PRI_AUTO)
 721                                 *action = EF10_FILTER_ADD_REFRESH;
 722                         else
 723                                 *action = EF10_FILTER_ADD_REPLACE;
 724                 } else {
 725                         /*
 726                          * Insert a lower priority filter, storing it in case
 727                          * the higher priority filter is removed.
 728                          *
 729                          * Currently there are only two priority levels, so this
 730                          * must be an AUTO filter.
 731                          */
 732                         EFX_STATIC_ASSERT(EFX_FILTER_NPRI == 2);
 733                         EFSYS_ASSERT(spec->efs_priority == EFX_FILTER_PRI_AUTO);
 734                         if (saved_spec->efs_overridden_spec != NULL) {
 735                                 *action = EF10_FILTER_ADD_REFRESH;
 736                         } else {
 737                                 overridden = spec;
 738                                 *action = EF10_FILTER_ADD_STORE;
 739                         }
 740                 }
 741         }
 742
 743         *overridden_spec = overridden;
 744 }
 745
 746 static  __checkReturn   efx_rc_t
 747 ef10_filter_add_execute_action(
 748         __in            efx_nic_t *enp,
 749         __in            efx_filter_spec_t *saved_spec,
 750         __in            efx_filter_spec_t *spec,
 751         __in            efx_filter_spec_t *overridden_spec,
 752         __in            enum ef10_filter_add_action_e action,
 753         __in            int ins_index)
 754 {
 755         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 756         efsys_lock_state_t state;
 757         efx_rc_t rc;
 758
 759         EFSYS_LOCK(enp->en_eslp, state);
 760
 761         if (action == EF10_FILTER_ADD_REFRESH) {
 762                 ef10_filter_set_entry_not_auto_old(eftp, ins_index);
 763                 goto out_unlock;
 764         } else if (action == EF10_FILTER_ADD_STORE) {
 765                 EFSYS_ASSERT(overridden_spec != NULL);
 766                 saved_spec->efs_overridden_spec = overridden_spec;
 767                 goto out_unlock;
 768         }
 769
 770         EFSYS_UNLOCK(enp->en_eslp, state);
 771
 772         switch (action) {
 773         case EF10_FILTER_ADD_REPLACE:
 774                 /*
 775                  * On replacing the filter handle may change after a
 776                  * successful replace operation.
 777                  */
 778                 rc = efx_mcdi_filter_op_add(enp, spec,
 779                     MC_CMD_FILTER_OP_IN_OP_REPLACE,
 780                     &eftp->eft_entry[ins_index].efe_handle);
 781                 break;
 782         case EF10_FILTER_ADD_NEW:
 783                 if (ef10_filter_is_exclusive(spec)) {
 784                         rc = efx_mcdi_filter_op_add(enp, spec,
 785                             MC_CMD_FILTER_OP_IN_OP_INSERT,
 786                             &eftp->eft_entry[ins_index].efe_handle);
 787                 } else {
 788                         rc = efx_mcdi_filter_op_add(enp, spec,
 789                             MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE,
 790                             &eftp->eft_entry[ins_index].efe_handle);
 791                 }
 792                 break;
 793         default:
 794                 rc = EINVAL;
 795                 EFSYS_ASSERT(0);
 796                 break;
 797         }
 798         if (rc != 0)
 799                 goto fail1;
 800
 801         EFSYS_LOCK(enp->en_eslp, state);
 802
 803         if (action == EF10_FILTER_ADD_REPLACE) {
 804                 /* Update the fields that may differ */
 805                 saved_spec->efs_priority = spec->efs_priority;
 806                 saved_spec->efs_flags = spec->efs_flags;
 807                 saved_spec->efs_rss_context = spec->efs_rss_context;
 808                 saved_spec->efs_dmaq_id = spec->efs_dmaq_id;
 809
 810                 if (overridden_spec != NULL)
 811                         saved_spec->efs_overridden_spec = overridden_spec;
 812         }
 813
 814 out_unlock:
 815         EFSYS_UNLOCK(enp->en_eslp, state);
 816
 817         return (0);
 818
 819 fail1:
 820         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 821
 822         return (rc);
 823 }
 824
 825 /*
 826  * An arbitrary search limit for the software hash table. As per the linux net
 827  * driver.
 828  */
 829 #define EF10_FILTER_SEARCH_LIMIT 200
 830
 831 static  __checkReturn   efx_rc_t
 832 ef10_filter_add_internal(
 833         __in            efx_nic_t *enp,
 834         __inout         efx_filter_spec_t *spec,
 835         __in            efx_filter_replacement_policy_t policy,
 836         __out_opt       uint32_t *filter_id)
 837 {
 838         efx_rc_t rc;
 839         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 840         enum ef10_filter_add_action_e action;
 841         efx_filter_spec_t *overridden_spec = NULL;
 842         efx_filter_spec_t *saved_spec;
 843         uint32_t hash;
 844         unsigned int depth;
 845         int ins_index;
 846         efsys_lock_state_t state;
 847         boolean_t locked = B_FALSE;
 848
 849         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
 850
 851         EFSYS_ASSERT(spec->efs_overridden_spec == NULL);
 852
 853         hash = ef10_filter_hash(spec);
 854
 855         /*
 856          * FIXME: Add support for inserting filters of different priorities
 857          * and removing lower priority multicast filters (bug 42378)
 858          */
 859
 860         /*
 861          * Find any existing filters with the same match tuple or
 862          * else a free slot to insert at.  If any of them are busy,
 863          * we have to wait and retry.
 864          */
 865 retry:
 866         EFSYS_LOCK(enp->en_eslp, state);
 867         locked = B_TRUE;
 868
 869         ins_index = -1;
 870
 871         for (depth = 1; depth <= EF10_FILTER_SEARCH_LIMIT; depth++) {
 872                 unsigned int probe_index;
 873                 efx_filter_spec_t *probe_spec;
 874
 875                 probe_index = (hash + depth) & (EFX_EF10_FILTER_TBL_ROWS - 1);
 876                 probe_spec = ef10_filter_entry_spec(eftp, probe_index);
 877
 878                 if (probe_spec == NULL) {
 879                         if (ins_index < 0)
 880                                 ins_index = probe_index;
 881                 } else if (ef10_filter_equal(spec, probe_spec)) {
 882                         boolean_t found;
 883
 884                         if (ef10_filter_entry_is_busy(eftp, probe_index)) {
 885                                 EFSYS_UNLOCK(enp->en_eslp, state);
 886                                 locked = B_FALSE;
 887                                 goto retry;
 888                         }
 889
 890                         rc = ef10_filter_add_lookup_equal_spec(spec,
 891                             probe_spec, policy, &found);
 892                         if (rc != 0)
 893                                 goto fail1;
 894
 895                         if (found != B_FALSE) {
 896                                 ins_index = probe_index;
 897                                 break;
 898                         }
 899                 }
 900         }
 901
 902         /*
 903          * Once we reach the maximum search depth, use the first suitable slot
 904          * or return EBUSY if there was none.
 905          */
 906         if (ins_index < 0) {
 907                 rc = EBUSY;
 908                 goto fail2;
 909         }
 910
 911         /*
 912          * Mark software table entry busy. We might yet fail to insert,
 913          * but any attempt to insert a conflicting filter while we're
 914          * waiting for the firmware must find the busy entry.
 915          */
 916         ef10_filter_set_entry_busy(eftp, ins_index);
 917
 918         saved_spec = ef10_filter_entry_spec(eftp, ins_index);
 919         ef10_filter_add_select_action(saved_spec, spec, &action,
 920             &overridden_spec);
 921
 922         /*
 923          * Allocate a new filter if found entry is empty or
 924          * a filter should be overridden.
 925          */
 926         if (overridden_spec != NULL || saved_spec == NULL) {
 927                 efx_filter_spec_t *new_spec;
 928
 929                 EFSYS_UNLOCK(enp->en_eslp, state);
 930                 locked = B_FALSE;
 931
 932                 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (*new_spec), new_spec);
 933                 if (new_spec == NULL) {
 934                         rc = ENOMEM;
 935                         overridden_spec = NULL;
 936                         goto fail3;
 937                 }
 938
 939                 EFSYS_LOCK(enp->en_eslp, state);
 940                 locked = B_TRUE;
 941
 942                 if (saved_spec == NULL) {
 943                         *new_spec = *spec;
 944                         ef10_filter_set_entry(eftp, ins_index, new_spec);
 945                 } else {
 946                         *new_spec = *overridden_spec;
 947                         overridden_spec = new_spec;
 948                 }
 949         }
 950
 951         EFSYS_UNLOCK(enp->en_eslp, state);
 952         locked = B_FALSE;
 953
 954         rc = ef10_filter_add_execute_action(enp, saved_spec, spec,
 955             overridden_spec, action, ins_index);
 956         if (rc != 0)
 957                 goto fail4;
 958
 959         if (filter_id)
 960                 *filter_id = ins_index;
 961
 962         EFSYS_LOCK(enp->en_eslp, state);
 963         ef10_filter_set_entry_not_busy(eftp, ins_index);
 964         EFSYS_UNLOCK(enp->en_eslp, state);
 965
 966         return (0);
 967
 968 fail4:
 969         EFSYS_PROBE(fail4);
 970
 971         EFSYS_ASSERT(locked == B_FALSE);
 972         EFSYS_LOCK(enp->en_eslp, state);
 973
 974         if (action == EF10_FILTER_ADD_NEW) {
 975                 EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec),
 976                     ef10_filter_entry_spec(eftp, ins_index));
 977                 ef10_filter_set_entry(eftp, ins_index, NULL);
 978         }
 979
 980         EFSYS_UNLOCK(enp->en_eslp, state);
 981
 982         if (overridden_spec != NULL)
 983                 EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec), overridden_spec);
 984
 985 fail3:
 986         EFSYS_PROBE(fail3);
 987
 988         EFSYS_ASSERT(locked == B_FALSE);
 989         EFSYS_LOCK(enp->en_eslp, state);
 990
 991         ef10_filter_set_entry_not_busy(eftp, ins_index);
 992
 993         EFSYS_UNLOCK(enp->en_eslp, state);
 994
 995 fail2:
 996         EFSYS_PROBE(fail2);
 997
 998 fail1:
 999         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1000
1001         if (locked)
1002                 EFSYS_UNLOCK(enp->en_eslp, state);
1003
1004         return (rc);
1005 }
1006
1007         __checkReturn   efx_rc_t
1008 ef10_filter_add(
1009         __in            efx_nic_t *enp,
1010         __inout         efx_filter_spec_t *spec,
1011         __in            enum efx_filter_replacement_policy_e policy)
1012 {
1013         efx_rc_t rc;
1014
1015         rc = ef10_filter_add_internal(enp, spec, policy, NULL);
1016         if (rc != 0)
1017                 goto fail1;
1018
1019         return (0);
1020
1021 fail1:
1022         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1023
1024         return (rc);
1025 }
1026
1027 /*
1028  * Delete a filter by index from the filter table with priority
1029  * that is not higher than specified.
1030  */
1031 static  __checkReturn   efx_rc_t
1032 ef10_filter_delete_internal(
1033         __in            efx_nic_t *enp,
1034         __in            uint32_t filter_id,
1035         __in            efx_filter_priority_t priority)
1036 {
1037         efx_rc_t rc;
1038         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1039         efx_filter_spec_t *spec;
1040         efsys_lock_state_t state;
1041         uint32_t filter_idx = filter_id % EFX_EF10_FILTER_TBL_ROWS;
1042
1043         /*
1044          * Find the software table entry and mark it busy.  Don't
1045          * remove it yet; any attempt to update while we're waiting
1046          * for the firmware must find the busy entry.
1047          *
1048          * FIXME: What if the busy flag is never cleared?
1049          */
1050         EFSYS_LOCK(enp->en_eslp, state);
1051         while (ef10_filter_entry_is_busy(table, filter_idx)) {
1052                 EFSYS_UNLOCK(enp->en_eslp, state);
1053                 EFSYS_SPIN(1);
1054                 EFSYS_LOCK(enp->en_eslp, state);
1055         }
1056         if ((spec = ef10_filter_entry_spec(table, filter_idx)) != NULL) {
1057                 if (spec->efs_priority <= priority)
1058                         ef10_filter_set_entry_busy(table, filter_idx);
1059         }
1060         EFSYS_UNLOCK(enp->en_eslp, state);
1061
1062         if (spec == NULL) {
1063                 rc = ENOENT;
1064                 goto fail1;
1065         }
1066
1067         if (spec->efs_priority > priority) {
1068                 /*
1069                  * Applied filter stays, but overridden filter is removed since
1070                  * next user request to delete the applied filter should not
1071                  * restore outdated filter.
1072                  */
1073                 if (spec->efs_overridden_spec != NULL) {
1074                         EFSYS_ASSERT(spec->efs_overridden_spec->efs_overridden_spec ==
1075                             NULL);
1076                         EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec),
1077                             spec->efs_overridden_spec);
1078                         spec->efs_overridden_spec = NULL;
1079                 }
1080         } else {
1081                 /*
1082                  * Try to remove the hardware filter or replace it with the
1083                  * saved automatic filter. This may fail if the MC has
1084                  * rebooted (which frees all hardware filter resources).
1085                  */
1086                 if (spec->efs_overridden_spec != NULL) {
1087                         rc = efx_mcdi_filter_op_add(enp,
1088                             spec->efs_overridden_spec,
1089                             MC_CMD_FILTER_OP_IN_OP_REPLACE,
1090                             &table->eft_entry[filter_idx].efe_handle);
1091                 } else if (ef10_filter_is_exclusive(spec)) {
1092                         rc = efx_mcdi_filter_op_delete(enp,
1093                             MC_CMD_FILTER_OP_IN_OP_REMOVE,
1094                             &table->eft_entry[filter_idx].efe_handle);
1095                 } else {
1096                         rc = efx_mcdi_filter_op_delete(enp,
1097                             MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE,
1098                             &table->eft_entry[filter_idx].efe_handle);
1099                 }
1100
1101                 /* Free the software table entry */
1102                 EFSYS_LOCK(enp->en_eslp, state);
1103                 ef10_filter_set_entry_not_busy(table, filter_idx);
1104                 ef10_filter_set_entry(table, filter_idx,
1105                     spec->efs_overridden_spec);
1106                 EFSYS_UNLOCK(enp->en_eslp, state);
1107
1108                 EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec), spec);
1109
1110                 /* Check result of hardware filter removal */
1111                 if (rc != 0)
1112                         goto fail2;
1113         }
1114
1115         return (0);
1116
1117 fail2:
1118         EFSYS_PROBE(fail2);
1119
1120 fail1:
1121         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1122
1123         return (rc);
1124 }
1125
1126 static                  void
1127 ef10_filter_delete_auto(
1128         __in            efx_nic_t *enp,
1129         __in            uint32_t filter_id)
1130 {
1131         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1132         uint32_t filter_idx = filter_id % EFX_EF10_FILTER_TBL_ROWS;
1133
1134         /*
1135          * AUTO_OLD flag is cleared since the auto filter that is to be removed
1136          * may not be the filter at the specified index itself, but the filter
1137          * that is overridden by it.
1138          */
1139         ef10_filter_set_entry_not_auto_old(table, filter_idx);
1140
1141         (void) ef10_filter_delete_internal(enp, filter_idx,
1142             EFX_FILTER_PRI_AUTO);
1143 }
1144
1145         __checkReturn   efx_rc_t
1146 ef10_filter_delete(
1147         __in            efx_nic_t *enp,
1148         __inout         efx_filter_spec_t *spec)
1149 {
1150         efx_rc_t rc;
1151         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1152         efx_filter_spec_t *saved_spec;
1153         unsigned int hash;
1154         unsigned int depth;
1155         unsigned int i;
1156         efsys_lock_state_t state;
1157         boolean_t locked = B_FALSE;
1158
1159         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
1160
1161         hash = ef10_filter_hash(spec);
1162
1163         EFSYS_LOCK(enp->en_eslp, state);
1164         locked = B_TRUE;
1165
1166         depth = 1;
1167         for (;;) {
1168                 i = (hash + depth) & (EFX_EF10_FILTER_TBL_ROWS - 1);
1169                 saved_spec = ef10_filter_entry_spec(table, i);
1170                 if (saved_spec && ef10_filter_equal(spec, saved_spec) &&
1171                     ef10_filter_same_dest(spec, saved_spec) &&
1172                     saved_spec->efs_priority == EFX_FILTER_PRI_MANUAL) {
1173                         break;
1174                 }
1175                 if (depth == EF10_FILTER_SEARCH_LIMIT) {
1176                         rc = ENOENT;
1177                         goto fail1;
1178                 }
1179                 depth++;
1180         }
1181
1182         EFSYS_UNLOCK(enp->en_eslp, state);
1183         locked = B_FALSE;
1184
1185         rc = ef10_filter_delete_internal(enp, i, EFX_FILTER_PRI_MANUAL);
1186         if (rc != 0)
1187                 goto fail2;
1188
1189         return (0);
1190
1191 fail2:
1192         EFSYS_PROBE(fail2);
1193
1194 fail1:
1195         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1196
1197         if (locked)
1198                 EFSYS_UNLOCK(enp->en_eslp, state);
1199
1200         return (rc);
1201 }
1202
1203 static  __checkReturn   efx_rc_t
1204 efx_mcdi_get_parser_disp_info(
1205         __in                            efx_nic_t *enp,
1206         __out_ecount(buffer_length)     uint32_t *buffer,
1207         __in                            size_t buffer_length,
1208         __in                            boolean_t encap,
1209         __out                           size_t *list_lengthp)
1210 {
1211         efx_mcdi_req_t req;
1212         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN,
1213                 MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
1214         size_t matches_count;
1215         size_t list_size;
1216         efx_rc_t rc;
1217
1218         req.emr_cmd = MC_CMD_GET_PARSER_DISP_INFO;
1219         req.emr_in_buf = payload;
1220         req.emr_in_length = MC_CMD_GET_PARSER_DISP_INFO_IN_LEN;
1221         req.emr_out_buf = payload;
1222         req.emr_out_length = MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX;
1223
1224         MCDI_IN_SET_DWORD(req, GET_PARSER_DISP_INFO_OUT_OP, encap ?
1225             MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES :
1226             MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
1227
1228         efx_mcdi_execute(enp, &req);
1229
1230         if (req.emr_rc != 0) {
1231                 rc = req.emr_rc;
1232                 goto fail1;
1233         }
1234
1235         if (req.emr_out_length_used < MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMIN) {
1236                 rc = EMSGSIZE;
1237                 goto fail2;
1238         }
1239
1240         matches_count = MCDI_OUT_DWORD(req,
1241             GET_PARSER_DISP_INFO_OUT_NUM_SUPPORTED_MATCHES);
1242
1243         if (req.emr_out_length_used <
1244             MC_CMD_GET_PARSER_DISP_INFO_OUT_LEN(matches_count)) {
1245                 rc = EMSGSIZE;
1246                 goto fail3;
1247         }
1248
1249         *list_lengthp = matches_count;
1250
1251         if (buffer_length < matches_count) {
1252                 rc = ENOSPC;
1253                 goto fail4;
1254         }
1255
1256         /*
1257          * Check that the elements in the list in the MCDI response are the size
1258          * we expect, so we can just copy them directly. Any conversion of the
1259          * flags is handled by the caller.
1260          */
1261         EFX_STATIC_ASSERT(sizeof (uint32_t) ==
1262             MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN);
1263
1264         list_size = matches_count *
1265                 MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN;
1266         memcpy(buffer,
1267             MCDI_OUT2(req, uint32_t,
1268                     GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES),
1269             list_size);
1270
1271         return (0);
1272
1273 fail4:
1274         EFSYS_PROBE(fail4);
1275 fail3:
1276         EFSYS_PROBE(fail3);
1277 fail2:
1278         EFSYS_PROBE(fail2);
1279 fail1:
1280         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1281
1282         return (rc);
1283 }
1284
1285         __checkReturn   efx_rc_t
1286 ef10_filter_supported_filters(
1287         __in                            efx_nic_t *enp,
1288         __out_ecount(buffer_length)     uint32_t *buffer,
1289         __in                            size_t buffer_length,
1290         __out                           size_t *list_lengthp)
1291 {
1292         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1293         size_t mcdi_list_length;
1294         size_t mcdi_encap_list_length;
1295         size_t list_length;
1296         uint32_t i;
1297         uint32_t next_buf_idx;
1298         size_t next_buf_length;
1299         efx_rc_t rc;
1300         boolean_t no_space = B_FALSE;
1301         efx_filter_match_flags_t all_filter_flags =
1302             (EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_LOC_HOST |
1303             EFX_FILTER_MATCH_REM_MAC | EFX_FILTER_MATCH_REM_PORT |
1304             EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_PORT |
1305             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_INNER_VID |
1306             EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_IP_PROTO |
1307             EFX_FILTER_MATCH_VNI_OR_VSID |
1308             EFX_FILTER_MATCH_IFRM_LOC_MAC |
1309             EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST |
1310             EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST |
1311             EFX_FILTER_MATCH_ENCAP_TYPE |
1312             EFX_FILTER_MATCH_UNKNOWN_MCAST_DST |
1313             EFX_FILTER_MATCH_UNKNOWN_UCAST_DST);
1314
1315         /*
1316          * Two calls to MC_CMD_GET_PARSER_DISP_INFO are needed: one to get the
1317          * list of supported filters for ordinary packets, and then another to
1318          * get the list of supported filters for encapsulated packets. To
1319          * distinguish the second list from the first, the
1320          * EFX_FILTER_MATCH_ENCAP_TYPE flag is added to each filter for
1321          * encapsulated packets.
1322          */
1323         rc = efx_mcdi_get_parser_disp_info(enp, buffer, buffer_length, B_FALSE,
1324             &mcdi_list_length);
1325         if (rc != 0) {
1326                 if (rc == ENOSPC)
1327                         no_space = B_TRUE;
1328                 else
1329                         goto fail1;
1330         }
1331
1332         if (no_space) {
1333                 next_buf_idx = 0;
1334                 next_buf_length = 0;
1335         } else {
1336                 EFSYS_ASSERT(mcdi_list_length <= buffer_length);
1337                 next_buf_idx = mcdi_list_length;
1338                 next_buf_length = buffer_length - mcdi_list_length;
1339         }
1340
1341         if (encp->enc_tunnel_encapsulations_supported != 0) {
1342                 rc = efx_mcdi_get_parser_disp_info(enp, &buffer[next_buf_idx],
1343                     next_buf_length, B_TRUE, &mcdi_encap_list_length);
1344                 if (rc != 0) {
1345                         if (rc == ENOSPC) {
1346                                 no_space = B_TRUE;
1347                         } else if (rc == EINVAL) {
1348                                 /*
1349                                  * Do not fail if the MCDI do not recognize the
1350                                  * query for encapsulated packet filters.
1351                                  */
1352                                 mcdi_encap_list_length = 0;
1353                         } else
1354                                 goto fail2;
1355                 } else {
1356                         for (i = next_buf_idx;
1357                             i < next_buf_idx + mcdi_encap_list_length; i++)
1358                                 buffer[i] |= EFX_FILTER_MATCH_ENCAP_TYPE;
1359                 }
1360         } else {
1361                 mcdi_encap_list_length = 0;
1362         }
1363
1364         if (no_space) {
1365                 *list_lengthp = mcdi_list_length + mcdi_encap_list_length;
1366                 rc = ENOSPC;
1367                 goto fail3;
1368         }
1369
1370         /*
1371          * The static assertions in ef10_filter_init() ensure that the values of
1372          * the EFX_FILTER_MATCH flags match those used by MCDI, so they don't
1373          * need to be converted.
1374          *
1375          * In case support is added to MCDI for additional flags, remove any
1376          * matches from the list which include flags we don't support. The order
1377          * of the matches is preserved as they are ordered from highest to
1378          * lowest priority.
1379          */
1380         EFSYS_ASSERT(mcdi_list_length + mcdi_encap_list_length <=
1381             buffer_length);
1382         list_length = 0;
1383         for (i = 0; i < mcdi_list_length + mcdi_encap_list_length; i++) {
1384                 if ((buffer[i] & ~all_filter_flags) == 0) {
1385                         buffer[list_length] = buffer[i];
1386                         list_length++;
1387                 }
1388         }
1389
1390         *list_lengthp = list_length;
1391
1392         return (0);
1393
1394 fail3:
1395         EFSYS_PROBE(fail3);
1396 fail2:
1397         EFSYS_PROBE(fail2);
1398 fail1:
1399         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1400
1401         return (rc);
1402 }
1403
1404 static  __checkReturn   efx_rc_t
1405 ef10_filter_insert_unicast(
1406         __in                            efx_nic_t *enp,
1407         __in_ecount(6)                  uint8_t const *addr,
1408         __in                            efx_filter_flags_t filter_flags)
1409 {
1410         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
1411         efx_filter_spec_t spec;
1412         efx_rc_t rc;
1413
1414         /* Insert the filter for the local station address */
1415         efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1416             filter_flags,
1417             eftp->eft_default_rxq);
1418         rc = efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC,
1419             addr);
1420         if (rc != 0)
1421                 goto fail1;
1422
1423         rc = ef10_filter_add_internal(enp, &spec, EFX_FILTER_REPLACEMENT_NEVER,
1424             &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]);
1425         if (rc != 0)
1426                 goto fail2;
1427
1428         eftp->eft_unicst_filter_count++;
1429         EFSYS_ASSERT(eftp->eft_unicst_filter_count <=
1430                     EFX_EF10_FILTER_UNICAST_FILTERS_MAX);
1431
1432         return (0);
1433
1434 fail2:
1435         EFSYS_PROBE(fail2);
1436 fail1:
1437         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1438         return (rc);
1439 }
1440
1441 static  __checkReturn   efx_rc_t
1442 ef10_filter_insert_all_unicast(
1443         __in                            efx_nic_t *enp,
1444         __in                            efx_filter_flags_t filter_flags)
1445 {
1446         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
1447         efx_filter_spec_t spec;
1448         efx_rc_t rc;
1449
1450         /* Insert the unknown unicast filter */
1451         efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1452             filter_flags,
1453             eftp->eft_default_rxq);
1454         rc = efx_filter_spec_set_uc_def(&spec);
1455         if (rc != 0)
1456                 goto fail1;
1457         rc = ef10_filter_add_internal(enp, &spec, EFX_FILTER_REPLACEMENT_NEVER,
1458             &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]);
1459         if (rc != 0)
1460                 goto fail2;
1461
1462         eftp->eft_unicst_filter_count++;
1463         EFSYS_ASSERT(eftp->eft_unicst_filter_count <=
1464                     EFX_EF10_FILTER_UNICAST_FILTERS_MAX);
1465
1466         return (0);
1467
1468 fail2:
1469         EFSYS_PROBE(fail2);
1470 fail1:
1471         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1472         return (rc);
1473 }
1474
1475 static  __checkReturn   efx_rc_t
1476 ef10_filter_insert_multicast_list(
1477         __in                            efx_nic_t *enp,
1478         __in                            boolean_t mulcst,
1479         __in                            boolean_t brdcst,
1480         __in_ecount(6*count)            uint8_t const *addrs,
1481         __in                            uint32_t count,
1482         __in                            efx_filter_flags_t filter_flags,
1483         __in                            boolean_t rollback)
1484 {
1485         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
1486         efx_filter_spec_t spec;
1487         uint8_t addr[6];
1488         uint32_t i;
1489         uint32_t filter_index;
1490         uint32_t filter_count;
1491         efx_rc_t rc;
1492
1493         if (mulcst == B_FALSE)
1494                 count = 0;
1495
1496         if (count + (brdcst ? 1 : 0) >
1497             EFX_ARRAY_SIZE(eftp->eft_mulcst_filter_indexes)) {
1498                 /* Too many MAC addresses */
1499                 rc = EINVAL;
1500                 goto fail1;
1501         }
1502
1503         /* Insert/renew multicast address list filters */
1504         filter_count = 0;
1505         for (i = 0; i < count; i++) {
1506                 efx_filter_spec_init_rx(&spec,
1507                     EFX_FILTER_PRI_AUTO,
1508                     filter_flags,
1509                     eftp->eft_default_rxq);
1510
1511                 rc = efx_filter_spec_set_eth_local(&spec,
1512                     EFX_FILTER_SPEC_VID_UNSPEC,
1513                     &addrs[i * EFX_MAC_ADDR_LEN]);
1514                 if (rc != 0) {
1515                         if (rollback == B_TRUE) {
1516                                 /* Only stop upon failure if told to rollback */
1517                                 goto rollback;
1518                         } else {
1519                                 /*
1520                                  * Don't try to add a filter with a corrupt
1521                                  * specification.
1522                                  */
1523                                 continue;
1524                         }
1525                 }
1526
1527                 rc = ef10_filter_add_internal(enp, &spec,
1528                     EFX_FILTER_REPLACEMENT_NEVER, &filter_index);
1529
1530                 if (rc == 0) {
1531                         eftp->eft_mulcst_filter_indexes[filter_count] =
1532                                 filter_index;
1533                         filter_count++;
1534                 } else if (rollback == B_TRUE) {
1535                         /* Only stop upon failure if told to rollback */
1536                         goto rollback;
1537                 }
1538
1539         }
1540
1541         if (brdcst == B_TRUE) {
1542                 /* Insert/renew broadcast address filter */
1543                 efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1544                     filter_flags,
1545                     eftp->eft_default_rxq);
1546
1547                 EFX_MAC_BROADCAST_ADDR_SET(addr);
1548                 rc = efx_filter_spec_set_eth_local(&spec,
1549                     EFX_FILTER_SPEC_VID_UNSPEC, addr);
1550                 if ((rc != 0) && (rollback == B_TRUE)) {
1551                         /* Only stop upon failure if told to rollback */
1552                         goto rollback;
1553                 }
1554
1555                 rc = ef10_filter_add_internal(enp, &spec,
1556                     EFX_FILTER_REPLACEMENT_NEVER, &filter_index);
1557
1558                 if (rc == 0) {
1559                         eftp->eft_mulcst_filter_indexes[filter_count] =
1560                                 filter_index;
1561                         filter_count++;
1562                 } else if (rollback == B_TRUE) {
1563                         /* Only stop upon failure if told to rollback */
1564                         goto rollback;
1565                 }
1566         }
1567
1568         eftp->eft_mulcst_filter_count = filter_count;
1569         eftp->eft_using_all_mulcst = B_FALSE;
1570
1571         return (0);
1572
1573 rollback:
1574         /* Remove any filters we have inserted */
1575         i = filter_count;
1576         while (i--) {
1577                 ef10_filter_delete_auto(enp,
1578                     eftp->eft_mulcst_filter_indexes[i]);
1579         }
1580         eftp->eft_mulcst_filter_count = 0;
1581
1582 fail1:
1583         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1584
1585         return (rc);
1586 }
1587
1588 static  __checkReturn   efx_rc_t
1589 ef10_filter_insert_all_multicast(
1590         __in                            efx_nic_t *enp,
1591         __in                            efx_filter_flags_t filter_flags)
1592 {
1593         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
1594         efx_filter_spec_t spec;
1595         efx_rc_t rc;
1596
1597         /* Insert the unknown multicast filter */
1598         efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1599             filter_flags,
1600             eftp->eft_default_rxq);
1601         rc = efx_filter_spec_set_mc_def(&spec);
1602         if (rc != 0)
1603                 goto fail1;
1604
1605         rc = ef10_filter_add_internal(enp, &spec, EFX_FILTER_REPLACEMENT_NEVER,
1606             &eftp->eft_mulcst_filter_indexes[0]);
1607         if (rc != 0)
1608                 goto fail2;
1609
1610         eftp->eft_mulcst_filter_count = 1;
1611         eftp->eft_using_all_mulcst = B_TRUE;
1612
1613         /*
1614          * FIXME: If brdcst == B_FALSE, add a filter to drop broadcast traffic.
1615          */
1616
1617         return (0);
1618
1619 fail2:
1620         EFSYS_PROBE(fail2);
1621 fail1:
1622         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1623
1624         return (rc);
1625 }
1626
1627 typedef struct ef10_filter_encap_entry_s {
1628         uint16_t                ether_type;
1629         efx_tunnel_protocol_t   encap_type;
1630         uint32_t                inner_frame_match;
1631 } ef10_filter_encap_entry_t;
1632
1633 #define EF10_ENCAP_FILTER_ENTRY(ipv, encap_type, inner_frame_match)     \
1634         { EFX_ETHER_TYPE_##ipv, EFX_TUNNEL_PROTOCOL_##encap_type,       \
1635             EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_##inner_frame_match }
1636
1637 static ef10_filter_encap_entry_t ef10_filter_encap_list[] = {
1638         EF10_ENCAP_FILTER_ENTRY(IPV4, VXLAN, UCAST_DST),
1639         EF10_ENCAP_FILTER_ENTRY(IPV4, VXLAN, MCAST_DST),
1640         EF10_ENCAP_FILTER_ENTRY(IPV6, VXLAN, UCAST_DST),
1641         EF10_ENCAP_FILTER_ENTRY(IPV6, VXLAN, MCAST_DST),
1642
1643         EF10_ENCAP_FILTER_ENTRY(IPV4, GENEVE, UCAST_DST),
1644         EF10_ENCAP_FILTER_ENTRY(IPV4, GENEVE, MCAST_DST),
1645         EF10_ENCAP_FILTER_ENTRY(IPV6, GENEVE, UCAST_DST),
1646         EF10_ENCAP_FILTER_ENTRY(IPV6, GENEVE, MCAST_DST),
1647
1648         EF10_ENCAP_FILTER_ENTRY(IPV4, NVGRE, UCAST_DST),
1649         EF10_ENCAP_FILTER_ENTRY(IPV4, NVGRE, MCAST_DST),
1650         EF10_ENCAP_FILTER_ENTRY(IPV6, NVGRE, UCAST_DST),
1651         EF10_ENCAP_FILTER_ENTRY(IPV6, NVGRE, MCAST_DST),
1652 };
1653
1654 #undef EF10_ENCAP_FILTER_ENTRY
1655
1656 static  __checkReturn   efx_rc_t
1657 ef10_filter_insert_encap_filters(
1658         __in            efx_nic_t *enp,
1659         __in            boolean_t mulcst,
1660         __in            efx_filter_flags_t filter_flags)
1661 {
1662         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1663         uint32_t i;
1664         efx_rc_t rc;
1665
1666         EFX_STATIC_ASSERT(EFX_ARRAY_SIZE(ef10_filter_encap_list) <=
1667                             EFX_ARRAY_SIZE(table->eft_encap_filter_indexes));
1668
1669         /*
1670          * On Medford, full-featured firmware can identify packets as being
1671          * tunnel encapsulated, even if no encapsulated packet offloads are in
1672          * use. When packets are identified as such, ordinary filters are not
1673          * applied, only ones specific to encapsulated packets. Hence we need to
1674          * insert filters for encapsulated packets in order to receive them.
1675          *
1676          * Separate filters need to be inserted for each ether type,
1677          * encapsulation type, and inner frame type (unicast or multicast). To
1678          * keep things simple and reduce the number of filters needed, catch-all
1679          * filters for all combinations of types are inserted, even if
1680          * all_unicst or all_mulcst have not been set. (These catch-all filters
1681          * may well, however, fail to insert on unprivileged functions.)
1682          */
1683         table->eft_encap_filter_count = 0;
1684         for (i = 0; i < EFX_ARRAY_SIZE(ef10_filter_encap_list); i++) {
1685                 efx_filter_spec_t spec;
1686                 ef10_filter_encap_entry_t *encap_filter =
1687                         &ef10_filter_encap_list[i];
1688
1689                 /*
1690                  * Skip multicast filters if we've not been asked for
1691                  * any multicast traffic.
1692                  */
1693                 if ((mulcst == B_FALSE) &&
1694                     (encap_filter->inner_frame_match ==
1695                     EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST))
1696                         continue;
1697
1698                 efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1699                                         filter_flags,
1700                                         table->eft_default_rxq);
1701                 efx_filter_spec_set_ether_type(&spec, encap_filter->ether_type);
1702                 rc = efx_filter_spec_set_encap_type(&spec,
1703                                             encap_filter->encap_type,
1704                                             encap_filter->inner_frame_match);
1705                 if (rc != 0)
1706                         goto fail1;
1707
1708                 rc = ef10_filter_add_internal(enp, &spec,
1709                     EFX_FILTER_REPLACEMENT_NEVER,
1710                     &table->eft_encap_filter_indexes[
1711                                     table->eft_encap_filter_count]);
1712                 if (rc != 0) {
1713                         if (rc != EACCES)
1714                                 goto fail2;
1715                 } else {
1716                         table->eft_encap_filter_count++;
1717                 }
1718         }
1719
1720         return (0);
1721
1722 fail2:
1723         EFSYS_PROBE(fail2);
1724 fail1:
1725         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1726
1727         return (rc);
1728 }
1729
1730 static                  void
1731 ef10_filter_remove_old(
1732         __in            efx_nic_t *enp)
1733 {
1734         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1735         uint32_t i;
1736
1737         for (i = 0; i < EFX_ARRAY_SIZE(table->eft_entry); i++) {
1738                 if (ef10_filter_entry_is_auto_old(table, i)) {
1739                         ef10_filter_delete_auto(enp, i);
1740                 }
1741         }
1742 }
1743
1744
1745 static  __checkReturn   efx_rc_t
1746 ef10_filter_get_workarounds(
1747         __in                            efx_nic_t *enp)
1748 {
1749         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
1750         uint32_t implemented = 0;
1751         uint32_t enabled = 0;
1752         efx_rc_t rc;
1753
1754         rc = efx_mcdi_get_workarounds(enp, &implemented, &enabled);
1755         if (rc == 0) {
1756                 /* Check if chained multicast filter support is enabled */
1757                 if (implemented & enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807)
1758                         encp->enc_bug26807_workaround = B_TRUE;
1759                 else
1760                         encp->enc_bug26807_workaround = B_FALSE;
1761         } else if (rc == ENOTSUP) {
1762                 /*
1763                  * Firmware is too old to support GET_WORKAROUNDS, and support
1764                  * for this workaround was implemented later.
1765                  */
1766                 encp->enc_bug26807_workaround = B_FALSE;
1767         } else {
1768                 goto fail1;
1769         }
1770
1771         return (0);
1772
1773 fail1:
1774         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1775
1776         return (rc);
1777
1778 }
1779
1780 static                  void
1781 ef10_filter_remove_all_existing_filters(
1782         __in                            efx_nic_t *enp)
1783 {
1784         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1785         efx_port_t *epp = &(enp->en_port);
1786         unsigned int i;
1787
1788         for (i = 0; i < table->eft_unicst_filter_count; i++) {
1789                 ef10_filter_delete_auto(enp,
1790                                 table->eft_unicst_filter_indexes[i]);
1791         }
1792         table->eft_unicst_filter_count = 0;
1793
1794         for (i = 0; i < table->eft_mulcst_filter_count; i++) {
1795                 ef10_filter_delete_auto(enp,
1796                                 table->eft_mulcst_filter_indexes[i]);
1797         }
1798         table->eft_mulcst_filter_count = 0;
1799
1800         for (i = 0; i < table->eft_encap_filter_count; i++) {
1801                 ef10_filter_delete_auto(enp,
1802                                 table->eft_encap_filter_indexes[i]);
1803         }
1804         table->eft_encap_filter_count = 0;
1805
1806         epp->ep_all_unicst_inserted = B_FALSE;
1807         epp->ep_all_mulcst_inserted = B_FALSE;
1808 }
1809
1810 static                  void
1811 ef10_filter_mark_old_filters(
1812         __in                            efx_nic_t *enp)
1813 {
1814         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1815         unsigned int i;
1816
1817         for (i = 0; i < table->eft_unicst_filter_count; i++) {
1818                 ef10_filter_set_entry_auto_old(table,
1819                                         table->eft_unicst_filter_indexes[i]);
1820         }
1821         for (i = 0; i < table->eft_mulcst_filter_count; i++) {
1822                 ef10_filter_set_entry_auto_old(table,
1823                                         table->eft_mulcst_filter_indexes[i]);
1824         }
1825         for (i = 0; i < table->eft_encap_filter_count; i++) {
1826                 ef10_filter_set_entry_auto_old(table,
1827                                         table->eft_encap_filter_indexes[i]);
1828         }
1829 }
1830
1831 static  __checkReturn   efx_rc_t
1832 ef10_filter_insert_renew_unicst_filters(
1833         __in                            efx_nic_t *enp,
1834         __in_ecount(6)                  uint8_t const *mac_addr,
1835         __in                            boolean_t all_unicst,
1836         __in                            efx_filter_flags_t filter_flags,
1837         __out                           boolean_t *all_unicst_inserted)
1838 {
1839         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1840         efx_port_t *epp = &(enp->en_port);
1841         efx_rc_t rc;
1842
1843         /*
1844          * Firmware does not perform chaining on unicast filters. As traffic is
1845          * therefore only delivered to the first matching filter, we should
1846          * always insert the specific filter for our MAC address, to try and
1847          * ensure we get that traffic.
1848          *
1849          * (If the filter for our MAC address has already been inserted by
1850          * another function, we won't receive traffic sent to us, even if we
1851          * insert a unicast mismatch filter. To prevent traffic stealing, this
1852          * therefore relies on the privilege model only allowing functions to
1853          * insert filters for their own MAC address unless explicitly given
1854          * additional privileges by the user. This also means that, even on a
1855          * privileged function, inserting a unicast mismatch filter may not
1856          * catch all traffic in multi PCI function scenarios.)
1857          */
1858         table->eft_unicst_filter_count = 0;
1859         rc = ef10_filter_insert_unicast(enp, mac_addr, filter_flags);
1860         *all_unicst_inserted = B_FALSE;
1861         if (all_unicst || (rc != 0)) {
1862                 efx_rc_t all_unicst_rc;
1863
1864                 all_unicst_rc = ef10_filter_insert_all_unicast(enp,
1865                                                     filter_flags);
1866                 if (all_unicst_rc == 0) {
1867                         *all_unicst_inserted = B_TRUE;
1868                         epp->ep_all_unicst_inserted = B_TRUE;
1869                 } else if (rc != 0)
1870                         goto fail1;
1871         }
1872
1873         return (0);
1874
1875 fail1:
1876         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1877
1878         return (rc);
1879 }
1880
1881 static  __checkReturn   efx_rc_t
1882 ef10_filter_insert_renew_mulcst_filters(
1883         __in                            efx_nic_t *enp,
1884         __in                            boolean_t mulcst,
1885         __in                            boolean_t all_mulcst,
1886         __in                            boolean_t brdcst,
1887         __in_ecount(6*count)            uint8_t const *addrs,
1888         __in                            uint32_t count,
1889         __in                            efx_filter_flags_t filter_flags,
1890         __in                            boolean_t all_unicst_inserted,
1891         __out                           boolean_t *all_mulcst_inserted)
1892 {
1893         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1894         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
1895         efx_port_t *epp = &(enp->en_port);
1896         efx_rc_t rc;
1897
1898         *all_mulcst_inserted = B_FALSE;
1899
1900         if (all_mulcst == B_TRUE) {
1901                 efx_rc_t all_mulcst_rc;
1902
1903                 /*
1904                  * Insert the all multicast filter. If that fails, try to insert
1905                  * all of our multicast filters (but without rollback on
1906                  * failure).
1907                  */
1908                 all_mulcst_rc = ef10_filter_insert_all_multicast(enp,
1909                                                             filter_flags);
1910                 if (all_mulcst_rc == 0) {
1911                         epp->ep_all_mulcst_inserted = B_TRUE;
1912                         *all_mulcst_inserted = B_TRUE;
1913                 } else {
1914                         rc = ef10_filter_insert_multicast_list(enp, B_TRUE,
1915                             brdcst, addrs, count, filter_flags, B_FALSE);
1916                         if (rc != 0)
1917                                 goto fail1;
1918                 }
1919         } else {
1920                 /*
1921                  * Insert filters for multicast addresses.
1922                  * If any insertion fails, then rollback and try to insert the
1923                  * all multicast filter instead.
1924                  * If that also fails, try to insert all of the multicast
1925                  * filters (but without rollback on failure).
1926                  */
1927                 rc = ef10_filter_insert_multicast_list(enp, mulcst, brdcst,
1928                             addrs, count, filter_flags, B_TRUE);
1929                 if (rc != 0) {
1930                         if ((table->eft_using_all_mulcst == B_FALSE) &&
1931                             (encp->enc_bug26807_workaround == B_TRUE)) {
1932                                 /*
1933                                  * Multicast filter chaining is on, so remove
1934                                  * old filters before inserting the multicast
1935                                  * all filter to avoid duplicate delivery caused
1936                                  * by packets matching multiple filters.
1937                                  */
1938                                 ef10_filter_remove_old(enp);
1939                                 if (all_unicst_inserted == B_FALSE)
1940                                         epp->ep_all_unicst_inserted = B_FALSE;
1941                                 if (*all_mulcst_inserted == B_FALSE)
1942                                         epp->ep_all_mulcst_inserted = B_FALSE;
1943                         }
1944
1945                         rc = ef10_filter_insert_all_multicast(enp,
1946                                                             filter_flags);
1947                         if (rc == 0) {
1948                                 epp->ep_all_mulcst_inserted = B_TRUE;
1949                                 *all_mulcst_inserted = B_TRUE;
1950                         } else {
1951                                 rc = ef10_filter_insert_multicast_list(enp,
1952                                     mulcst, brdcst,
1953                                     addrs, count, filter_flags, B_FALSE);
1954                                 if (rc != 0)
1955                                         goto fail2;
1956                         }
1957                 }
1958         }
1959
1960         return (0);
1961
1962 fail2:
1963         EFSYS_PROBE1(fail2, efx_rc_t, rc);
1964
1965 fail1:
1966         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1967
1968         return (rc);
1969 }
1970
1971 /*
1972  * Reconfigure all filters.
1973  * If all_unicst and/or all mulcst filters cannot be applied then
1974  * return ENOTSUP (Note the filters for the specified addresses are
1975  * still applied in this case).
1976  */
1977         __checkReturn   efx_rc_t
1978 ef10_filter_reconfigure(
1979         __in                            efx_nic_t *enp,
1980         __in_ecount(6)                  uint8_t const *mac_addr,
1981         __in                            boolean_t all_unicst,
1982         __in                            boolean_t mulcst,
1983         __in                            boolean_t all_mulcst,
1984         __in                            boolean_t brdcst,
1985         __in_ecount(6*count)            uint8_t const *addrs,
1986         __in                            uint32_t count)
1987 {
1988         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
1989         efx_port_t *epp = &(enp->en_port);
1990         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1991         efx_filter_flags_t filter_flags;
1992         unsigned int i;
1993         boolean_t all_unicst_inserted = B_FALSE;
1994         boolean_t all_mulcst_inserted = B_FALSE;
1995         efx_rc_t rc;
1996
1997         if (table->eft_default_rxq == NULL) {
1998                 /*
1999                  * Filters direct traffic to the default RXQ, and so cannot be
2000                  * inserted until it is available. Any currently configured
2001                  * filters must be removed (ignore errors in case the MC
2002                  * has rebooted, which removes hardware filters).
2003                  */
2004                 ef10_filter_remove_all_existing_filters(enp);
2005                 return (0);
2006         }
2007
2008         if (table->eft_using_rss)
2009                 filter_flags = EFX_FILTER_FLAG_RX_RSS;
2010         else
2011                 filter_flags = 0;
2012
2013         /* Mark old filters which may need to be removed */
2014         ef10_filter_mark_old_filters(enp);
2015
2016         /* Insert or renew unicast filters */
2017         rc = ef10_filter_insert_renew_unicst_filters(enp, mac_addr, all_unicst,
2018                                                      filter_flags,
2019                                                      &all_unicst_inserted);
2020         if (rc != 0)
2021                 goto fail1;
2022
2023         /*
2024          * WORKAROUND_BUG26807 controls firmware support for chained multicast
2025          * filters, and can only be enabled or disabled when the hardware filter
2026          * table is empty.
2027          *
2028          * Chained multicast filters require support from the datapath firmware,
2029          * and may not be available (e.g. low-latency variants or old Huntington
2030          * firmware).
2031          *
2032          * Firmware will reset (FLR) functions which have inserted filters in
2033          * the hardware filter table when the workaround is enabled/disabled.
2034          * Functions without any hardware filters are not reset.
2035          *
2036          * Re-check if the workaround is enabled after adding unicast hardware
2037          * filters. This ensures that encp->enc_bug26807_workaround matches the
2038          * firmware state, and that later changes to enable/disable the
2039          * workaround will result in this function seeing a reset (FLR).
2040          *
2041          * In common-code drivers, we only support multiple PCI function
2042          * scenarios with firmware that supports multicast chaining, so we can
2043          * assume it is enabled for such cases and hence simplify the filter
2044          * insertion logic. Firmware that does not support multicast chaining
2045          * does not support multiple PCI function configurations either, so
2046          * filter insertion is much simpler and the same strategies can still be
2047          * used.
2048          */
2049         if ((rc = ef10_filter_get_workarounds(enp)) != 0)
2050                 goto fail2;
2051
2052         if ((table->eft_using_all_mulcst != all_mulcst) &&
2053             (encp->enc_bug26807_workaround == B_TRUE)) {
2054                 /*
2055                  * Multicast filter chaining is enabled, so traffic that matches
2056                  * more than one multicast filter will be replicated and
2057                  * delivered to multiple recipients.  To avoid this duplicate
2058                  * delivery, remove old multicast filters before inserting new
2059                  * multicast filters.
2060                  */
2061                 ef10_filter_remove_old(enp);
2062                 if (all_unicst_inserted == B_FALSE)
2063                         epp->ep_all_unicst_inserted = B_FALSE;
2064
2065                 epp->ep_all_mulcst_inserted = B_FALSE;
2066         }
2067
2068         /* Insert or renew multicast filters */
2069         rc = ef10_filter_insert_renew_mulcst_filters(enp, mulcst, all_mulcst,
2070                                                      brdcst, addrs, count,
2071                                                      filter_flags,
2072                                                      all_unicst_inserted,
2073                                                      &all_mulcst_inserted);
2074         if (rc != 0)
2075                 goto fail3;
2076
2077         if (encp->enc_tunnel_encapsulations_supported != 0) {
2078                 /* Try to insert filters for encapsulated packets. */
2079                 (void) ef10_filter_insert_encap_filters(enp,
2080                                             mulcst || all_mulcst || brdcst,
2081                                             filter_flags);
2082         }
2083
2084         /* Remove old filters which were not renewed */
2085         ef10_filter_remove_old(enp);
2086         if (all_unicst_inserted == B_FALSE)
2087                 epp->ep_all_unicst_inserted = B_FALSE;
2088         if (all_mulcst_inserted == B_FALSE)
2089                 epp->ep_all_mulcst_inserted = B_FALSE;
2090
2091         /* report if any optional flags were rejected */
2092         if (((all_unicst != B_FALSE) && (all_unicst_inserted == B_FALSE)) ||
2093             ((all_mulcst != B_FALSE) && (all_mulcst_inserted == B_FALSE))) {
2094                 rc = ENOTSUP;
2095         }
2096
2097         return (rc);
2098
2099 fail3:
2100         EFSYS_PROBE(fail3);
2101 fail2:
2102         EFSYS_PROBE(fail2);
2103 fail1:
2104         EFSYS_PROBE1(fail1, efx_rc_t, rc);
2105
2106         /* Clear auto old flags */
2107         for (i = 0; i < EFX_ARRAY_SIZE(table->eft_entry); i++) {
2108                 if (ef10_filter_entry_is_auto_old(table, i)) {
2109                         ef10_filter_set_entry_not_auto_old(table, i);
2110                 }
2111         }
2112
2113         return (rc);
2114 }
2115
2116                 void
2117 ef10_filter_get_default_rxq(
2118         __in            efx_nic_t *enp,
2119         __out           efx_rxq_t **erpp,
2120         __out           boolean_t *using_rss)
2121 {
2122         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
2123
2124         *erpp = table->eft_default_rxq;
2125         *using_rss = table->eft_using_rss;
2126 }
2127
2128
2129                 void
2130 ef10_filter_default_rxq_set(
2131         __in            efx_nic_t *enp,
2132         __in            efx_rxq_t *erp,
2133         __in            boolean_t using_rss)
2134 {
2135         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
2136
2137 #if EFSYS_OPT_RX_SCALE
2138         EFSYS_ASSERT((using_rss == B_FALSE) ||
2139             (enp->en_rss_context != EF10_RSS_CONTEXT_INVALID));
2140         table->eft_using_rss = using_rss;
2141 #else
2142         EFSYS_ASSERT(using_rss == B_FALSE);
2143         table->eft_using_rss = B_FALSE;
2144 #endif
2145         table->eft_default_rxq = erp;
2146 }
2147
2148                 void
2149 ef10_filter_default_rxq_clear(
2150         __in            efx_nic_t *enp)
2151 {
2152         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
2153
2154         table->eft_default_rxq = NULL;
2155         table->eft_using_rss = B_FALSE;
2156 }
2157
2158
2159 #endif /* EFSYS_OPT_FILTER */
2160
2161 #endif /* EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() */