drivers/common/sfc_efx/base/ef10_filter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2019-2021 Xilinx, Inc.
   4  * Copyright(c) 2007-2019 Solarflare Communications Inc.
   5  */
   6
   7 #include "efx.h"
   8 #include "efx_impl.h"
   9
  10 #if EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10()
  11
  12 #if EFSYS_OPT_FILTER
  13
  14 #define EFE_SPEC(eftp, index)   ((eftp)->eft_entry[(index)].efe_spec)
  15
  16 static                  efx_filter_spec_t *
  17 ef10_filter_entry_spec(
  18         __in            const ef10_filter_table_t *eftp,
  19         __in            unsigned int index)
  20 {
  21         return ((efx_filter_spec_t *)(EFE_SPEC(eftp, index) &
  22                 ~(uintptr_t)EFX_EF10_FILTER_FLAGS));
  23 }
  24
  25 static                  boolean_t
  26 ef10_filter_entry_is_busy(
  27         __in            const ef10_filter_table_t *eftp,
  28         __in            unsigned int index)
  29 {
  30         if (EFE_SPEC(eftp, index) & EFX_EF10_FILTER_FLAG_BUSY)
  31                 return (B_TRUE);
  32         else
  33                 return (B_FALSE);
  34 }
  35
  36 static                  boolean_t
  37 ef10_filter_entry_is_auto_old(
  38         __in            const ef10_filter_table_t *eftp,
  39         __in            unsigned int index)
  40 {
  41         if (EFE_SPEC(eftp, index) & EFX_EF10_FILTER_FLAG_AUTO_OLD)
  42                 return (B_TRUE);
  43         else
  44                 return (B_FALSE);
  45 }
  46
  47 static                  void
  48 ef10_filter_set_entry(
  49         __inout         ef10_filter_table_t *eftp,
  50         __in            unsigned int index,
  51         __in_opt        const efx_filter_spec_t *efsp)
  52 {
  53         EFE_SPEC(eftp, index) = (uintptr_t)efsp;
  54 }
  55
  56 static                  void
  57 ef10_filter_set_entry_busy(
  58         __inout         ef10_filter_table_t *eftp,
  59         __in            unsigned int index)
  60 {
  61         EFE_SPEC(eftp, index) |= (uintptr_t)EFX_EF10_FILTER_FLAG_BUSY;
  62 }
  63
  64 static                  void
  65 ef10_filter_set_entry_not_busy(
  66         __inout         ef10_filter_table_t *eftp,
  67         __in            unsigned int index)
  68 {
  69         EFE_SPEC(eftp, index) &= ~(uintptr_t)EFX_EF10_FILTER_FLAG_BUSY;
  70 }
  71
  72 static                  void
  73 ef10_filter_set_entry_auto_old(
  74         __inout         ef10_filter_table_t *eftp,
  75         __in            unsigned int index)
  76 {
  77         EFSYS_ASSERT(ef10_filter_entry_spec(eftp, index) != NULL);
  78         EFE_SPEC(eftp, index) |= (uintptr_t)EFX_EF10_FILTER_FLAG_AUTO_OLD;
  79 }
  80
  81 static                  void
  82 ef10_filter_set_entry_not_auto_old(
  83         __inout         ef10_filter_table_t *eftp,
  84         __in            unsigned int index)
  85 {
  86         EFE_SPEC(eftp, index) &= ~(uintptr_t)EFX_EF10_FILTER_FLAG_AUTO_OLD;
  87         EFSYS_ASSERT(ef10_filter_entry_spec(eftp, index) != NULL);
  88 }
  89
  90         __checkReturn   efx_rc_t
  91 ef10_filter_init(
  92         __in            efx_nic_t *enp)
  93 {
  94         efx_rc_t rc;
  95         ef10_filter_table_t *eftp;
  96
  97         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
  98
  99 #define MATCH_MASK(match) (EFX_MASK32(match) << EFX_LOW_BIT(match))
 100         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_HOST ==
 101             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_IP));
 102         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_HOST ==
 103             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_IP));
 104         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_MAC ==
 105             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_MAC));
 106         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_REM_PORT ==
 107             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_SRC_PORT));
 108         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_MAC ==
 109             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_MAC));
 110         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_LOC_PORT ==
 111             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_DST_PORT));
 112         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_ETHER_TYPE ==
 113             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE));
 114         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_INNER_VID ==
 115             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_INNER_VLAN));
 116         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_OUTER_VID ==
 117             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_OUTER_VLAN));
 118         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IP_PROTO ==
 119             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO));
 120         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_VNI_OR_VSID ==
 121             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_VNI_OR_VSID));
 122         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_LOC_MAC ==
 123             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_DST_MAC));
 124         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST ==
 125             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST));
 126         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST ==
 127             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST));
 128         EFX_STATIC_ASSERT(EFX_FILTER_MATCH_UNKNOWN_MCAST_DST ==
 129             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST));
 130         EFX_STATIC_ASSERT((uint32_t)EFX_FILTER_MATCH_UNKNOWN_UCAST_DST ==
 131             MATCH_MASK(MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST));
 132 #undef MATCH_MASK
 133
 134         EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (ef10_filter_table_t), eftp);
 135
 136         if (!eftp) {
 137                 rc = ENOMEM;
 138                 goto fail1;
 139         }
 140
 141         enp->en_filter.ef_ef10_filter_table = eftp;
 142
 143         return (0);
 144
 145 fail1:
 146         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 147
 148         return (rc);
 149 }
 150
 151                         void
 152 ef10_filter_fini(
 153         __in            efx_nic_t *enp)
 154 {
 155         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
 156
 157         if (enp->en_filter.ef_ef10_filter_table != NULL) {
 158                 EFSYS_KMEM_FREE(enp->en_esip, sizeof (ef10_filter_table_t),
 159                     enp->en_filter.ef_ef10_filter_table);
 160         }
 161 }
 162
 163 static  __checkReturn   efx_rc_t
 164 efx_mcdi_filter_op_add(
 165         __in            efx_nic_t *enp,
 166         __in            efx_filter_spec_t *spec,
 167         __in            unsigned int filter_op,
 168         __inout         ef10_filter_handle_t *handle)
 169 {
 170         efx_mcdi_req_t req;
 171         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FILTER_OP_V3_IN_LEN,
 172                 MC_CMD_FILTER_OP_EXT_OUT_LEN);
 173         efx_filter_match_flags_t match_flags;
 174         efx_rc_t rc;
 175
 176         req.emr_cmd = MC_CMD_FILTER_OP;
 177         req.emr_in_buf = payload;
 178         req.emr_in_length = MC_CMD_FILTER_OP_V3_IN_LEN;
 179         req.emr_out_buf = payload;
 180         req.emr_out_length = MC_CMD_FILTER_OP_EXT_OUT_LEN;
 181
 182         /*
 183          * Remove match flag for encapsulated filters that does not correspond
 184          * to the MCDI match flags
 185          */
 186         match_flags = spec->efs_match_flags & ~EFX_FILTER_MATCH_ENCAP_TYPE;
 187
 188         switch (filter_op) {
 189         case MC_CMD_FILTER_OP_IN_OP_REPLACE:
 190                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_LO,
 191                     handle->efh_lo);
 192                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_HI,
 193                     handle->efh_hi);
 194                 /* Fall through */
 195         case MC_CMD_FILTER_OP_IN_OP_INSERT:
 196         case MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE:
 197                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP, filter_op);
 198                 break;
 199         default:
 200                 EFSYS_ASSERT(0);
 201                 rc = EINVAL;
 202                 goto fail1;
 203         }
 204
 205         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_PORT_ID, enp->en_vport_id);
 206         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_MATCH_FIELDS,
 207             match_flags);
 208         if (spec->efs_dmaq_id == EFX_FILTER_SPEC_RX_DMAQ_ID_DROP) {
 209                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_DEST,
 210                     MC_CMD_FILTER_OP_EXT_IN_RX_DEST_DROP);
 211         } else {
 212                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_DEST,
 213                     MC_CMD_FILTER_OP_EXT_IN_RX_DEST_HOST);
 214                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_QUEUE,
 215                     spec->efs_dmaq_id);
 216         }
 217
 218 #if EFSYS_OPT_RX_SCALE
 219         if (spec->efs_flags & EFX_FILTER_FLAG_RX_RSS) {
 220                 uint32_t rss_context;
 221
 222                 if (spec->efs_rss_context == EFX_RSS_CONTEXT_DEFAULT)
 223                         rss_context = enp->en_rss_context;
 224                 else
 225                         rss_context = spec->efs_rss_context;
 226                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_CONTEXT,
 227                     rss_context);
 228         }
 229 #endif
 230
 231         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_RX_MODE,
 232             spec->efs_flags & EFX_FILTER_FLAG_RX_RSS ?
 233             MC_CMD_FILTER_OP_EXT_IN_RX_MODE_RSS :
 234             MC_CMD_FILTER_OP_EXT_IN_RX_MODE_SIMPLE);
 235         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_TX_DEST,
 236             MC_CMD_FILTER_OP_EXT_IN_TX_DEST_DEFAULT);
 237
 238         if (filter_op != MC_CMD_FILTER_OP_IN_OP_REPLACE) {
 239                 /*
 240                  * NOTE: Unlike most MCDI requests, the filter fields
 241                  * are presented in network (big endian) byte order.
 242                  */
 243                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_SRC_MAC),
 244                     spec->efs_rem_mac, EFX_MAC_ADDR_LEN);
 245                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_DST_MAC),
 246                     spec->efs_loc_mac, EFX_MAC_ADDR_LEN);
 247
 248                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_SRC_PORT,
 249                     __CPU_TO_BE_16(spec->efs_rem_port));
 250                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_DST_PORT,
 251                     __CPU_TO_BE_16(spec->efs_loc_port));
 252
 253                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_ETHER_TYPE,
 254                     __CPU_TO_BE_16(spec->efs_ether_type));
 255
 256                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_INNER_VLAN,
 257                     __CPU_TO_BE_16(spec->efs_inner_vid));
 258                 MCDI_IN_SET_WORD(req, FILTER_OP_EXT_IN_OUTER_VLAN,
 259                     __CPU_TO_BE_16(spec->efs_outer_vid));
 260
 261                 /* IP protocol (in low byte, high byte is zero) */
 262                 MCDI_IN_SET_BYTE(req, FILTER_OP_EXT_IN_IP_PROTO,
 263                     spec->efs_ip_proto);
 264
 265                 EFX_STATIC_ASSERT(sizeof (spec->efs_rem_host) ==
 266                     MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN);
 267                 EFX_STATIC_ASSERT(sizeof (spec->efs_loc_host) ==
 268                     MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN);
 269
 270                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_SRC_IP),
 271                     &spec->efs_rem_host.eo_byte[0],
 272                     MC_CMD_FILTER_OP_EXT_IN_SRC_IP_LEN);
 273                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_DST_IP),
 274                     &spec->efs_loc_host.eo_byte[0],
 275                     MC_CMD_FILTER_OP_EXT_IN_DST_IP_LEN);
 276
 277                 /*
 278                  * On Medford, filters for encapsulated packets match based on
 279                  * the ether type and IP protocol in the outer frame.  In
 280                  * addition we need to fill in the VNI or VSID type field.
 281                  */
 282                 switch (spec->efs_encap_type) {
 283                 case EFX_TUNNEL_PROTOCOL_NONE:
 284                         break;
 285                 case EFX_TUNNEL_PROTOCOL_VXLAN:
 286                 case EFX_TUNNEL_PROTOCOL_GENEVE:
 287                         MCDI_IN_POPULATE_DWORD_1(req,
 288                             FILTER_OP_EXT_IN_VNI_OR_VSID,
 289                             FILTER_OP_EXT_IN_VNI_TYPE,
 290                             spec->efs_encap_type == EFX_TUNNEL_PROTOCOL_VXLAN ?
 291                                     MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN :
 292                                     MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE);
 293                         break;
 294                 case EFX_TUNNEL_PROTOCOL_NVGRE:
 295                         MCDI_IN_POPULATE_DWORD_1(req,
 296                             FILTER_OP_EXT_IN_VNI_OR_VSID,
 297                             FILTER_OP_EXT_IN_VSID_TYPE,
 298                             MC_CMD_FILTER_OP_EXT_IN_VSID_TYPE_NVGRE);
 299                         break;
 300                 default:
 301                         EFSYS_ASSERT(0);
 302                         rc = EINVAL;
 303                         goto fail2;
 304                 }
 305
 306                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_VNI_OR_VSID),
 307                     spec->efs_vni_or_vsid, EFX_VNI_OR_VSID_LEN);
 308
 309                 memcpy(MCDI_IN2(req, uint8_t, FILTER_OP_EXT_IN_IFRM_DST_MAC),
 310                     spec->efs_ifrm_loc_mac, EFX_MAC_ADDR_LEN);
 311         }
 312
 313         /*
 314          * Set the "MARK" or "FLAG" action for all packets matching this filter
 315          * if necessary (only useful with equal stride packed stream Rx mode
 316          * which provide the information in pseudo-header).
 317          * These actions require MC_CMD_FILTER_OP_V3_IN msgrequest.
 318          */
 319         if ((spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) &&
 320             (spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG)) {
 321                 rc = EINVAL;
 322                 goto fail3;
 323         }
 324         if (spec->efs_flags & EFX_FILTER_FLAG_ACTION_MARK) {
 325                 MCDI_IN_SET_DWORD(req, FILTER_OP_V3_IN_MATCH_ACTION,
 326                     MC_CMD_FILTER_OP_V3_IN_MATCH_ACTION_MARK);
 327                 MCDI_IN_SET_DWORD(req, FILTER_OP_V3_IN_MATCH_MARK_VALUE,
 328                     spec->efs_mark);
 329         } else if (spec->efs_flags & EFX_FILTER_FLAG_ACTION_FLAG) {
 330                 MCDI_IN_SET_DWORD(req, FILTER_OP_V3_IN_MATCH_ACTION,
 331                     MC_CMD_FILTER_OP_V3_IN_MATCH_ACTION_FLAG);
 332         }
 333
 334         efx_mcdi_execute(enp, &req);
 335
 336         if (req.emr_rc != 0) {
 337                 rc = req.emr_rc;
 338                 goto fail4;
 339         }
 340
 341         if (req.emr_out_length_used < MC_CMD_FILTER_OP_EXT_OUT_LEN) {
 342                 rc = EMSGSIZE;
 343                 goto fail5;
 344         }
 345
 346         handle->efh_lo = MCDI_OUT_DWORD(req, FILTER_OP_EXT_OUT_HANDLE_LO);
 347         handle->efh_hi = MCDI_OUT_DWORD(req, FILTER_OP_EXT_OUT_HANDLE_HI);
 348
 349         return (0);
 350
 351 fail5:
 352         EFSYS_PROBE(fail5);
 353 fail4:
 354         EFSYS_PROBE(fail4);
 355 fail3:
 356         EFSYS_PROBE(fail3);
 357 fail2:
 358         EFSYS_PROBE(fail2);
 359 fail1:
 360         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 361
 362         return (rc);
 363
 364 }
 365
 366 static  __checkReturn   efx_rc_t
 367 efx_mcdi_filter_op_delete(
 368         __in            efx_nic_t *enp,
 369         __in            unsigned int filter_op,
 370         __inout         ef10_filter_handle_t *handle)
 371 {
 372         efx_mcdi_req_t req;
 373         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FILTER_OP_EXT_IN_LEN,
 374                 MC_CMD_FILTER_OP_EXT_OUT_LEN);
 375         efx_rc_t rc;
 376
 377         req.emr_cmd = MC_CMD_FILTER_OP;
 378         req.emr_in_buf = payload;
 379         req.emr_in_length = MC_CMD_FILTER_OP_EXT_IN_LEN;
 380         req.emr_out_buf = payload;
 381         req.emr_out_length = MC_CMD_FILTER_OP_EXT_OUT_LEN;
 382
 383         switch (filter_op) {
 384         case MC_CMD_FILTER_OP_IN_OP_REMOVE:
 385                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP,
 386                     MC_CMD_FILTER_OP_IN_OP_REMOVE);
 387                 break;
 388         case MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE:
 389                 MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_OP,
 390                     MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE);
 391                 break;
 392         default:
 393                 EFSYS_ASSERT(0);
 394                 rc = EINVAL;
 395                 goto fail1;
 396         }
 397
 398         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_LO, handle->efh_lo);
 399         MCDI_IN_SET_DWORD(req, FILTER_OP_EXT_IN_HANDLE_HI, handle->efh_hi);
 400
 401         efx_mcdi_execute_quiet(enp, &req);
 402
 403         if (req.emr_rc != 0) {
 404                 rc = req.emr_rc;
 405                 goto fail2;
 406         }
 407
 408         if (req.emr_out_length_used < MC_CMD_FILTER_OP_EXT_OUT_LEN) {
 409                 rc = EMSGSIZE;
 410                 goto fail3;
 411         }
 412
 413         return (0);
 414
 415 fail3:
 416         EFSYS_PROBE(fail3);
 417
 418 fail2:
 419         EFSYS_PROBE(fail2);
 420 fail1:
 421         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 422
 423         return (rc);
 424 }
 425
 426 static  __checkReturn   boolean_t
 427 ef10_filter_equal(
 428         __in            const efx_filter_spec_t *left,
 429         __in            const efx_filter_spec_t *right)
 430 {
 431         /* FIXME: Consider rx vs tx filters (look at efs_flags) */
 432         if (left->efs_match_flags != right->efs_match_flags)
 433                 return (B_FALSE);
 434         if (!EFX_OWORD_IS_EQUAL(left->efs_rem_host, right->efs_rem_host))
 435                 return (B_FALSE);
 436         if (!EFX_OWORD_IS_EQUAL(left->efs_loc_host, right->efs_loc_host))
 437                 return (B_FALSE);
 438         if (memcmp(left->efs_rem_mac, right->efs_rem_mac, EFX_MAC_ADDR_LEN))
 439                 return (B_FALSE);
 440         if (memcmp(left->efs_loc_mac, right->efs_loc_mac, EFX_MAC_ADDR_LEN))
 441                 return (B_FALSE);
 442         if (left->efs_rem_port != right->efs_rem_port)
 443                 return (B_FALSE);
 444         if (left->efs_loc_port != right->efs_loc_port)
 445                 return (B_FALSE);
 446         if (left->efs_inner_vid != right->efs_inner_vid)
 447                 return (B_FALSE);
 448         if (left->efs_outer_vid != right->efs_outer_vid)
 449                 return (B_FALSE);
 450         if (left->efs_ether_type != right->efs_ether_type)
 451                 return (B_FALSE);
 452         if (left->efs_ip_proto != right->efs_ip_proto)
 453                 return (B_FALSE);
 454         if (left->efs_encap_type != right->efs_encap_type)
 455                 return (B_FALSE);
 456         if (memcmp(left->efs_vni_or_vsid, right->efs_vni_or_vsid,
 457             EFX_VNI_OR_VSID_LEN))
 458                 return (B_FALSE);
 459         if (memcmp(left->efs_ifrm_loc_mac, right->efs_ifrm_loc_mac,
 460             EFX_MAC_ADDR_LEN))
 461                 return (B_FALSE);
 462
 463         return (B_TRUE);
 464
 465 }
 466
 467 static  __checkReturn   boolean_t
 468 ef10_filter_same_dest(
 469         __in            const efx_filter_spec_t *left,
 470         __in            const efx_filter_spec_t *right)
 471 {
 472         if ((left->efs_flags & EFX_FILTER_FLAG_RX_RSS) &&
 473             (right->efs_flags & EFX_FILTER_FLAG_RX_RSS)) {
 474                 if (left->efs_rss_context == right->efs_rss_context)
 475                         return (B_TRUE);
 476         } else if ((~(left->efs_flags) & EFX_FILTER_FLAG_RX_RSS) &&
 477             (~(right->efs_flags) & EFX_FILTER_FLAG_RX_RSS)) {
 478                 if (left->efs_dmaq_id == right->efs_dmaq_id)
 479                         return (B_TRUE);
 480         }
 481         return (B_FALSE);
 482 }
 483
 484 static  __checkReturn   uint32_t
 485 ef10_filter_hash(
 486         __in            efx_filter_spec_t *spec)
 487 {
 488         EFX_STATIC_ASSERT((sizeof (efx_filter_spec_t) % sizeof (uint32_t))
 489                             == 0);
 490         EFX_STATIC_ASSERT((EFX_FIELD_OFFSET(efx_filter_spec_t, efs_outer_vid) %
 491                             sizeof (uint32_t)) == 0);
 492
 493         /*
 494          * As the area of the efx_filter_spec_t we need to hash is DWORD
 495          * aligned and an exact number of DWORDs in size we can use the
 496          * optimised efx_hash_dwords() rather than efx_hash_bytes()
 497          */
 498         return (efx_hash_dwords((const uint32_t *)&spec->efs_outer_vid,
 499                         (sizeof (efx_filter_spec_t) -
 500                         EFX_FIELD_OFFSET(efx_filter_spec_t, efs_outer_vid)) /
 501                         sizeof (uint32_t), 0));
 502 }
 503
 504 /*
 505  * Decide whether a filter should be exclusive or else should allow
 506  * delivery to additional recipients.  Currently we decide that
 507  * filters for specific local unicast MAC and IP addresses are
 508  * exclusive.
 509  */
 510 static  __checkReturn   boolean_t
 511 ef10_filter_is_exclusive(
 512         __in            efx_filter_spec_t *spec)
 513 {
 514         if ((spec->efs_match_flags & EFX_FILTER_MATCH_LOC_MAC) &&
 515             !EFX_MAC_ADDR_IS_MULTICAST(spec->efs_loc_mac))
 516                 return (B_TRUE);
 517
 518         if ((spec->efs_match_flags &
 519                 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) ==
 520             (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) {
 521                 if ((spec->efs_ether_type == EFX_ETHER_TYPE_IPV4) &&
 522                     ((spec->efs_loc_host.eo_u8[0] & 0xf) != 0xe))
 523                         return (B_TRUE);
 524                 if ((spec->efs_ether_type == EFX_ETHER_TYPE_IPV6) &&
 525                     (spec->efs_loc_host.eo_u8[0] != 0xff))
 526                         return (B_TRUE);
 527         }
 528
 529         return (B_FALSE);
 530 }
 531
 532         __checkReturn   efx_rc_t
 533 ef10_filter_restore(
 534         __in            efx_nic_t *enp)
 535 {
 536         int tbl_id;
 537         efx_filter_spec_t *spec;
 538         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 539         boolean_t restoring;
 540         efsys_lock_state_t state;
 541         efx_rc_t rc;
 542
 543         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
 544
 545         for (tbl_id = 0; tbl_id < EFX_EF10_FILTER_TBL_ROWS; tbl_id++) {
 546
 547                 EFSYS_LOCK(enp->en_eslp, state);
 548
 549                 spec = ef10_filter_entry_spec(eftp, tbl_id);
 550                 if (spec == NULL) {
 551                         restoring = B_FALSE;
 552                 } else if (ef10_filter_entry_is_busy(eftp, tbl_id)) {
 553                         /* Ignore busy entries. */
 554                         restoring = B_FALSE;
 555                 } else {
 556                         ef10_filter_set_entry_busy(eftp, tbl_id);
 557                         restoring = B_TRUE;
 558                 }
 559
 560                 EFSYS_UNLOCK(enp->en_eslp, state);
 561
 562                 if (restoring == B_FALSE)
 563                         continue;
 564
 565                 if (ef10_filter_is_exclusive(spec)) {
 566                         rc = efx_mcdi_filter_op_add(enp, spec,
 567                             MC_CMD_FILTER_OP_IN_OP_INSERT,
 568                             &eftp->eft_entry[tbl_id].efe_handle);
 569                 } else {
 570                         rc = efx_mcdi_filter_op_add(enp, spec,
 571                             MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE,
 572                             &eftp->eft_entry[tbl_id].efe_handle);
 573                 }
 574
 575                 if (rc != 0)
 576                         goto fail1;
 577
 578                 EFSYS_LOCK(enp->en_eslp, state);
 579
 580                 ef10_filter_set_entry_not_busy(eftp, tbl_id);
 581
 582                 EFSYS_UNLOCK(enp->en_eslp, state);
 583         }
 584
 585         return (0);
 586
 587 fail1:
 588         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 589
 590         return (rc);
 591 }
 592
 593 enum ef10_filter_add_action_e {
 594         /* Insert a new filter */
 595         EF10_FILTER_ADD_NEW,
 596         /*
 597          * Replace old filter with a new, overriding the old one
 598          * if it has lower priority.
 599          */
 600         EF10_FILTER_ADD_REPLACE,
 601         /* Store new, lower priority filter as overridden by old filter */
 602         EF10_FILTER_ADD_STORE,
 603         /* Special case for AUTO filters, remove AUTO_OLD flag */
 604         EF10_FILTER_ADD_REFRESH,
 605 };
 606
 607 static  __checkReturn   efx_rc_t
 608 ef10_filter_add_lookup_equal_spec(
 609         __in            efx_filter_spec_t *spec,
 610         __in            efx_filter_spec_t *probe_spec,
 611         __in            efx_filter_replacement_policy_t policy,
 612         __out           boolean_t *found)
 613 {
 614         efx_rc_t rc;
 615
 616         /* Refreshing AUTO filter */
 617         if (spec->efs_priority == EFX_FILTER_PRI_AUTO &&
 618             probe_spec->efs_priority == EFX_FILTER_PRI_AUTO) {
 619                 *found = B_TRUE;
 620                 return (0);
 621         }
 622
 623         /*
 624          * With exclusive filters, higher priority ones
 625          * override lower priority ones, and lower priority
 626          * ones are stored in case the higher priority one
 627          * is removed.
 628          */
 629         if (ef10_filter_is_exclusive(spec)) {
 630                 switch (policy) {
 631                 case EFX_FILTER_REPLACEMENT_HIGHER_OR_EQUAL_PRIORITY:
 632                         if (spec->efs_priority == probe_spec->efs_priority) {
 633                                 *found = B_TRUE;
 634                                 break;
 635                         }
 636                         /* Fall-through */
 637                 case EFX_FILTER_REPLACEMENT_HIGHER_PRIORITY:
 638                         if (spec->efs_priority > probe_spec->efs_priority) {
 639                                 *found = B_TRUE;
 640                                 break;
 641                         }
 642                         /* Fall-through */
 643                 case EFX_FILTER_REPLACEMENT_NEVER:
 644                         /*
 645                          * Lower priority filter needs to be
 646                          * stored. It does *not* replace the
 647                          * old one. That is why EEXIST is not
 648                          * returned in that case.
 649                          */
 650                         if (spec->efs_priority < probe_spec->efs_priority) {
 651                                 *found = B_TRUE;
 652                                 break;
 653                         } else {
 654                                 rc = EEXIST;
 655                                 goto fail1;
 656                         }
 657                 default:
 658                         EFSYS_ASSERT(0);
 659                         rc = EEXIST;
 660                         goto fail2;
 661                 }
 662         } else {
 663                 *found = B_FALSE;
 664         }
 665
 666         return (0);
 667
 668 fail2:
 669         EFSYS_PROBE(fail2);
 670
 671 fail1:
 672         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 673
 674         return (rc);
 675 }
 676
 677
 678 static                  void
 679 ef10_filter_add_select_action(
 680         __in            efx_filter_spec_t *saved_spec,
 681         __in            efx_filter_spec_t *spec,
 682         __out           enum ef10_filter_add_action_e *action,
 683         __out           efx_filter_spec_t **overridden_spec)
 684 {
 685         efx_filter_spec_t *overridden = NULL;
 686
 687         if (saved_spec == NULL) {
 688                 *action = EF10_FILTER_ADD_NEW;
 689         } else if (ef10_filter_is_exclusive(spec) == B_FALSE) {
 690                 /*
 691                  * Non-exclusive filters are always stored in separate entries
 692                  * in the table. The only case involving a saved spec is
 693                  * refreshing an AUTO filter.
 694                  */
 695                 EFSYS_ASSERT(saved_spec->efs_overridden_spec == NULL);
 696                 EFSYS_ASSERT(spec->efs_priority == EFX_FILTER_PRI_AUTO);
 697                 EFSYS_ASSERT(saved_spec->efs_priority == EFX_FILTER_PRI_AUTO);
 698                 *action = EF10_FILTER_ADD_REFRESH;
 699         } else {
 700                 /* Exclusive filters stored in the same entry */
 701                 if (spec->efs_priority > saved_spec->efs_priority) {
 702                         /*
 703                          * Insert a high priority filter over a lower priority
 704                          * one. Only two priority levels are implemented, so
 705                          * there must not already be an overridden filter.
 706                          */
 707                         EFX_STATIC_ASSERT(EFX_FILTER_NPRI == 2);
 708                         EFSYS_ASSERT(saved_spec->efs_overridden_spec == NULL);
 709                         overridden = saved_spec;
 710                         *action = EF10_FILTER_ADD_REPLACE;
 711                 } else if (spec->efs_priority == saved_spec->efs_priority) {
 712                         /* Replace in-place or refresh an existing filter */
 713                         if (spec->efs_priority == EFX_FILTER_PRI_AUTO)
 714                                 *action = EF10_FILTER_ADD_REFRESH;
 715                         else
 716                                 *action = EF10_FILTER_ADD_REPLACE;
 717                 } else {
 718                         /*
 719                          * Insert a lower priority filter, storing it in case
 720                          * the higher priority filter is removed.
 721                          *
 722                          * Currently there are only two priority levels, so this
 723                          * must be an AUTO filter.
 724                          */
 725                         EFX_STATIC_ASSERT(EFX_FILTER_NPRI == 2);
 726                         EFSYS_ASSERT(spec->efs_priority == EFX_FILTER_PRI_AUTO);
 727                         if (saved_spec->efs_overridden_spec != NULL) {
 728                                 *action = EF10_FILTER_ADD_REFRESH;
 729                         } else {
 730                                 overridden = spec;
 731                                 *action = EF10_FILTER_ADD_STORE;
 732                         }
 733                 }
 734         }
 735
 736         *overridden_spec = overridden;
 737 }
 738
 739 static  __checkReturn   efx_rc_t
 740 ef10_filter_add_execute_action(
 741         __in            efx_nic_t *enp,
 742         __in            efx_filter_spec_t *saved_spec,
 743         __in            efx_filter_spec_t *spec,
 744         __in            efx_filter_spec_t *overridden_spec,
 745         __in            enum ef10_filter_add_action_e action,
 746         __in            int ins_index)
 747 {
 748         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 749         efsys_lock_state_t state;
 750         efx_rc_t rc;
 751
 752         EFSYS_LOCK(enp->en_eslp, state);
 753
 754         if (action == EF10_FILTER_ADD_REFRESH) {
 755                 ef10_filter_set_entry_not_auto_old(eftp, ins_index);
 756                 goto out_unlock;
 757         } else if (action == EF10_FILTER_ADD_STORE) {
 758                 EFSYS_ASSERT(overridden_spec != NULL);
 759                 saved_spec->efs_overridden_spec = overridden_spec;
 760                 goto out_unlock;
 761         }
 762
 763         EFSYS_UNLOCK(enp->en_eslp, state);
 764
 765         switch (action) {
 766         case EF10_FILTER_ADD_REPLACE:
 767                 /*
 768                  * On replacing the filter handle may change after a
 769                  * successful replace operation.
 770                  */
 771                 rc = efx_mcdi_filter_op_add(enp, spec,
 772                     MC_CMD_FILTER_OP_IN_OP_REPLACE,
 773                     &eftp->eft_entry[ins_index].efe_handle);
 774                 break;
 775         case EF10_FILTER_ADD_NEW:
 776                 if (ef10_filter_is_exclusive(spec)) {
 777                         rc = efx_mcdi_filter_op_add(enp, spec,
 778                             MC_CMD_FILTER_OP_IN_OP_INSERT,
 779                             &eftp->eft_entry[ins_index].efe_handle);
 780                 } else {
 781                         rc = efx_mcdi_filter_op_add(enp, spec,
 782                             MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE,
 783                             &eftp->eft_entry[ins_index].efe_handle);
 784                 }
 785                 break;
 786         default:
 787                 rc = EINVAL;
 788                 EFSYS_ASSERT(0);
 789                 break;
 790         }
 791         if (rc != 0)
 792                 goto fail1;
 793
 794         EFSYS_LOCK(enp->en_eslp, state);
 795
 796         if (action == EF10_FILTER_ADD_REPLACE) {
 797                 /* Update the fields that may differ */
 798                 saved_spec->efs_priority = spec->efs_priority;
 799                 saved_spec->efs_flags = spec->efs_flags;
 800                 saved_spec->efs_rss_context = spec->efs_rss_context;
 801                 saved_spec->efs_dmaq_id = spec->efs_dmaq_id;
 802
 803                 if (overridden_spec != NULL)
 804                         saved_spec->efs_overridden_spec = overridden_spec;
 805         }
 806
 807 out_unlock:
 808         EFSYS_UNLOCK(enp->en_eslp, state);
 809
 810         return (0);
 811
 812 fail1:
 813         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 814
 815         return (rc);
 816 }
 817
 818 /*
 819  * An arbitrary search limit for the software hash table. As per the linux net
 820  * driver.
 821  */
 822 #define EF10_FILTER_SEARCH_LIMIT 200
 823
 824 static  __checkReturn   efx_rc_t
 825 ef10_filter_add_internal(
 826         __in            efx_nic_t *enp,
 827         __inout         efx_filter_spec_t *spec,
 828         __in            efx_filter_replacement_policy_t policy,
 829         __out_opt       uint32_t *filter_id)
 830 {
 831         efx_rc_t rc;
 832         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
 833         enum ef10_filter_add_action_e action;
 834         efx_filter_spec_t *overridden_spec = NULL;
 835         efx_filter_spec_t *saved_spec;
 836         uint32_t hash;
 837         unsigned int depth;
 838         int ins_index;
 839         efsys_lock_state_t state;
 840         boolean_t locked = B_FALSE;
 841
 842         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
 843
 844         EFSYS_ASSERT(spec->efs_overridden_spec == NULL);
 845
 846         hash = ef10_filter_hash(spec);
 847
 848         /*
 849          * FIXME: Add support for inserting filters of different priorities
 850          * and removing lower priority multicast filters (bug 42378)
 851          */
 852
 853         /*
 854          * Find any existing filters with the same match tuple or
 855          * else a free slot to insert at.  If any of them are busy,
 856          * we have to wait and retry.
 857          */
 858 retry:
 859         EFSYS_LOCK(enp->en_eslp, state);
 860         locked = B_TRUE;
 861
 862         ins_index = -1;
 863
 864         for (depth = 1; depth <= EF10_FILTER_SEARCH_LIMIT; depth++) {
 865                 unsigned int probe_index;
 866                 efx_filter_spec_t *probe_spec;
 867
 868                 probe_index = (hash + depth) & (EFX_EF10_FILTER_TBL_ROWS - 1);
 869                 probe_spec = ef10_filter_entry_spec(eftp, probe_index);
 870
 871                 if (probe_spec == NULL) {
 872                         if (ins_index < 0)
 873                                 ins_index = probe_index;
 874                 } else if (ef10_filter_equal(spec, probe_spec)) {
 875                         boolean_t found;
 876
 877                         if (ef10_filter_entry_is_busy(eftp, probe_index)) {
 878                                 EFSYS_UNLOCK(enp->en_eslp, state);
 879                                 locked = B_FALSE;
 880                                 goto retry;
 881                         }
 882
 883                         rc = ef10_filter_add_lookup_equal_spec(spec,
 884                             probe_spec, policy, &found);
 885                         if (rc != 0)
 886                                 goto fail1;
 887
 888                         if (found != B_FALSE) {
 889                                 ins_index = probe_index;
 890                                 break;
 891                         }
 892                 }
 893         }
 894
 895         /*
 896          * Once we reach the maximum search depth, use the first suitable slot
 897          * or return EBUSY if there was none.
 898          */
 899         if (ins_index < 0) {
 900                 rc = EBUSY;
 901                 goto fail2;
 902         }
 903
 904         /*
 905          * Mark software table entry busy. We might yet fail to insert,
 906          * but any attempt to insert a conflicting filter while we're
 907          * waiting for the firmware must find the busy entry.
 908          */
 909         ef10_filter_set_entry_busy(eftp, ins_index);
 910
 911         saved_spec = ef10_filter_entry_spec(eftp, ins_index);
 912         ef10_filter_add_select_action(saved_spec, spec, &action,
 913             &overridden_spec);
 914
 915         /*
 916          * Allocate a new filter if found entry is empty or
 917          * a filter should be overridden.
 918          */
 919         if (overridden_spec != NULL || saved_spec == NULL) {
 920                 efx_filter_spec_t *new_spec;
 921
 922                 EFSYS_UNLOCK(enp->en_eslp, state);
 923                 locked = B_FALSE;
 924
 925                 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (*new_spec), new_spec);
 926                 if (new_spec == NULL) {
 927                         rc = ENOMEM;
 928                         overridden_spec = NULL;
 929                         goto fail3;
 930                 }
 931
 932                 EFSYS_LOCK(enp->en_eslp, state);
 933                 locked = B_TRUE;
 934
 935                 if (saved_spec == NULL) {
 936                         *new_spec = *spec;
 937                         ef10_filter_set_entry(eftp, ins_index, new_spec);
 938                 } else {
 939                         *new_spec = *overridden_spec;
 940                         overridden_spec = new_spec;
 941                 }
 942         }
 943
 944         EFSYS_UNLOCK(enp->en_eslp, state);
 945         locked = B_FALSE;
 946
 947         rc = ef10_filter_add_execute_action(enp, saved_spec, spec,
 948             overridden_spec, action, ins_index);
 949         if (rc != 0)
 950                 goto fail4;
 951
 952         if (filter_id)
 953                 *filter_id = ins_index;
 954
 955         EFSYS_LOCK(enp->en_eslp, state);
 956         ef10_filter_set_entry_not_busy(eftp, ins_index);
 957         EFSYS_UNLOCK(enp->en_eslp, state);
 958
 959         return (0);
 960
 961 fail4:
 962         EFSYS_PROBE(fail4);
 963
 964         EFSYS_ASSERT(locked == B_FALSE);
 965         EFSYS_LOCK(enp->en_eslp, state);
 966
 967         if (action == EF10_FILTER_ADD_NEW) {
 968                 EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec),
 969                     ef10_filter_entry_spec(eftp, ins_index));
 970                 ef10_filter_set_entry(eftp, ins_index, NULL);
 971         }
 972
 973         EFSYS_UNLOCK(enp->en_eslp, state);
 974
 975         if (overridden_spec != NULL)
 976                 EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec), overridden_spec);
 977
 978 fail3:
 979         EFSYS_PROBE(fail3);
 980
 981         EFSYS_ASSERT(locked == B_FALSE);
 982         EFSYS_LOCK(enp->en_eslp, state);
 983
 984         ef10_filter_set_entry_not_busy(eftp, ins_index);
 985
 986         EFSYS_UNLOCK(enp->en_eslp, state);
 987
 988 fail2:
 989         EFSYS_PROBE(fail2);
 990
 991 fail1:
 992         EFSYS_PROBE1(fail1, efx_rc_t, rc);
 993
 994         if (locked)
 995                 EFSYS_UNLOCK(enp->en_eslp, state);
 996
 997         return (rc);
 998 }
 999
1000         __checkReturn   efx_rc_t
1001 ef10_filter_add(
1002         __in            efx_nic_t *enp,
1003         __inout         efx_filter_spec_t *spec,
1004         __in            enum efx_filter_replacement_policy_e policy)
1005 {
1006         efx_rc_t rc;
1007
1008         rc = ef10_filter_add_internal(enp, spec, policy, NULL);
1009         if (rc != 0)
1010                 goto fail1;
1011
1012         return (0);
1013
1014 fail1:
1015         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1016
1017         return (rc);
1018 }
1019
1020 /*
1021  * Delete a filter by index from the filter table with priority
1022  * that is not higher than specified.
1023  */
1024 static  __checkReturn   efx_rc_t
1025 ef10_filter_delete_internal(
1026         __in            efx_nic_t *enp,
1027         __in            uint32_t filter_id,
1028         __in            efx_filter_priority_t priority)
1029 {
1030         efx_rc_t rc;
1031         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1032         efx_filter_spec_t *spec;
1033         efsys_lock_state_t state;
1034         uint32_t filter_idx = filter_id % EFX_EF10_FILTER_TBL_ROWS;
1035
1036         /*
1037          * Find the software table entry and mark it busy.  Don't
1038          * remove it yet; any attempt to update while we're waiting
1039          * for the firmware must find the busy entry.
1040          *
1041          * FIXME: What if the busy flag is never cleared?
1042          */
1043         EFSYS_LOCK(enp->en_eslp, state);
1044         while (ef10_filter_entry_is_busy(table, filter_idx)) {
1045                 EFSYS_UNLOCK(enp->en_eslp, state);
1046                 EFSYS_SPIN(1);
1047                 EFSYS_LOCK(enp->en_eslp, state);
1048         }
1049         if ((spec = ef10_filter_entry_spec(table, filter_idx)) != NULL) {
1050                 if (spec->efs_priority <= priority)
1051                         ef10_filter_set_entry_busy(table, filter_idx);
1052         }
1053         EFSYS_UNLOCK(enp->en_eslp, state);
1054
1055         if (spec == NULL) {
1056                 rc = ENOENT;
1057                 goto fail1;
1058         }
1059
1060         if (spec->efs_priority > priority) {
1061                 /*
1062                  * Applied filter stays, but overridden filter is removed since
1063                  * next user request to delete the applied filter should not
1064                  * restore outdated filter.
1065                  */
1066                 if (spec->efs_overridden_spec != NULL) {
1067                         EFSYS_ASSERT(spec->efs_overridden_spec->efs_overridden_spec ==
1068                             NULL);
1069                         EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec),
1070                             spec->efs_overridden_spec);
1071                         spec->efs_overridden_spec = NULL;
1072                 }
1073         } else {
1074                 /*
1075                  * Try to remove the hardware filter or replace it with the
1076                  * saved automatic filter. This may fail if the MC has
1077                  * rebooted (which frees all hardware filter resources).
1078                  */
1079                 if (spec->efs_overridden_spec != NULL) {
1080                         rc = efx_mcdi_filter_op_add(enp,
1081                             spec->efs_overridden_spec,
1082                             MC_CMD_FILTER_OP_IN_OP_REPLACE,
1083                             &table->eft_entry[filter_idx].efe_handle);
1084                 } else if (ef10_filter_is_exclusive(spec)) {
1085                         rc = efx_mcdi_filter_op_delete(enp,
1086                             MC_CMD_FILTER_OP_IN_OP_REMOVE,
1087                             &table->eft_entry[filter_idx].efe_handle);
1088                 } else {
1089                         rc = efx_mcdi_filter_op_delete(enp,
1090                             MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE,
1091                             &table->eft_entry[filter_idx].efe_handle);
1092                 }
1093
1094                 /* Free the software table entry */
1095                 EFSYS_LOCK(enp->en_eslp, state);
1096                 ef10_filter_set_entry_not_busy(table, filter_idx);
1097                 ef10_filter_set_entry(table, filter_idx,
1098                     spec->efs_overridden_spec);
1099                 EFSYS_UNLOCK(enp->en_eslp, state);
1100
1101                 EFSYS_KMEM_FREE(enp->en_esip, sizeof (*spec), spec);
1102
1103                 /* Check result of hardware filter removal */
1104                 if (rc != 0)
1105                         goto fail2;
1106         }
1107
1108         return (0);
1109
1110 fail2:
1111         EFSYS_PROBE(fail2);
1112
1113 fail1:
1114         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1115
1116         return (rc);
1117 }
1118
1119 static                  void
1120 ef10_filter_delete_auto(
1121         __in            efx_nic_t *enp,
1122         __in            uint32_t filter_id)
1123 {
1124         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1125         uint32_t filter_idx = filter_id % EFX_EF10_FILTER_TBL_ROWS;
1126
1127         /*
1128          * AUTO_OLD flag is cleared since the auto filter that is to be removed
1129          * may not be the filter at the specified index itself, but the filter
1130          * that is overridden by it.
1131          */
1132         ef10_filter_set_entry_not_auto_old(table, filter_idx);
1133
1134         (void) ef10_filter_delete_internal(enp, filter_idx,
1135             EFX_FILTER_PRI_AUTO);
1136 }
1137
1138         __checkReturn   efx_rc_t
1139 ef10_filter_delete(
1140         __in            efx_nic_t *enp,
1141         __inout         efx_filter_spec_t *spec)
1142 {
1143         efx_rc_t rc;
1144         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1145         efx_filter_spec_t *saved_spec;
1146         unsigned int hash;
1147         unsigned int depth;
1148         unsigned int i;
1149         efsys_lock_state_t state;
1150         boolean_t locked = B_FALSE;
1151
1152         EFSYS_ASSERT(EFX_FAMILY_IS_EF100(enp) || EFX_FAMILY_IS_EF10(enp));
1153
1154         hash = ef10_filter_hash(spec);
1155
1156         EFSYS_LOCK(enp->en_eslp, state);
1157         locked = B_TRUE;
1158
1159         depth = 1;
1160         for (;;) {
1161                 i = (hash + depth) & (EFX_EF10_FILTER_TBL_ROWS - 1);
1162                 saved_spec = ef10_filter_entry_spec(table, i);
1163                 if (saved_spec && ef10_filter_equal(spec, saved_spec) &&
1164                     ef10_filter_same_dest(spec, saved_spec) &&
1165                     saved_spec->efs_priority == EFX_FILTER_PRI_MANUAL) {
1166                         break;
1167                 }
1168                 if (depth == EF10_FILTER_SEARCH_LIMIT) {
1169                         rc = ENOENT;
1170                         goto fail1;
1171                 }
1172                 depth++;
1173         }
1174
1175         EFSYS_UNLOCK(enp->en_eslp, state);
1176         locked = B_FALSE;
1177
1178         rc = ef10_filter_delete_internal(enp, i, EFX_FILTER_PRI_MANUAL);
1179         if (rc != 0)
1180                 goto fail2;
1181
1182         return (0);
1183
1184 fail2:
1185         EFSYS_PROBE(fail2);
1186
1187 fail1:
1188         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1189
1190         if (locked)
1191                 EFSYS_UNLOCK(enp->en_eslp, state);
1192
1193         return (rc);
1194 }
1195
1196 static  __checkReturn   efx_rc_t
1197 efx_mcdi_get_parser_disp_info(
1198         __in                            efx_nic_t *enp,
1199         __out_ecount(buffer_length)     uint32_t *buffer,
1200         __in                            size_t buffer_length,
1201         __in                            boolean_t encap,
1202         __out                           size_t *list_lengthp)
1203 {
1204         efx_mcdi_req_t req;
1205         EFX_MCDI_DECLARE_BUF(payload, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN,
1206                 MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX);
1207         size_t matches_count;
1208         size_t list_size;
1209         efx_rc_t rc;
1210
1211         req.emr_cmd = MC_CMD_GET_PARSER_DISP_INFO;
1212         req.emr_in_buf = payload;
1213         req.emr_in_length = MC_CMD_GET_PARSER_DISP_INFO_IN_LEN;
1214         req.emr_out_buf = payload;
1215         req.emr_out_length = MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX;
1216
1217         MCDI_IN_SET_DWORD(req, GET_PARSER_DISP_INFO_OUT_OP, encap ?
1218             MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES :
1219             MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES);
1220
1221         efx_mcdi_execute(enp, &req);
1222
1223         if (req.emr_rc != 0) {
1224                 rc = req.emr_rc;
1225                 goto fail1;
1226         }
1227
1228         if (req.emr_out_length_used < MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMIN) {
1229                 rc = EMSGSIZE;
1230                 goto fail2;
1231         }
1232
1233         matches_count = MCDI_OUT_DWORD(req,
1234             GET_PARSER_DISP_INFO_OUT_NUM_SUPPORTED_MATCHES);
1235
1236         if (req.emr_out_length_used <
1237             MC_CMD_GET_PARSER_DISP_INFO_OUT_LEN(matches_count)) {
1238                 rc = EMSGSIZE;
1239                 goto fail3;
1240         }
1241
1242         *list_lengthp = matches_count;
1243
1244         if (buffer_length < matches_count) {
1245                 rc = ENOSPC;
1246                 goto fail4;
1247         }
1248
1249         /*
1250          * Check that the elements in the list in the MCDI response are the size
1251          * we expect, so we can just copy them directly. Any conversion of the
1252          * flags is handled by the caller.
1253          */
1254         EFX_STATIC_ASSERT(sizeof (uint32_t) ==
1255             MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN);
1256
1257         list_size = matches_count *
1258                 MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_LEN;
1259         memcpy(buffer,
1260             MCDI_OUT2(req, uint32_t,
1261                     GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES),
1262             list_size);
1263
1264         return (0);
1265
1266 fail4:
1267         EFSYS_PROBE(fail4);
1268 fail3:
1269         EFSYS_PROBE(fail3);
1270 fail2:
1271         EFSYS_PROBE(fail2);
1272 fail1:
1273         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1274
1275         return (rc);
1276 }
1277
1278         __checkReturn   efx_rc_t
1279 ef10_filter_supported_filters(
1280         __in                            efx_nic_t *enp,
1281         __out_ecount(buffer_length)     uint32_t *buffer,
1282         __in                            size_t buffer_length,
1283         __out                           size_t *list_lengthp)
1284 {
1285         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1286         size_t mcdi_list_length;
1287         size_t mcdi_encap_list_length;
1288         size_t list_length;
1289         uint32_t i;
1290         uint32_t next_buf_idx;
1291         size_t next_buf_length;
1292         efx_rc_t rc;
1293         boolean_t no_space = B_FALSE;
1294         efx_filter_match_flags_t all_filter_flags =
1295             (EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_LOC_HOST |
1296             EFX_FILTER_MATCH_REM_MAC | EFX_FILTER_MATCH_REM_PORT |
1297             EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_PORT |
1298             EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_INNER_VID |
1299             EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_IP_PROTO |
1300             EFX_FILTER_MATCH_VNI_OR_VSID |
1301             EFX_FILTER_MATCH_IFRM_LOC_MAC |
1302             EFX_FILTER_MATCH_IFRM_UNKNOWN_MCAST_DST |
1303             EFX_FILTER_MATCH_IFRM_UNKNOWN_UCAST_DST |
1304             EFX_FILTER_MATCH_ENCAP_TYPE |
1305             EFX_FILTER_MATCH_UNKNOWN_MCAST_DST |
1306             EFX_FILTER_MATCH_UNKNOWN_UCAST_DST);
1307
1308         /*
1309          * Two calls to MC_CMD_GET_PARSER_DISP_INFO are needed: one to get the
1310          * list of supported filters for ordinary packets, and then another to
1311          * get the list of supported filters for encapsulated packets. To
1312          * distinguish the second list from the first, the
1313          * EFX_FILTER_MATCH_ENCAP_TYPE flag is added to each filter for
1314          * encapsulated packets.
1315          */
1316         rc = efx_mcdi_get_parser_disp_info(enp, buffer, buffer_length, B_FALSE,
1317             &mcdi_list_length);
1318         if (rc != 0) {
1319                 if (rc == ENOSPC)
1320                         no_space = B_TRUE;
1321                 else
1322                         goto fail1;
1323         }
1324
1325         if (no_space) {
1326                 next_buf_idx = 0;
1327                 next_buf_length = 0;
1328         } else {
1329                 EFSYS_ASSERT(mcdi_list_length <= buffer_length);
1330                 next_buf_idx = mcdi_list_length;
1331                 next_buf_length = buffer_length - mcdi_list_length;
1332         }
1333
1334         if (encp->enc_tunnel_encapsulations_supported != 0) {
1335                 rc = efx_mcdi_get_parser_disp_info(enp, &buffer[next_buf_idx],
1336                     next_buf_length, B_TRUE, &mcdi_encap_list_length);
1337                 if (rc != 0) {
1338                         if (rc == ENOSPC) {
1339                                 no_space = B_TRUE;
1340                         } else if (rc == EINVAL) {
1341                                 /*
1342                                  * Do not fail if the MCDI do not recognize the
1343                                  * query for encapsulated packet filters.
1344                                  */
1345                                 mcdi_encap_list_length = 0;
1346                         } else
1347                                 goto fail2;
1348                 } else {
1349                         for (i = next_buf_idx;
1350                             i < next_buf_idx + mcdi_encap_list_length; i++)
1351                                 buffer[i] |= EFX_FILTER_MATCH_ENCAP_TYPE;
1352                 }
1353         } else {
1354                 mcdi_encap_list_length = 0;
1355         }
1356
1357         if (no_space) {
1358                 *list_lengthp = mcdi_list_length + mcdi_encap_list_length;
1359                 rc = ENOSPC;
1360                 goto fail3;
1361         }
1362
1363         /*
1364          * The static assertions in ef10_filter_init() ensure that the values of
1365          * the EFX_FILTER_MATCH flags match those used by MCDI, so they don't
1366          * need to be converted.
1367          *
1368          * In case support is added to MCDI for additional flags, remove any
1369          * matches from the list which include flags we don't support. The order
1370          * of the matches is preserved as they are ordered from highest to
1371          * lowest priority.
1372          */
1373         EFSYS_ASSERT(mcdi_list_length + mcdi_encap_list_length <=
1374             buffer_length);
1375         list_length = 0;
1376         for (i = 0; i < mcdi_list_length + mcdi_encap_list_length; i++) {
1377                 if ((buffer[i] & ~all_filter_flags) == 0) {
1378                         buffer[list_length] = buffer[i];
1379                         list_length++;
1380                 }
1381         }
1382
1383         *list_lengthp = list_length;
1384
1385         return (0);
1386
1387 fail3:
1388         EFSYS_PROBE(fail3);
1389 fail2:
1390         EFSYS_PROBE(fail2);
1391 fail1:
1392         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1393
1394         return (rc);
1395 }
1396
1397 static  __checkReturn   efx_rc_t
1398 ef10_filter_insert_unicast(
1399         __in                            efx_nic_t *enp,
1400         __in_ecount(6)                  uint8_t const *addr,
1401         __in                            efx_filter_flags_t filter_flags)
1402 {
1403         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
1404         efx_filter_spec_t spec;
1405         efx_rc_t rc;
1406
1407         /* Insert the filter for the local station address */
1408         efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1409             filter_flags,
1410             eftp->eft_default_rxq);
1411         rc = efx_filter_spec_set_eth_local(&spec, EFX_FILTER_SPEC_VID_UNSPEC,
1412             addr);
1413         if (rc != 0)
1414                 goto fail1;
1415
1416         rc = ef10_filter_add_internal(enp, &spec, EFX_FILTER_REPLACEMENT_NEVER,
1417             &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]);
1418         if (rc != 0)
1419                 goto fail2;
1420
1421         eftp->eft_unicst_filter_count++;
1422         EFSYS_ASSERT(eftp->eft_unicst_filter_count <=
1423                     EFX_EF10_FILTER_UNICAST_FILTERS_MAX);
1424
1425         return (0);
1426
1427 fail2:
1428         EFSYS_PROBE(fail2);
1429 fail1:
1430         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1431         return (rc);
1432 }
1433
1434 static  __checkReturn   efx_rc_t
1435 ef10_filter_insert_all_unicast(
1436         __in                            efx_nic_t *enp,
1437         __in                            efx_filter_flags_t filter_flags)
1438 {
1439         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
1440         efx_filter_spec_t spec;
1441         efx_rc_t rc;
1442
1443         /* Insert the unknown unicast filter */
1444         efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1445             filter_flags,
1446             eftp->eft_default_rxq);
1447         rc = efx_filter_spec_set_uc_def(&spec);
1448         if (rc != 0)
1449                 goto fail1;
1450         rc = ef10_filter_add_internal(enp, &spec, EFX_FILTER_REPLACEMENT_NEVER,
1451             &eftp->eft_unicst_filter_indexes[eftp->eft_unicst_filter_count]);
1452         if (rc != 0)
1453                 goto fail2;
1454
1455         eftp->eft_unicst_filter_count++;
1456         EFSYS_ASSERT(eftp->eft_unicst_filter_count <=
1457                     EFX_EF10_FILTER_UNICAST_FILTERS_MAX);
1458
1459         return (0);
1460
1461 fail2:
1462         EFSYS_PROBE(fail2);
1463 fail1:
1464         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1465         return (rc);
1466 }
1467
1468 static  __checkReturn   efx_rc_t
1469 ef10_filter_insert_multicast_list(
1470         __in                            efx_nic_t *enp,
1471         __in                            boolean_t mulcst,
1472         __in                            boolean_t brdcst,
1473         __in_ecount(6*count)            uint8_t const *addrs,
1474         __in                            uint32_t count,
1475         __in                            efx_filter_flags_t filter_flags,
1476         __in                            boolean_t rollback)
1477 {
1478         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
1479         efx_filter_spec_t spec;
1480         uint8_t addr[6];
1481         uint32_t i;
1482         uint32_t filter_index;
1483         uint32_t filter_count;
1484         efx_rc_t rc;
1485
1486         if (mulcst == B_FALSE)
1487                 count = 0;
1488
1489         if (count + (brdcst ? 1 : 0) >
1490             EFX_ARRAY_SIZE(eftp->eft_mulcst_filter_indexes)) {
1491                 /* Too many MAC addresses */
1492                 rc = EINVAL;
1493                 goto fail1;
1494         }
1495
1496         /* Insert/renew multicast address list filters */
1497         filter_count = 0;
1498         for (i = 0; i < count; i++) {
1499                 efx_filter_spec_init_rx(&spec,
1500                     EFX_FILTER_PRI_AUTO,
1501                     filter_flags,
1502                     eftp->eft_default_rxq);
1503
1504                 rc = efx_filter_spec_set_eth_local(&spec,
1505                     EFX_FILTER_SPEC_VID_UNSPEC,
1506                     &addrs[i * EFX_MAC_ADDR_LEN]);
1507                 if (rc != 0) {
1508                         if (rollback == B_TRUE) {
1509                                 /* Only stop upon failure if told to rollback */
1510                                 goto rollback;
1511                         } else {
1512                                 /*
1513                                  * Don't try to add a filter with a corrupt
1514                                  * specification.
1515                                  */
1516                                 continue;
1517                         }
1518                 }
1519
1520                 rc = ef10_filter_add_internal(enp, &spec,
1521                     EFX_FILTER_REPLACEMENT_NEVER, &filter_index);
1522
1523                 if (rc == 0) {
1524                         eftp->eft_mulcst_filter_indexes[filter_count] =
1525                                 filter_index;
1526                         filter_count++;
1527                 } else if (rollback == B_TRUE) {
1528                         /* Only stop upon failure if told to rollback */
1529                         goto rollback;
1530                 }
1531
1532         }
1533
1534         if (brdcst == B_TRUE) {
1535                 /* Insert/renew broadcast address filter */
1536                 efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1537                     filter_flags,
1538                     eftp->eft_default_rxq);
1539
1540                 EFX_MAC_BROADCAST_ADDR_SET(addr);
1541                 rc = efx_filter_spec_set_eth_local(&spec,
1542                     EFX_FILTER_SPEC_VID_UNSPEC, addr);
1543                 if ((rc != 0) && (rollback == B_TRUE)) {
1544                         /* Only stop upon failure if told to rollback */
1545                         goto rollback;
1546                 }
1547
1548                 rc = ef10_filter_add_internal(enp, &spec,
1549                     EFX_FILTER_REPLACEMENT_NEVER, &filter_index);
1550
1551                 if (rc == 0) {
1552                         eftp->eft_mulcst_filter_indexes[filter_count] =
1553                                 filter_index;
1554                         filter_count++;
1555                 } else if (rollback == B_TRUE) {
1556                         /* Only stop upon failure if told to rollback */
1557                         goto rollback;
1558                 }
1559         }
1560
1561         eftp->eft_mulcst_filter_count = filter_count;
1562         eftp->eft_using_all_mulcst = B_FALSE;
1563
1564         return (0);
1565
1566 rollback:
1567         /* Remove any filters we have inserted */
1568         i = filter_count;
1569         while (i--) {
1570                 ef10_filter_delete_auto(enp,
1571                     eftp->eft_mulcst_filter_indexes[i]);
1572         }
1573         eftp->eft_mulcst_filter_count = 0;
1574
1575 fail1:
1576         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1577
1578         return (rc);
1579 }
1580
1581 static  __checkReturn   efx_rc_t
1582 ef10_filter_insert_all_multicast(
1583         __in                            efx_nic_t *enp,
1584         __in                            efx_filter_flags_t filter_flags)
1585 {
1586         ef10_filter_table_t *eftp = enp->en_filter.ef_ef10_filter_table;
1587         efx_filter_spec_t spec;
1588         efx_rc_t rc;
1589
1590         /* Insert the unknown multicast filter */
1591         efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1592             filter_flags,
1593             eftp->eft_default_rxq);
1594         rc = efx_filter_spec_set_mc_def(&spec);
1595         if (rc != 0)
1596                 goto fail1;
1597
1598         rc = ef10_filter_add_internal(enp, &spec, EFX_FILTER_REPLACEMENT_NEVER,
1599             &eftp->eft_mulcst_filter_indexes[0]);
1600         if (rc != 0)
1601                 goto fail2;
1602
1603         eftp->eft_mulcst_filter_count = 1;
1604         eftp->eft_using_all_mulcst = B_TRUE;
1605
1606         /*
1607          * FIXME: If brdcst == B_FALSE, add a filter to drop broadcast traffic.
1608          */
1609
1610         return (0);
1611
1612 fail2:
1613         EFSYS_PROBE(fail2);
1614 fail1:
1615         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1616
1617         return (rc);
1618 }
1619
1620 typedef struct ef10_filter_encap_entry_s {
1621         uint16_t                ether_type;
1622         efx_tunnel_protocol_t   encap_type;
1623         uint32_t                inner_frame_match;
1624 } ef10_filter_encap_entry_t;
1625
1626 #define EF10_ENCAP_FILTER_ENTRY(ipv, encap_type, inner_frame_match)     \
1627         { EFX_ETHER_TYPE_##ipv, EFX_TUNNEL_PROTOCOL_##encap_type,       \
1628             EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_##inner_frame_match }
1629
1630 static ef10_filter_encap_entry_t ef10_filter_encap_list[] = {
1631         EF10_ENCAP_FILTER_ENTRY(IPV4, VXLAN, UCAST_DST),
1632         EF10_ENCAP_FILTER_ENTRY(IPV4, VXLAN, MCAST_DST),
1633         EF10_ENCAP_FILTER_ENTRY(IPV6, VXLAN, UCAST_DST),
1634         EF10_ENCAP_FILTER_ENTRY(IPV6, VXLAN, MCAST_DST),
1635
1636         EF10_ENCAP_FILTER_ENTRY(IPV4, GENEVE, UCAST_DST),
1637         EF10_ENCAP_FILTER_ENTRY(IPV4, GENEVE, MCAST_DST),
1638         EF10_ENCAP_FILTER_ENTRY(IPV6, GENEVE, UCAST_DST),
1639         EF10_ENCAP_FILTER_ENTRY(IPV6, GENEVE, MCAST_DST),
1640
1641         EF10_ENCAP_FILTER_ENTRY(IPV4, NVGRE, UCAST_DST),
1642         EF10_ENCAP_FILTER_ENTRY(IPV4, NVGRE, MCAST_DST),
1643         EF10_ENCAP_FILTER_ENTRY(IPV6, NVGRE, UCAST_DST),
1644         EF10_ENCAP_FILTER_ENTRY(IPV6, NVGRE, MCAST_DST),
1645 };
1646
1647 #undef EF10_ENCAP_FILTER_ENTRY
1648
1649 static  __checkReturn   efx_rc_t
1650 ef10_filter_insert_encap_filters(
1651         __in            efx_nic_t *enp,
1652         __in            boolean_t mulcst,
1653         __in            efx_filter_flags_t filter_flags)
1654 {
1655         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1656         uint32_t i;
1657         efx_rc_t rc;
1658
1659         EFX_STATIC_ASSERT(EFX_ARRAY_SIZE(ef10_filter_encap_list) <=
1660                             EFX_ARRAY_SIZE(table->eft_encap_filter_indexes));
1661
1662         /*
1663          * On Medford, full-featured firmware can identify packets as being
1664          * tunnel encapsulated, even if no encapsulated packet offloads are in
1665          * use. When packets are identified as such, ordinary filters are not
1666          * applied, only ones specific to encapsulated packets. Hence we need to
1667          * insert filters for encapsulated packets in order to receive them.
1668          *
1669          * Separate filters need to be inserted for each ether type,
1670          * encapsulation type, and inner frame type (unicast or multicast). To
1671          * keep things simple and reduce the number of filters needed, catch-all
1672          * filters for all combinations of types are inserted, even if
1673          * all_unicst or all_mulcst have not been set. (These catch-all filters
1674          * may well, however, fail to insert on unprivileged functions.)
1675          */
1676         table->eft_encap_filter_count = 0;
1677         for (i = 0; i < EFX_ARRAY_SIZE(ef10_filter_encap_list); i++) {
1678                 efx_filter_spec_t spec;
1679                 ef10_filter_encap_entry_t *encap_filter =
1680                         &ef10_filter_encap_list[i];
1681
1682                 /*
1683                  * Skip multicast filters if we've not been asked for
1684                  * any multicast traffic.
1685                  */
1686                 if ((mulcst == B_FALSE) &&
1687                     (encap_filter->inner_frame_match ==
1688                     EFX_FILTER_INNER_FRAME_MATCH_UNKNOWN_MCAST_DST))
1689                         continue;
1690
1691                 efx_filter_spec_init_rx(&spec, EFX_FILTER_PRI_AUTO,
1692                                         filter_flags,
1693                                         table->eft_default_rxq);
1694                 efx_filter_spec_set_ether_type(&spec, encap_filter->ether_type);
1695                 rc = efx_filter_spec_set_encap_type(&spec,
1696                                             encap_filter->encap_type,
1697                                             encap_filter->inner_frame_match);
1698                 if (rc != 0)
1699                         goto fail1;
1700
1701                 rc = ef10_filter_add_internal(enp, &spec,
1702                     EFX_FILTER_REPLACEMENT_NEVER,
1703                     &table->eft_encap_filter_indexes[
1704                                     table->eft_encap_filter_count]);
1705                 if (rc != 0) {
1706                         if (rc != EACCES)
1707                                 goto fail2;
1708                 } else {
1709                         table->eft_encap_filter_count++;
1710                 }
1711         }
1712
1713         return (0);
1714
1715 fail2:
1716         EFSYS_PROBE(fail2);
1717 fail1:
1718         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1719
1720         return (rc);
1721 }
1722
1723 static                  void
1724 ef10_filter_remove_old(
1725         __in            efx_nic_t *enp)
1726 {
1727         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1728         uint32_t i;
1729
1730         for (i = 0; i < EFX_ARRAY_SIZE(table->eft_entry); i++) {
1731                 if (ef10_filter_entry_is_auto_old(table, i)) {
1732                         ef10_filter_delete_auto(enp, i);
1733                 }
1734         }
1735 }
1736
1737
1738 static  __checkReturn   efx_rc_t
1739 ef10_filter_get_workarounds(
1740         __in                            efx_nic_t *enp)
1741 {
1742         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
1743         uint32_t implemented = 0;
1744         uint32_t enabled = 0;
1745         efx_rc_t rc;
1746
1747         rc = efx_mcdi_get_workarounds(enp, &implemented, &enabled);
1748         if (rc == 0) {
1749                 /* Check if chained multicast filter support is enabled */
1750                 if (implemented & enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807)
1751                         encp->enc_bug26807_workaround = B_TRUE;
1752                 else
1753                         encp->enc_bug26807_workaround = B_FALSE;
1754         } else if (rc == ENOTSUP) {
1755                 /*
1756                  * Firmware is too old to support GET_WORKAROUNDS, and support
1757                  * for this workaround was implemented later.
1758                  */
1759                 encp->enc_bug26807_workaround = B_FALSE;
1760         } else {
1761                 goto fail1;
1762         }
1763
1764         return (0);
1765
1766 fail1:
1767         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1768
1769         return (rc);
1770
1771 }
1772
1773 static                  void
1774 ef10_filter_remove_all_existing_filters(
1775         __in                            efx_nic_t *enp)
1776 {
1777         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1778         efx_port_t *epp = &(enp->en_port);
1779         unsigned int i;
1780
1781         for (i = 0; i < table->eft_unicst_filter_count; i++) {
1782                 ef10_filter_delete_auto(enp,
1783                                 table->eft_unicst_filter_indexes[i]);
1784         }
1785         table->eft_unicst_filter_count = 0;
1786
1787         for (i = 0; i < table->eft_mulcst_filter_count; i++) {
1788                 ef10_filter_delete_auto(enp,
1789                                 table->eft_mulcst_filter_indexes[i]);
1790         }
1791         table->eft_mulcst_filter_count = 0;
1792
1793         for (i = 0; i < table->eft_encap_filter_count; i++) {
1794                 ef10_filter_delete_auto(enp,
1795                                 table->eft_encap_filter_indexes[i]);
1796         }
1797         table->eft_encap_filter_count = 0;
1798
1799         epp->ep_all_unicst_inserted = B_FALSE;
1800         epp->ep_all_mulcst_inserted = B_FALSE;
1801 }
1802
1803 static                  void
1804 ef10_filter_mark_old_filters(
1805         __in                            efx_nic_t *enp)
1806 {
1807         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1808         unsigned int i;
1809
1810         for (i = 0; i < table->eft_unicst_filter_count; i++) {
1811                 ef10_filter_set_entry_auto_old(table,
1812                                         table->eft_unicst_filter_indexes[i]);
1813         }
1814         for (i = 0; i < table->eft_mulcst_filter_count; i++) {
1815                 ef10_filter_set_entry_auto_old(table,
1816                                         table->eft_mulcst_filter_indexes[i]);
1817         }
1818         for (i = 0; i < table->eft_encap_filter_count; i++) {
1819                 ef10_filter_set_entry_auto_old(table,
1820                                         table->eft_encap_filter_indexes[i]);
1821         }
1822 }
1823
1824 static  __checkReturn   efx_rc_t
1825 ef10_filter_insert_renew_unicst_filters(
1826         __in                            efx_nic_t *enp,
1827         __in_ecount(6)                  uint8_t const *mac_addr,
1828         __in                            boolean_t all_unicst,
1829         __in                            efx_filter_flags_t filter_flags,
1830         __out                           boolean_t *all_unicst_inserted)
1831 {
1832         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1833         efx_port_t *epp = &(enp->en_port);
1834         efx_rc_t rc;
1835
1836         /*
1837          * Firmware does not perform chaining on unicast filters. As traffic is
1838          * therefore only delivered to the first matching filter, we should
1839          * always insert the specific filter for our MAC address, to try and
1840          * ensure we get that traffic.
1841          *
1842          * (If the filter for our MAC address has already been inserted by
1843          * another function, we won't receive traffic sent to us, even if we
1844          * insert a unicast mismatch filter. To prevent traffic stealing, this
1845          * therefore relies on the privilege model only allowing functions to
1846          * insert filters for their own MAC address unless explicitly given
1847          * additional privileges by the user. This also means that, even on a
1848          * privileged function, inserting a unicast mismatch filter may not
1849          * catch all traffic in multi PCI function scenarios.)
1850          */
1851         table->eft_unicst_filter_count = 0;
1852         rc = ef10_filter_insert_unicast(enp, mac_addr, filter_flags);
1853         *all_unicst_inserted = B_FALSE;
1854         if (all_unicst || (rc != 0)) {
1855                 efx_rc_t all_unicst_rc;
1856
1857                 all_unicst_rc = ef10_filter_insert_all_unicast(enp,
1858                                                     filter_flags);
1859                 if (all_unicst_rc == 0) {
1860                         *all_unicst_inserted = B_TRUE;
1861                         epp->ep_all_unicst_inserted = B_TRUE;
1862                 } else if (rc != 0)
1863                         goto fail1;
1864         }
1865
1866         return (0);
1867
1868 fail1:
1869         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1870
1871         return (rc);
1872 }
1873
1874 static  __checkReturn   efx_rc_t
1875 ef10_filter_insert_renew_mulcst_filters(
1876         __in                            efx_nic_t *enp,
1877         __in                            boolean_t mulcst,
1878         __in                            boolean_t all_mulcst,
1879         __in                            boolean_t brdcst,
1880         __in_ecount(6*count)            uint8_t const *addrs,
1881         __in                            uint32_t count,
1882         __in                            efx_filter_flags_t filter_flags,
1883         __in                            boolean_t all_unicst_inserted,
1884         __out                           boolean_t *all_mulcst_inserted)
1885 {
1886         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1887         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
1888         efx_port_t *epp = &(enp->en_port);
1889         efx_rc_t rc;
1890
1891         *all_mulcst_inserted = B_FALSE;
1892
1893         if (all_mulcst == B_TRUE) {
1894                 efx_rc_t all_mulcst_rc;
1895
1896                 /*
1897                  * Insert the all multicast filter. If that fails, try to insert
1898                  * all of our multicast filters (but without rollback on
1899                  * failure).
1900                  */
1901                 all_mulcst_rc = ef10_filter_insert_all_multicast(enp,
1902                                                             filter_flags);
1903                 if (all_mulcst_rc == 0) {
1904                         epp->ep_all_mulcst_inserted = B_TRUE;
1905                         *all_mulcst_inserted = B_TRUE;
1906                 } else {
1907                         rc = ef10_filter_insert_multicast_list(enp, B_TRUE,
1908                             brdcst, addrs, count, filter_flags, B_FALSE);
1909                         if (rc != 0)
1910                                 goto fail1;
1911                 }
1912         } else {
1913                 /*
1914                  * Insert filters for multicast addresses.
1915                  * If any insertion fails, then rollback and try to insert the
1916                  * all multicast filter instead.
1917                  * If that also fails, try to insert all of the multicast
1918                  * filters (but without rollback on failure).
1919                  */
1920                 rc = ef10_filter_insert_multicast_list(enp, mulcst, brdcst,
1921                             addrs, count, filter_flags, B_TRUE);
1922                 if (rc != 0) {
1923                         if ((table->eft_using_all_mulcst == B_FALSE) &&
1924                             (encp->enc_bug26807_workaround == B_TRUE)) {
1925                                 /*
1926                                  * Multicast filter chaining is on, so remove
1927                                  * old filters before inserting the multicast
1928                                  * all filter to avoid duplicate delivery caused
1929                                  * by packets matching multiple filters.
1930                                  */
1931                                 ef10_filter_remove_old(enp);
1932                                 if (all_unicst_inserted == B_FALSE)
1933                                         epp->ep_all_unicst_inserted = B_FALSE;
1934                                 if (*all_mulcst_inserted == B_FALSE)
1935                                         epp->ep_all_mulcst_inserted = B_FALSE;
1936                         }
1937
1938                         rc = ef10_filter_insert_all_multicast(enp,
1939                                                             filter_flags);
1940                         if (rc == 0) {
1941                                 epp->ep_all_mulcst_inserted = B_TRUE;
1942                                 *all_mulcst_inserted = B_TRUE;
1943                         } else {
1944                                 rc = ef10_filter_insert_multicast_list(enp,
1945                                     mulcst, brdcst,
1946                                     addrs, count, filter_flags, B_FALSE);
1947                                 if (rc != 0)
1948                                         goto fail2;
1949                         }
1950                 }
1951         }
1952
1953         return (0);
1954
1955 fail2:
1956         EFSYS_PROBE1(fail2, efx_rc_t, rc);
1957
1958 fail1:
1959         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1960
1961         return (rc);
1962 }
1963
1964 /*
1965  * Reconfigure all filters.
1966  * If all_unicst and/or all mulcst filters cannot be applied then
1967  * return ENOTSUP (Note the filters for the specified addresses are
1968  * still applied in this case).
1969  */
1970         __checkReturn   efx_rc_t
1971 ef10_filter_reconfigure(
1972         __in                            efx_nic_t *enp,
1973         __in_ecount(6)                  uint8_t const *mac_addr,
1974         __in                            boolean_t all_unicst,
1975         __in                            boolean_t mulcst,
1976         __in                            boolean_t all_mulcst,
1977         __in                            boolean_t brdcst,
1978         __in_ecount(6*count)            uint8_t const *addrs,
1979         __in                            uint32_t count)
1980 {
1981         efx_nic_cfg_t *encp = &enp->en_nic_cfg;
1982         efx_port_t *epp = &(enp->en_port);
1983         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
1984         efx_filter_flags_t filter_flags;
1985         unsigned int i;
1986         boolean_t all_unicst_inserted = B_FALSE;
1987         boolean_t all_mulcst_inserted = B_FALSE;
1988         efx_rc_t rc;
1989
1990         if (table->eft_default_rxq == NULL) {
1991                 /*
1992                  * Filters direct traffic to the default RXQ, and so cannot be
1993                  * inserted until it is available. Any currently configured
1994                  * filters must be removed (ignore errors in case the MC
1995                  * has rebooted, which removes hardware filters).
1996                  */
1997                 ef10_filter_remove_all_existing_filters(enp);
1998                 return (0);
1999         }
2000
2001         if (table->eft_using_rss)
2002                 filter_flags = EFX_FILTER_FLAG_RX_RSS;
2003         else
2004                 filter_flags = 0;
2005
2006         /* Mark old filters which may need to be removed */
2007         ef10_filter_mark_old_filters(enp);
2008
2009         /* Insert or renew unicast filters */
2010         rc = ef10_filter_insert_renew_unicst_filters(enp, mac_addr, all_unicst,
2011                                                      filter_flags,
2012                                                      &all_unicst_inserted);
2013         if (rc != 0)
2014                 goto fail1;
2015
2016         /*
2017          * WORKAROUND_BUG26807 controls firmware support for chained multicast
2018          * filters, and can only be enabled or disabled when the hardware filter
2019          * table is empty.
2020          *
2021          * Chained multicast filters require support from the datapath firmware,
2022          * and may not be available (e.g. low-latency variants or old Huntington
2023          * firmware).
2024          *
2025          * Firmware will reset (FLR) functions which have inserted filters in
2026          * the hardware filter table when the workaround is enabled/disabled.
2027          * Functions without any hardware filters are not reset.
2028          *
2029          * Re-check if the workaround is enabled after adding unicast hardware
2030          * filters. This ensures that encp->enc_bug26807_workaround matches the
2031          * firmware state, and that later changes to enable/disable the
2032          * workaround will result in this function seeing a reset (FLR).
2033          *
2034          * In common-code drivers, we only support multiple PCI function
2035          * scenarios with firmware that supports multicast chaining, so we can
2036          * assume it is enabled for such cases and hence simplify the filter
2037          * insertion logic. Firmware that does not support multicast chaining
2038          * does not support multiple PCI function configurations either, so
2039          * filter insertion is much simpler and the same strategies can still be
2040          * used.
2041          */
2042         if ((rc = ef10_filter_get_workarounds(enp)) != 0)
2043                 goto fail2;
2044
2045         if ((table->eft_using_all_mulcst != all_mulcst) &&
2046             (encp->enc_bug26807_workaround == B_TRUE)) {
2047                 /*
2048                  * Multicast filter chaining is enabled, so traffic that matches
2049                  * more than one multicast filter will be replicated and
2050                  * delivered to multiple recipients.  To avoid this duplicate
2051                  * delivery, remove old multicast filters before inserting new
2052                  * multicast filters.
2053                  */
2054                 ef10_filter_remove_old(enp);
2055                 if (all_unicst_inserted == B_FALSE)
2056                         epp->ep_all_unicst_inserted = B_FALSE;
2057
2058                 epp->ep_all_mulcst_inserted = B_FALSE;
2059         }
2060
2061         /* Insert or renew multicast filters */
2062         rc = ef10_filter_insert_renew_mulcst_filters(enp, mulcst, all_mulcst,
2063                                                      brdcst, addrs, count,
2064                                                      filter_flags,
2065                                                      all_unicst_inserted,
2066                                                      &all_mulcst_inserted);
2067         if (rc != 0)
2068                 goto fail3;
2069
2070         if (encp->enc_tunnel_encapsulations_supported != 0) {
2071                 /* Try to insert filters for encapsulated packets. */
2072                 (void) ef10_filter_insert_encap_filters(enp,
2073                                             mulcst || all_mulcst || brdcst,
2074                                             filter_flags);
2075         }
2076
2077         /* Remove old filters which were not renewed */
2078         ef10_filter_remove_old(enp);
2079         if (all_unicst_inserted == B_FALSE)
2080                 epp->ep_all_unicst_inserted = B_FALSE;
2081         if (all_mulcst_inserted == B_FALSE)
2082                 epp->ep_all_mulcst_inserted = B_FALSE;
2083
2084         /* report if any optional flags were rejected */
2085         if (((all_unicst != B_FALSE) && (all_unicst_inserted == B_FALSE)) ||
2086             ((all_mulcst != B_FALSE) && (all_mulcst_inserted == B_FALSE))) {
2087                 rc = ENOTSUP;
2088         }
2089
2090         return (rc);
2091
2092 fail3:
2093         EFSYS_PROBE(fail3);
2094 fail2:
2095         EFSYS_PROBE(fail2);
2096 fail1:
2097         EFSYS_PROBE1(fail1, efx_rc_t, rc);
2098
2099         /* Clear auto old flags */
2100         for (i = 0; i < EFX_ARRAY_SIZE(table->eft_entry); i++) {
2101                 if (ef10_filter_entry_is_auto_old(table, i)) {
2102                         ef10_filter_set_entry_not_auto_old(table, i);
2103                 }
2104         }
2105
2106         return (rc);
2107 }
2108
2109                 void
2110 ef10_filter_get_default_rxq(
2111         __in            efx_nic_t *enp,
2112         __out           efx_rxq_t **erpp,
2113         __out           boolean_t *using_rss)
2114 {
2115         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
2116
2117         *erpp = table->eft_default_rxq;
2118         *using_rss = table->eft_using_rss;
2119 }
2120
2121
2122                 void
2123 ef10_filter_default_rxq_set(
2124         __in            efx_nic_t *enp,
2125         __in            efx_rxq_t *erp,
2126         __in            boolean_t using_rss)
2127 {
2128         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
2129
2130 #if EFSYS_OPT_RX_SCALE
2131         EFSYS_ASSERT((using_rss == B_FALSE) ||
2132             (enp->en_rss_context != EF10_RSS_CONTEXT_INVALID));
2133         table->eft_using_rss = using_rss;
2134 #else
2135         EFSYS_ASSERT(using_rss == B_FALSE);
2136         table->eft_using_rss = B_FALSE;
2137 #endif
2138         table->eft_default_rxq = erp;
2139 }
2140
2141                 void
2142 ef10_filter_default_rxq_clear(
2143         __in            efx_nic_t *enp)
2144 {
2145         ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table;
2146
2147         table->eft_default_rxq = NULL;
2148         table->eft_using_rss = B_FALSE;
2149 }
2150
2151
2152 #endif /* EFSYS_OPT_FILTER */
2153
2154 #endif /* EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() */