common/sfc_efx/base: update EF100 registers definitions
[dpdk.git] / drivers / common / sfc_efx / base / efx_mae.c
index 80fe155..c22206e 100644 (file)
@@ -67,6 +67,9 @@ efx_mae_get_capabilities(
        maep->em_max_nfields =
            MCDI_OUT_DWORD(req, MAE_GET_CAPS_OUT_MATCH_FIELD_COUNT);
 
+       maep->em_max_ncounters =
+           MCDI_OUT_DWORD(req, MAE_GET_CAPS_OUT_COUNTERS);
+
        return (0);
 
 fail2:
@@ -109,17 +112,22 @@ efx_mae_get_outer_rule_caps(
                goto fail2;
        }
 
+       if (req.emr_out_length_used < MC_CMD_MAE_GET_OR_CAPS_OUT_LENMIN) {
+               rc = EMSGSIZE;
+               goto fail3;
+       }
+
        mcdi_field_ncaps = MCDI_OUT_DWORD(req, MAE_GET_OR_CAPS_OUT_COUNT);
 
        if (req.emr_out_length_used <
            MC_CMD_MAE_GET_OR_CAPS_OUT_LEN(mcdi_field_ncaps)) {
                rc = EMSGSIZE;
-               goto fail3;
+               goto fail4;
        }
 
        if (mcdi_field_ncaps > field_ncaps) {
                rc = EMSGSIZE;
-               goto fail4;
+               goto fail5;
        }
 
        for (i = 0; i < mcdi_field_ncaps; ++i) {
@@ -147,6 +155,8 @@ efx_mae_get_outer_rule_caps(
 
        return (0);
 
+fail5:
+       EFSYS_PROBE(fail5);
 fail4:
        EFSYS_PROBE(fail4);
 fail3:
@@ -191,17 +201,22 @@ efx_mae_get_action_rule_caps(
                goto fail2;
        }
 
-       mcdi_field_ncaps = MCDI_OUT_DWORD(req, MAE_GET_OR_CAPS_OUT_COUNT);
+       if (req.emr_out_length_used < MC_CMD_MAE_GET_AR_CAPS_OUT_LENMIN) {
+               rc = EMSGSIZE;
+               goto fail3;
+       }
+
+       mcdi_field_ncaps = MCDI_OUT_DWORD(req, MAE_GET_AR_CAPS_OUT_COUNT);
 
        if (req.emr_out_length_used <
            MC_CMD_MAE_GET_AR_CAPS_OUT_LEN(mcdi_field_ncaps)) {
                rc = EMSGSIZE;
-               goto fail3;
+               goto fail4;
        }
 
        if (mcdi_field_ncaps > field_ncaps) {
                rc = EMSGSIZE;
-               goto fail4;
+               goto fail5;
        }
 
        for (i = 0; i < mcdi_field_ncaps; ++i) {
@@ -229,6 +244,8 @@ efx_mae_get_action_rule_caps(
 
        return (0);
 
+fail5:
+       EFSYS_PROBE(fail5);
 fail4:
        EFSYS_PROBE(fail4);
 fail3:
@@ -357,6 +374,7 @@ efx_mae_get_limits(
        emlp->eml_encap_types_supported = maep->em_encap_types_supported;
        emlp->eml_encap_header_size_limit =
            MC_CMD_MAE_ENCAP_HEADER_ALLOC_IN_HDR_DATA_MAXNUM_MCDI2;
+       emlp->eml_max_n_counters = maep->em_max_ncounters;
 
        return (0);
 
@@ -451,6 +469,10 @@ typedef enum efx_mae_field_cap_id_e {
        EFX_MAE_FIELD_ID_ENC_L4_DPORT_BE = MAE_FIELD_ENC_L4_DPORT,
        EFX_MAE_FIELD_ID_ENC_VNET_ID_BE = MAE_FIELD_ENC_VNET_ID,
        EFX_MAE_FIELD_ID_OUTER_RULE_ID = MAE_FIELD_OUTER_RULE_ID,
+       EFX_MAE_FIELD_ID_HAS_OVLAN = MAE_FIELD_HAS_OVLAN,
+       EFX_MAE_FIELD_ID_HAS_IVLAN = MAE_FIELD_HAS_IVLAN,
+       EFX_MAE_FIELD_ID_ENC_HAS_OVLAN = MAE_FIELD_ENC_HAS_OVLAN,
+       EFX_MAE_FIELD_ID_ENC_HAS_IVLAN = MAE_FIELD_ENC_HAS_IVLAN,
 
        EFX_MAE_FIELD_CAP_NIDS
 } efx_mae_field_cap_id_t;
@@ -577,6 +599,65 @@ static const efx_mae_mv_desc_t __efx_mae_outer_rule_mv_desc_set[] = {
 
 #undef EFX_MAE_MV_DESC_ALT
 #undef EFX_MAE_MV_DESC
+};
+
+/*
+ * The following structure is a means to describe an MAE bit.
+ * The information in it is meant to be used internally by
+ * APIs for addressing a given flag in a mask-value pairs
+ * structure and for validation purposes.
+ */
+typedef struct efx_mae_mv_bit_desc_s {
+       /*
+        * Arrays using this struct are indexed by field IDs.
+        * Fields which aren't meant to be referenced by these
+        * arrays comprise gaps (invalid entries). Below field
+        * helps to identify such entries.
+        */
+       boolean_t                       emmbd_entry_is_valid;
+       efx_mae_field_cap_id_t          emmbd_bit_cap_id;
+       size_t                          emmbd_value_ofst;
+       unsigned int                    emmbd_value_lbn;
+       size_t                          emmbd_mask_ofst;
+       unsigned int                    emmbd_mask_lbn;
+} efx_mae_mv_bit_desc_t;
+
+static const efx_mae_mv_bit_desc_t __efx_mae_outer_rule_mv_bit_desc_set[] = {
+#define        EFX_MAE_MV_BIT_DESC(_name)                                      \
+       [EFX_MAE_FIELD_##_name] =                                       \
+       {                                                               \
+               B_TRUE,                                                 \
+               EFX_MAE_FIELD_ID_##_name,                               \
+               MAE_ENC_FIELD_PAIRS_##_name##_OFST,                     \
+               MAE_ENC_FIELD_PAIRS_##_name##_LBN,                      \
+               MAE_ENC_FIELD_PAIRS_##_name##_MASK_OFST,                \
+               MAE_ENC_FIELD_PAIRS_##_name##_MASK_LBN,                 \
+       }
+
+       EFX_MAE_MV_BIT_DESC(ENC_HAS_OVLAN),
+       EFX_MAE_MV_BIT_DESC(ENC_HAS_IVLAN),
+
+#undef EFX_MAE_MV_BIT_DESC
+};
+
+static const efx_mae_mv_bit_desc_t __efx_mae_action_rule_mv_bit_desc_set[] = {
+#define        EFX_MAE_MV_BIT_DESC(_name)                                      \
+       [EFX_MAE_FIELD_##_name] =                                       \
+       {                                                               \
+               B_TRUE,                                                 \
+               EFX_MAE_FIELD_ID_##_name,                               \
+               MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_OFST,               \
+               MAE_FIELD_MASK_VALUE_PAIRS_V2_##_name##_LBN,            \
+               MAE_FIELD_MASK_VALUE_PAIRS_V2_FLAGS_MASK_OFST,          \
+               MAE_FIELD_MASK_VALUE_PAIRS_V2_##_name##_LBN,            \
+       }
+
+       EFX_MAE_MV_BIT_DESC(HAS_OVLAN),
+       EFX_MAE_MV_BIT_DESC(HAS_IVLAN),
+       EFX_MAE_MV_BIT_DESC(ENC_HAS_OVLAN),
+       EFX_MAE_MV_BIT_DESC(ENC_HAS_IVLAN),
+
+#undef EFX_MAE_MV_BIT_DESC
 };
 
        __checkReturn                   efx_rc_t
@@ -775,6 +856,70 @@ fail5:
        EFSYS_PROBE(fail5);
 fail4:
        EFSYS_PROBE(fail4);
+fail3:
+       EFSYS_PROBE(fail3);
+fail2:
+       EFSYS_PROBE(fail2);
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+       return (rc);
+}
+
+       __checkReturn                   efx_rc_t
+efx_mae_match_spec_bit_set(
+       __in                            efx_mae_match_spec_t *spec,
+       __in                            efx_mae_field_id_t field_id,
+       __in                            boolean_t value)
+{
+       const efx_mae_mv_bit_desc_t *bit_descp;
+       unsigned int bit_desc_set_nentries;
+       unsigned int byte_idx;
+       unsigned int bit_idx;
+       uint8_t *mvp;
+       efx_rc_t rc;
+
+       switch (spec->emms_type) {
+       case EFX_MAE_RULE_OUTER:
+               bit_desc_set_nentries =
+                   EFX_ARRAY_SIZE(__efx_mae_outer_rule_mv_bit_desc_set);
+               bit_descp = &__efx_mae_outer_rule_mv_bit_desc_set[field_id];
+               mvp = spec->emms_mask_value_pairs.outer;
+               break;
+       case EFX_MAE_RULE_ACTION:
+               bit_desc_set_nentries =
+                   EFX_ARRAY_SIZE(__efx_mae_action_rule_mv_bit_desc_set);
+               bit_descp = &__efx_mae_action_rule_mv_bit_desc_set[field_id];
+               mvp = spec->emms_mask_value_pairs.action;
+               break;
+       default:
+               rc = ENOTSUP;
+               goto fail1;
+       }
+
+       if ((unsigned int)field_id >= bit_desc_set_nentries) {
+               rc = EINVAL;
+               goto fail2;
+       }
+
+       if (bit_descp->emmbd_entry_is_valid == B_FALSE) {
+               rc = EINVAL;
+               goto fail3;
+       }
+
+       byte_idx = bit_descp->emmbd_value_ofst + bit_descp->emmbd_value_lbn / 8;
+       bit_idx = bit_descp->emmbd_value_lbn % 8;
+
+       if (value != B_FALSE)
+               mvp[byte_idx] |= (1U << bit_idx);
+       else
+               mvp[byte_idx] &= ~(1U << bit_idx);
+
+       byte_idx = bit_descp->emmbd_mask_ofst + bit_descp->emmbd_mask_lbn / 8;
+       bit_idx = bit_descp->emmbd_mask_lbn % 8;
+       mvp[byte_idx] |= (1U << bit_idx);
+
+       return (0);
+
 fail3:
        EFSYS_PROBE(fail3);
 fail2:
@@ -891,6 +1036,8 @@ efx_mae_match_spec_is_valid(
        const efx_mae_field_cap_t *field_caps;
        const efx_mae_mv_desc_t *desc_setp;
        unsigned int desc_set_nentries;
+       const efx_mae_mv_bit_desc_t *bit_desc_setp;
+       unsigned int bit_desc_set_nentries;
        boolean_t is_valid = B_TRUE;
        efx_mae_field_id_t field_id;
        const uint8_t *mvp;
@@ -901,6 +1048,9 @@ efx_mae_match_spec_is_valid(
                desc_setp = __efx_mae_outer_rule_mv_desc_set;
                desc_set_nentries =
                    EFX_ARRAY_SIZE(__efx_mae_outer_rule_mv_desc_set);
+               bit_desc_setp = __efx_mae_outer_rule_mv_bit_desc_set;
+               bit_desc_set_nentries =
+                   EFX_ARRAY_SIZE(__efx_mae_outer_rule_mv_bit_desc_set);
                mvp = spec->emms_mask_value_pairs.outer;
                break;
        case EFX_MAE_RULE_ACTION:
@@ -908,6 +1058,9 @@ efx_mae_match_spec_is_valid(
                desc_setp = __efx_mae_action_rule_mv_desc_set;
                desc_set_nentries =
                    EFX_ARRAY_SIZE(__efx_mae_action_rule_mv_desc_set);
+               bit_desc_setp = __efx_mae_action_rule_mv_bit_desc_set;
+               bit_desc_set_nentries =
+                   EFX_ARRAY_SIZE(__efx_mae_action_rule_mv_bit_desc_set);
                mvp = spec->emms_mask_value_pairs.action;
                break;
        default:
@@ -975,6 +1128,48 @@ efx_mae_match_spec_is_valid(
                        break;
                }
 
+               if (is_valid == B_FALSE)
+                       return (B_FALSE);
+       }
+
+       for (field_id = 0; (unsigned int)field_id < bit_desc_set_nentries;
+            ++field_id) {
+               const efx_mae_mv_bit_desc_t *bit_descp =
+                   &bit_desc_setp[field_id];
+               unsigned int byte_idx =
+                   bit_descp->emmbd_mask_ofst +
+                   bit_descp->emmbd_mask_lbn / 8;
+               unsigned int bit_idx =
+                   bit_descp->emmbd_mask_lbn % 8;
+               efx_mae_field_cap_id_t bit_cap_id =
+                   bit_descp->emmbd_bit_cap_id;
+
+               if (bit_descp->emmbd_entry_is_valid == B_FALSE)
+                       continue; /* Skip array gap */
+
+               if ((unsigned int)bit_cap_id >= field_ncaps) {
+                       /* No capability for this bit = unsupported. */
+                       is_valid = ((mvp[byte_idx] & (1U << bit_idx)) == 0);
+                       if (is_valid == B_FALSE)
+                               break;
+                       else
+                               continue;
+               }
+
+               switch (field_caps[bit_cap_id].emfc_support) {
+               case MAE_FIELD_SUPPORTED_MATCH_OPTIONAL:
+                       is_valid = B_TRUE;
+                       break;
+               case MAE_FIELD_SUPPORTED_MATCH_ALWAYS:
+                       is_valid = ((mvp[byte_idx] & (1U << bit_idx)) != 0);
+                       break;
+               case MAE_FIELD_SUPPORTED_MATCH_NEVER:
+               case MAE_FIELD_UNSUPPORTED:
+               default:
+                       is_valid = ((mvp[byte_idx] & (1U << bit_idx)) == 0);
+                       break;
+               }
+
                if (is_valid == B_FALSE)
                        break;
        }
@@ -997,6 +1192,7 @@ efx_mae_action_set_spec_init(
        }
 
        spec->ema_rsrc.emar_eh_id.id = EFX_MAE_RSRC_ID_INVALID;
+       spec->ema_rsrc.emar_counter_id.id = EFX_MAE_RSRC_ID_INVALID;
 
        *specp = spec;
 
@@ -1164,6 +1360,50 @@ fail1:
        return (rc);
 }
 
+static __checkReturn                   efx_rc_t
+efx_mae_action_set_add_count(
+       __in                            efx_mae_actions_t *spec,
+       __in                            size_t arg_size,
+       __in_bcount(arg_size)           const uint8_t *arg)
+{
+       efx_rc_t rc;
+
+       EFX_STATIC_ASSERT(EFX_MAE_RSRC_ID_INVALID ==
+                         MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_NULL);
+
+       /*
+        * Preparing an action set spec to update a counter requires
+        * two steps: first add this action to the action spec, and then
+        * add the counter ID to the spec. This allows validity checking
+        * and resource allocation to be done separately.
+        * Mark the counter ID as invalid in the spec to ensure that the
+        * caller must also invoke efx_mae_action_set_fill_in_counter_id()
+        * before action set allocation.
+        */
+       spec->ema_rsrc.emar_counter_id.id = EFX_MAE_RSRC_ID_INVALID;
+
+       /* Nothing else is supposed to take place over here. */
+       if (arg_size != 0) {
+               rc = EINVAL;
+               goto fail1;
+       }
+
+       if (arg != NULL) {
+               rc = EINVAL;
+               goto fail2;
+       }
+
+       ++(spec->ema_n_count_actions);
+
+       return (0);
+
+fail2:
+       EFSYS_PROBE(fail2);
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+       return (rc);
+}
+
 static __checkReturn                   efx_rc_t
 efx_mae_action_set_add_flag(
        __in                            efx_mae_actions_t *spec,
@@ -1272,6 +1512,9 @@ static const efx_mae_action_desc_t efx_mae_actions[EFX_MAE_NACTIONS] = {
        [EFX_MAE_ACTION_ENCAP] = {
                .emad_add = efx_mae_action_set_add_encap
        },
+       [EFX_MAE_ACTION_COUNT] = {
+               .emad_add = efx_mae_action_set_add_count
+       },
        [EFX_MAE_ACTION_FLAG] = {
                .emad_add = efx_mae_action_set_add_flag
        },
@@ -1287,6 +1530,12 @@ static const uint32_t efx_mae_action_ordered_map =
        (1U << EFX_MAE_ACTION_DECAP) |
        (1U << EFX_MAE_ACTION_VLAN_POP) |
        (1U << EFX_MAE_ACTION_VLAN_PUSH) |
+       /*
+        * HW will conduct action COUNT after
+        * the matching packet has been modified by
+        * length-affecting actions except for ENCAP.
+        */
+       (1U << EFX_MAE_ACTION_COUNT) |
        (1U << EFX_MAE_ACTION_ENCAP) |
        (1U << EFX_MAE_ACTION_FLAG) |
        (1U << EFX_MAE_ACTION_MARK) |
@@ -1298,12 +1547,14 @@ static const uint32_t efx_mae_action_ordered_map =
  * strictly ordered actions.
  */
 static const uint32_t efx_mae_action_nonstrict_map =
+       (1U << EFX_MAE_ACTION_COUNT) |
        (1U << EFX_MAE_ACTION_FLAG) |
        (1U << EFX_MAE_ACTION_MARK);
 
 static const uint32_t efx_mae_action_repeat_map =
        (1U << EFX_MAE_ACTION_VLAN_POP) |
-       (1U << EFX_MAE_ACTION_VLAN_PUSH);
+       (1U << EFX_MAE_ACTION_VLAN_PUSH) |
+       (1U << EFX_MAE_ACTION_COUNT);
 
 /*
  * Add an action to an action set.
@@ -1426,6 +1677,20 @@ efx_mae_action_set_populate_encap(
            EFX_MAE_ACTION_ENCAP, 0, NULL));
 }
 
+       __checkReturn                   efx_rc_t
+efx_mae_action_set_populate_count(
+       __in                            efx_mae_actions_t *spec)
+{
+       /*
+        * There is no argument to pass counter ID, thus, one does not
+        * need to allocate a counter while parsing application input.
+        * This is useful since building an action set may be done simply to
+        * validate a rule, whilst resource allocation usually consumes time.
+        */
+       return (efx_mae_action_set_spec_populate(spec,
+           EFX_MAE_ACTION_COUNT, 0, NULL));
+}
+
        __checkReturn                   efx_rc_t
 efx_mae_action_set_populate_flag(
        __in                            efx_mae_actions_t *spec)
@@ -1528,6 +1793,8 @@ efx_mae_match_specs_class_cmp(
        const efx_mae_field_cap_t *field_caps;
        const efx_mae_mv_desc_t *desc_setp;
        unsigned int desc_set_nentries;
+       const efx_mae_mv_bit_desc_t *bit_desc_setp;
+       unsigned int bit_desc_set_nentries;
        boolean_t have_same_class = B_TRUE;
        efx_mae_field_id_t field_id;
        const uint8_t *mvpl;
@@ -1540,6 +1807,9 @@ efx_mae_match_specs_class_cmp(
                desc_setp = __efx_mae_outer_rule_mv_desc_set;
                desc_set_nentries =
                    EFX_ARRAY_SIZE(__efx_mae_outer_rule_mv_desc_set);
+               bit_desc_setp = __efx_mae_outer_rule_mv_bit_desc_set;
+               bit_desc_set_nentries =
+                   EFX_ARRAY_SIZE(__efx_mae_outer_rule_mv_bit_desc_set);
                mvpl = left->emms_mask_value_pairs.outer;
                mvpr = right->emms_mask_value_pairs.outer;
                break;
@@ -1548,6 +1818,9 @@ efx_mae_match_specs_class_cmp(
                desc_setp = __efx_mae_action_rule_mv_desc_set;
                desc_set_nentries =
                    EFX_ARRAY_SIZE(__efx_mae_action_rule_mv_desc_set);
+               bit_desc_setp = __efx_mae_action_rule_mv_bit_desc_set;
+               bit_desc_set_nentries =
+                   EFX_ARRAY_SIZE(__efx_mae_action_rule_mv_bit_desc_set);
                mvpl = left->emms_mask_value_pairs.action;
                mvpr = right->emms_mask_value_pairs.action;
                break;
@@ -1620,6 +1893,52 @@ efx_mae_match_specs_class_cmp(
                }
        }
 
+       if (have_same_class == B_FALSE)
+               goto done;
+
+       for (field_id = 0; (unsigned int)field_id < bit_desc_set_nentries;
+            ++field_id) {
+               const efx_mae_mv_bit_desc_t *bit_descp =
+                   &bit_desc_setp[field_id];
+               efx_mae_field_cap_id_t bit_cap_id =
+                   bit_descp->emmbd_bit_cap_id;
+               unsigned int byte_idx;
+               unsigned int bit_idx;
+
+               if (bit_descp->emmbd_entry_is_valid == B_FALSE)
+                       continue; /* Skip array gap */
+
+               if ((unsigned int)bit_cap_id >= field_ncaps)
+                       break;
+
+               byte_idx =
+                   bit_descp->emmbd_mask_ofst +
+                   bit_descp->emmbd_mask_lbn / 8;
+               bit_idx =
+                   bit_descp->emmbd_mask_lbn % 8;
+
+               if (field_caps[bit_cap_id].emfc_mask_affects_class &&
+                   (mvpl[byte_idx] & (1U << bit_idx)) !=
+                   (mvpr[byte_idx] & (1U << bit_idx))) {
+                       have_same_class = B_FALSE;
+                       break;
+               }
+
+               byte_idx =
+                   bit_descp->emmbd_value_ofst +
+                   bit_descp->emmbd_value_lbn / 8;
+               bit_idx =
+                   bit_descp->emmbd_value_lbn % 8;
+
+               if (field_caps[bit_cap_id].emfc_match_affects_class &&
+                   (mvpl[byte_idx] & (1U << bit_idx)) !=
+                   (mvpr[byte_idx] & (1U << bit_idx))) {
+                       have_same_class = B_FALSE;
+                       break;
+               }
+       }
+
+done:
        *have_same_classp = have_same_class;
 
        return (0);
@@ -1773,15 +2092,22 @@ efx_mae_outer_rule_remove(
                goto fail2;
        }
 
+       if (req.emr_out_length_used < MC_CMD_MAE_OUTER_RULE_REMOVE_OUT_LENMIN) {
+               rc = EMSGSIZE;
+               goto fail3;
+       }
+
        if (MCDI_OUT_DWORD(req, MAE_OUTER_RULE_REMOVE_OUT_REMOVED_OR_ID) !=
            or_idp->id) {
                /* Firmware failed to remove the outer rule. */
                rc = EAGAIN;
-               goto fail3;
+               goto fail4;
        }
 
        return (0);
 
+fail4:
+       EFSYS_PROBE(fail4);
 fail3:
        EFSYS_PROBE(fail3);
 fail2:
@@ -2051,8 +2377,6 @@ efx_mae_action_set_alloc(
         */
        MCDI_IN_SET_DWORD(req,
            MAE_ACTION_SET_ALLOC_IN_COUNTER_LIST_ID, EFX_MAE_RSRC_ID_INVALID);
-       MCDI_IN_SET_DWORD(req,
-           MAE_ACTION_SET_ALLOC_IN_COUNTER_ID, EFX_MAE_RSRC_ID_INVALID);
 
        if ((spec->ema_actions & (1U << EFX_MAE_ACTION_DECAP)) != 0) {
                MCDI_IN_SET_DWORD_FIELD(req, MAE_ACTION_SET_ALLOC_IN_FLAGS,
@@ -2089,6 +2413,8 @@ efx_mae_action_set_alloc(
 
        MCDI_IN_SET_DWORD(req, MAE_ACTION_SET_ALLOC_IN_ENCAP_HEADER_ID,
            spec->ema_rsrc.emar_eh_id.id);
+       MCDI_IN_SET_DWORD(req, MAE_ACTION_SET_ALLOC_IN_COUNTER_ID,
+           spec->ema_rsrc.emar_counter_id.id);
 
        if ((spec->ema_actions & (1U << EFX_MAE_ACTION_FLAG)) != 0) {
                MCDI_IN_SET_DWORD_FIELD(req, MAE_ACTION_SET_ALLOC_IN_FLAGS,
@@ -2144,6 +2470,357 @@ fail1:
        return (rc);
 }
 
+       __checkReturn                   unsigned int
+efx_mae_action_set_get_nb_count(
+       __in                            const efx_mae_actions_t *spec)
+{
+       return (spec->ema_n_count_actions);
+}
+
+       __checkReturn                   efx_rc_t
+efx_mae_action_set_fill_in_counter_id(
+       __in                            efx_mae_actions_t *spec,
+       __in                            const efx_counter_t *counter_idp)
+{
+       efx_rc_t rc;
+
+       if ((spec->ema_actions & (1U << EFX_MAE_ACTION_COUNT)) == 0) {
+               /*
+                * Invalid to add counter ID if spec does not have COUNT action.
+                */
+               rc = EINVAL;
+               goto fail1;
+       }
+
+       if (spec->ema_n_count_actions != 1) {
+               /*
+                * Having multiple COUNT actions in the spec requires a counter
+                * list to be used. This API must only be used for a single
+                * counter per spec. Turn down the request as inappropriate.
+                */
+               rc = EINVAL;
+               goto fail2;
+       }
+
+       if (spec->ema_rsrc.emar_counter_id.id != EFX_MAE_RSRC_ID_INVALID) {
+               /* The caller attempts to indicate counter ID twice. */
+               rc = EALREADY;
+               goto fail3;
+       }
+
+       if (counter_idp->id == EFX_MAE_RSRC_ID_INVALID) {
+               rc = EINVAL;
+               goto fail4;
+       }
+
+       spec->ema_rsrc.emar_counter_id.id = counter_idp->id;
+
+       return (0);
+
+fail4:
+       EFSYS_PROBE(fail4);
+fail3:
+       EFSYS_PROBE(fail3);
+fail2:
+       EFSYS_PROBE(fail2);
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+       return (rc);
+}
+
+       __checkReturn                   efx_rc_t
+efx_mae_counters_alloc(
+       __in                            efx_nic_t *enp,
+       __in                            uint32_t n_counters,
+       __out                           uint32_t *n_allocatedp,
+       __out_ecount(n_counters)        efx_counter_t *countersp,
+       __out_opt                       uint32_t *gen_countp)
+{
+       EFX_MCDI_DECLARE_BUF(payload,
+           MC_CMD_MAE_COUNTER_ALLOC_IN_LEN,
+           MC_CMD_MAE_COUNTER_ALLOC_OUT_LENMAX_MCDI2);
+       efx_mae_t *maep = enp->en_maep;
+       uint32_t n_allocated;
+       efx_mcdi_req_t req;
+       unsigned int i;
+       efx_rc_t rc;
+
+       if (n_counters > maep->em_max_ncounters ||
+           n_counters < MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_MINNUM ||
+           n_counters > MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_MAXNUM_MCDI2) {
+               rc = EINVAL;
+               goto fail1;
+       }
+
+       req.emr_cmd = MC_CMD_MAE_COUNTER_ALLOC;
+       req.emr_in_buf = payload;
+       req.emr_in_length = MC_CMD_MAE_COUNTER_ALLOC_IN_LEN;
+       req.emr_out_buf = payload;
+       req.emr_out_length = MC_CMD_MAE_COUNTER_ALLOC_OUT_LEN(n_counters);
+
+       MCDI_IN_SET_DWORD(req, MAE_COUNTER_ALLOC_IN_REQUESTED_COUNT,
+           n_counters);
+
+       efx_mcdi_execute(enp, &req);
+
+       if (req.emr_rc != 0) {
+               rc = req.emr_rc;
+               goto fail2;
+       }
+
+       if (req.emr_out_length_used < MC_CMD_MAE_COUNTER_ALLOC_OUT_LENMIN) {
+               rc = EMSGSIZE;
+               goto fail3;
+       }
+
+       n_allocated = MCDI_OUT_DWORD(req,
+           MAE_COUNTER_ALLOC_OUT_COUNTER_ID_COUNT);
+       if (n_allocated < MC_CMD_MAE_COUNTER_ALLOC_OUT_COUNTER_ID_MINNUM) {
+               rc = EFAULT;
+               goto fail4;
+       }
+
+       for (i = 0; i < n_allocated; i++) {
+               countersp[i].id = MCDI_OUT_INDEXED_DWORD(req,
+                   MAE_COUNTER_ALLOC_OUT_COUNTER_ID, i);
+       }
+
+       if (gen_countp != NULL) {
+               *gen_countp = MCDI_OUT_DWORD(req,
+                                   MAE_COUNTER_ALLOC_OUT_GENERATION_COUNT);
+       }
+
+       *n_allocatedp = n_allocated;
+
+       return (0);
+
+fail4:
+       EFSYS_PROBE(fail4);
+fail3:
+       EFSYS_PROBE(fail3);
+fail2:
+       EFSYS_PROBE(fail2);
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+       return (rc);
+}
+
+       __checkReturn                   efx_rc_t
+efx_mae_counters_free(
+       __in                            efx_nic_t *enp,
+       __in                            uint32_t n_counters,
+       __out                           uint32_t *n_freedp,
+       __in_ecount(n_counters)         const efx_counter_t *countersp,
+       __out_opt                       uint32_t *gen_countp)
+{
+       EFX_MCDI_DECLARE_BUF(payload,
+           MC_CMD_MAE_COUNTER_FREE_IN_LENMAX_MCDI2,
+           MC_CMD_MAE_COUNTER_FREE_OUT_LENMAX_MCDI2);
+       efx_mae_t *maep = enp->en_maep;
+       efx_mcdi_req_t req;
+       uint32_t n_freed;
+       unsigned int i;
+       efx_rc_t rc;
+
+       if (n_counters > maep->em_max_ncounters ||
+           n_counters < MC_CMD_MAE_COUNTER_FREE_IN_FREE_COUNTER_ID_MINNUM ||
+           n_counters >
+           MC_CMD_MAE_COUNTER_FREE_IN_FREE_COUNTER_ID_MAXNUM_MCDI2) {
+               rc = EINVAL;
+               goto fail1;
+       }
+
+       req.emr_cmd = MC_CMD_MAE_COUNTER_FREE;
+       req.emr_in_buf = payload;
+       req.emr_in_length = MC_CMD_MAE_COUNTER_FREE_IN_LEN(n_counters);
+       req.emr_out_buf = payload;
+       req.emr_out_length = MC_CMD_MAE_COUNTER_FREE_OUT_LEN(n_counters);
+
+       for (i = 0; i < n_counters; i++) {
+               MCDI_IN_SET_INDEXED_DWORD(req,
+                   MAE_COUNTER_FREE_IN_FREE_COUNTER_ID, i, countersp[i].id);
+       }
+       MCDI_IN_SET_DWORD(req, MAE_COUNTER_FREE_IN_COUNTER_ID_COUNT,
+                         n_counters);
+
+       efx_mcdi_execute(enp, &req);
+
+       if (req.emr_rc != 0) {
+               rc = req.emr_rc;
+               goto fail2;
+       }
+
+       if (req.emr_out_length_used < MC_CMD_MAE_COUNTER_FREE_OUT_LENMIN) {
+               rc = EMSGSIZE;
+               goto fail3;
+       }
+
+       n_freed = MCDI_OUT_DWORD(req, MAE_COUNTER_FREE_OUT_COUNTER_ID_COUNT);
+
+       if (n_freed < MC_CMD_MAE_COUNTER_FREE_OUT_FREED_COUNTER_ID_MINNUM) {
+               rc = EFAULT;
+               goto fail4;
+       }
+
+       if (gen_countp != NULL) {
+               *gen_countp = MCDI_OUT_DWORD(req,
+                                   MAE_COUNTER_FREE_OUT_GENERATION_COUNT);
+       }
+
+       *n_freedp = n_freed;
+
+       return (0);
+
+fail4:
+       EFSYS_PROBE(fail4);
+fail3:
+       EFSYS_PROBE(fail3);
+fail2:
+       EFSYS_PROBE(fail2);
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+       return (rc);
+}
+
+       __checkReturn                   efx_rc_t
+efx_mae_counters_stream_start(
+       __in                            efx_nic_t *enp,
+       __in                            uint16_t rxq_id,
+       __in                            uint16_t packet_size,
+       __in                            uint32_t flags_in,
+       __out                           uint32_t *flags_out)
+{
+       efx_mcdi_req_t req;
+       EFX_MCDI_DECLARE_BUF(payload, MC_CMD_MAE_COUNTERS_STREAM_START_IN_LEN,
+                            MC_CMD_MAE_COUNTERS_STREAM_START_OUT_LEN);
+       efx_rc_t rc;
+
+       EFX_STATIC_ASSERT(EFX_MAE_COUNTERS_STREAM_IN_ZERO_SQUASH_DISABLE ==
+           1U << MC_CMD_MAE_COUNTERS_STREAM_START_IN_ZERO_SQUASH_DISABLE_LBN);
+
+       EFX_STATIC_ASSERT(EFX_MAE_COUNTERS_STREAM_OUT_USES_CREDITS ==
+           1U << MC_CMD_MAE_COUNTERS_STREAM_START_OUT_USES_CREDITS_LBN);
+
+       req.emr_cmd = MC_CMD_MAE_COUNTERS_STREAM_START;
+       req.emr_in_buf = payload;
+       req.emr_in_length = MC_CMD_MAE_COUNTERS_STREAM_START_IN_LEN;
+       req.emr_out_buf = payload;
+       req.emr_out_length = MC_CMD_MAE_COUNTERS_STREAM_START_OUT_LEN;
+
+       MCDI_IN_SET_WORD(req, MAE_COUNTERS_STREAM_START_IN_QID, rxq_id);
+       MCDI_IN_SET_WORD(req, MAE_COUNTERS_STREAM_START_IN_PACKET_SIZE,
+                        packet_size);
+       MCDI_IN_SET_DWORD(req, MAE_COUNTERS_STREAM_START_IN_FLAGS, flags_in);
+
+       efx_mcdi_execute(enp, &req);
+
+       if (req.emr_rc != 0) {
+               rc = req.emr_rc;
+               goto fail1;
+       }
+
+       if (req.emr_out_length_used <
+           MC_CMD_MAE_COUNTERS_STREAM_START_OUT_LEN) {
+               rc = EMSGSIZE;
+               goto fail2;
+       }
+
+       *flags_out = MCDI_OUT_DWORD(req, MAE_COUNTERS_STREAM_START_OUT_FLAGS);
+
+       return (0);
+
+fail2:
+       EFSYS_PROBE(fail2);
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+       return (rc);
+}
+
+       __checkReturn                   efx_rc_t
+efx_mae_counters_stream_stop(
+       __in                            efx_nic_t *enp,
+       __in                            uint16_t rxq_id,
+       __out_opt                       uint32_t *gen_countp)
+{
+       efx_mcdi_req_t req;
+       EFX_MCDI_DECLARE_BUF(payload, MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_LEN,
+                            MC_CMD_MAE_COUNTERS_STREAM_STOP_OUT_LEN);
+       efx_rc_t rc;
+
+       req.emr_cmd = MC_CMD_MAE_COUNTERS_STREAM_STOP;
+       req.emr_in_buf = payload;
+       req.emr_in_length = MC_CMD_MAE_COUNTERS_STREAM_STOP_IN_LEN;
+       req.emr_out_buf = payload;
+       req.emr_out_length = MC_CMD_MAE_COUNTERS_STREAM_STOP_OUT_LEN;
+
+       MCDI_IN_SET_WORD(req, MAE_COUNTERS_STREAM_STOP_IN_QID, rxq_id);
+
+       efx_mcdi_execute(enp, &req);
+
+       if (req.emr_rc != 0) {
+               rc = req.emr_rc;
+               goto fail1;
+       }
+
+       if (req.emr_out_length_used <
+           MC_CMD_MAE_COUNTERS_STREAM_STOP_OUT_LEN) {
+               rc = EMSGSIZE;
+               goto fail2;
+       }
+
+       if (gen_countp != NULL) {
+               *gen_countp = MCDI_OUT_DWORD(req,
+                           MAE_COUNTERS_STREAM_STOP_OUT_GENERATION_COUNT);
+       }
+
+       return (0);
+
+fail2:
+       EFSYS_PROBE(fail2);
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+       return (rc);
+}
+
+       __checkReturn                   efx_rc_t
+efx_mae_counters_stream_give_credits(
+       __in                            efx_nic_t *enp,
+       __in                            uint32_t n_credits)
+{
+       efx_mcdi_req_t req;
+       EFX_MCDI_DECLARE_BUF(payload,
+                            MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_LEN,
+                            MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_OUT_LEN);
+       efx_rc_t rc;
+
+       req.emr_cmd = MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS;
+       req.emr_in_buf = payload;
+       req.emr_in_length = MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_LEN;
+       req.emr_out_buf = payload;
+       req.emr_out_length = MC_CMD_MAE_COUNTERS_STREAM_GIVE_CREDITS_OUT_LEN;
+
+       MCDI_IN_SET_DWORD(req, MAE_COUNTERS_STREAM_GIVE_CREDITS_IN_NUM_CREDITS,
+                        n_credits);
+
+       efx_mcdi_execute(enp, &req);
+
+       if (req.emr_rc != 0) {
+               rc = req.emr_rc;
+               goto fail1;
+       }
+
+       return (0);
+
+fail1:
+       EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+       return (rc);
+}
+
        __checkReturn                   efx_rc_t
 efx_mae_action_set_free(
        __in                            efx_nic_t *enp,
@@ -2176,15 +2853,22 @@ efx_mae_action_set_free(
                goto fail2;
        }
 
+       if (req.emr_out_length_used < MC_CMD_MAE_ACTION_SET_FREE_OUT_LENMIN) {
+               rc = EMSGSIZE;
+               goto fail3;
+       }
+
        if (MCDI_OUT_DWORD(req, MAE_ACTION_SET_FREE_OUT_FREED_AS_ID) !=
            aset_idp->id) {
                /* Firmware failed to free the action set. */
                rc = EAGAIN;
-               goto fail3;
+               goto fail4;
        }
 
        return (0);
 
+fail4:
+       EFSYS_PROBE(fail4);
 fail3:
        EFSYS_PROBE(fail3);
 fail2:
@@ -2254,10 +2938,10 @@ efx_mae_action_rule_insert(
         * MCDI request and are thus safe to be copied directly to the buffer.
         */
        EFX_STATIC_ASSERT(sizeof (spec->emms_mask_value_pairs.action) >=
-           MAE_FIELD_MASK_VALUE_PAIRS_LEN);
+           MAE_FIELD_MASK_VALUE_PAIRS_V2_LEN);
        offset = MC_CMD_MAE_ACTION_RULE_INSERT_IN_MATCH_CRITERIA_OFST;
        memcpy(payload + offset, spec->emms_mask_value_pairs.action,
-           MAE_FIELD_MASK_VALUE_PAIRS_LEN);
+           MAE_FIELD_MASK_VALUE_PAIRS_V2_LEN);
 
        efx_mcdi_execute(enp, &req);
 
@@ -2326,15 +3010,23 @@ efx_mae_action_rule_remove(
                goto fail2;
        }
 
+       if (req.emr_out_length_used <
+           MC_CMD_MAE_ACTION_RULE_DELETE_OUT_LENMIN) {
+               rc = EMSGSIZE;
+               goto fail3;
+       }
+
        if (MCDI_OUT_DWORD(req, MAE_ACTION_RULE_DELETE_OUT_DELETED_AR_ID) !=
            ar_idp->id) {
                /* Firmware failed to delete the action rule. */
                rc = EAGAIN;
-               goto fail3;
+               goto fail4;
        }
 
        return (0);
 
+fail4:
+       EFSYS_PROBE(fail4);
 fail3:
        EFSYS_PROBE(fail3);
 fail2: