drivers/net/hns3/hns3_fdir.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2019 Hisilicon Limited.
   3  */
   4
   5 #include <stdbool.h>
   6 #include <rte_ethdev_driver.h>
   7 #include <rte_hash.h>
   8 #include <rte_hash_crc.h>
   9 #include <rte_io.h>
  10 #include <rte_malloc.h>
  11
  12 #include "hns3_ethdev.h"
  13 #include "hns3_logs.h"
  14
  15 #define HNS3_VLAN_TAG_TYPE_NONE         0
  16 #define HNS3_VLAN_TAG_TYPE_TAG2         1
  17 #define HNS3_VLAN_TAG_TYPE_TAG1         2
  18 #define HNS3_VLAN_TAG_TYPE_TAG1_2       3
  19
  20 #define HNS3_PF_ID_S                    0
  21 #define HNS3_PF_ID_M                    GENMASK(2, 0)
  22 #define HNS3_VF_ID_S                    3
  23 #define HNS3_VF_ID_M                    GENMASK(10, 3)
  24 #define HNS3_PORT_TYPE_B                11
  25 #define HNS3_NETWORK_PORT_ID_S          0
  26 #define HNS3_NETWORK_PORT_ID_M          GENMASK(3, 0)
  27
  28 #define HNS3_FD_EPORT_SW_EN_B           0
  29
  30 #define HNS3_FD_AD_DATA_S               32
  31 #define HNS3_FD_AD_DROP_B               0
  32 #define HNS3_FD_AD_DIRECT_QID_B 1
  33 #define HNS3_FD_AD_QID_S                2
  34 #define HNS3_FD_AD_QID_M                GENMASK(12, 2)
  35 #define HNS3_FD_AD_USE_COUNTER_B        12
  36 #define HNS3_FD_AD_COUNTER_NUM_S        13
  37 #define HNS3_FD_AD_COUNTER_NUM_M        GENMASK(20, 13)
  38 #define HNS3_FD_AD_NXT_STEP_B           20
  39 #define HNS3_FD_AD_NXT_KEY_S            21
  40 #define HNS3_FD_AD_NXT_KEY_M            GENMASK(26, 21)
  41 #define HNS3_FD_AD_WR_RULE_ID_B 0
  42 #define HNS3_FD_AD_RULE_ID_S            1
  43 #define HNS3_FD_AD_RULE_ID_M            GENMASK(13, 1)
  44
  45 enum HNS3_PORT_TYPE {
  46         HOST_PORT,
  47         NETWORK_PORT
  48 };
  49
  50 enum HNS3_FD_MODE {
  51         HNS3_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1,
  52         HNS3_FD_MODE_DEPTH_1K_WIDTH_400B_STAGE_2,
  53         HNS3_FD_MODE_DEPTH_4K_WIDTH_200B_STAGE_1,
  54         HNS3_FD_MODE_DEPTH_2K_WIDTH_200B_STAGE_2,
  55 };
  56
  57 enum HNS3_FD_KEY_TYPE {
  58         HNS3_FD_KEY_BASE_ON_PTYPE,
  59         HNS3_FD_KEY_BASE_ON_TUPLE,
  60 };
  61
  62 enum HNS3_FD_META_DATA {
  63         PACKET_TYPE_ID,
  64         IP_FRAGEMENT,
  65         ROCE_TYPE,
  66         NEXT_KEY,
  67         VLAN_NUMBER,
  68         SRC_VPORT,
  69         DST_VPORT,
  70         TUNNEL_PACKET,
  71         MAX_META_DATA,
  72 };
  73
  74 struct key_info {
  75         uint8_t key_type;
  76         uint8_t key_length;
  77 };
  78
  79 static const struct key_info meta_data_key_info[] = {
  80         {PACKET_TYPE_ID, 6},
  81         {IP_FRAGEMENT, 1},
  82         {ROCE_TYPE, 1},
  83         {NEXT_KEY, 5},
  84         {VLAN_NUMBER, 2},
  85         {SRC_VPORT, 12},
  86         {DST_VPORT, 12},
  87         {TUNNEL_PACKET, 1},
  88 };
  89
  90 static const struct key_info tuple_key_info[] = {
  91         {OUTER_DST_MAC, 48},
  92         {OUTER_SRC_MAC, 48},
  93         {OUTER_VLAN_TAG_FST, 16},
  94         {OUTER_VLAN_TAG_SEC, 16},
  95         {OUTER_ETH_TYPE, 16},
  96         {OUTER_L2_RSV, 16},
  97         {OUTER_IP_TOS, 8},
  98         {OUTER_IP_PROTO, 8},
  99         {OUTER_SRC_IP, 32},
 100         {OUTER_DST_IP, 32},
 101         {OUTER_L3_RSV, 16},
 102         {OUTER_SRC_PORT, 16},
 103         {OUTER_DST_PORT, 16},
 104         {OUTER_L4_RSV, 32},
 105         {OUTER_TUN_VNI, 24},
 106         {OUTER_TUN_FLOW_ID, 8},
 107         {INNER_DST_MAC, 48},
 108         {INNER_SRC_MAC, 48},
 109         {INNER_VLAN_TAG1, 16},
 110         {INNER_VLAN_TAG2, 16},
 111         {INNER_ETH_TYPE, 16},
 112         {INNER_L2_RSV, 16},
 113         {INNER_IP_TOS, 8},
 114         {INNER_IP_PROTO, 8},
 115         {INNER_SRC_IP, 32},
 116         {INNER_DST_IP, 32},
 117         {INNER_L3_RSV, 16},
 118         {INNER_SRC_PORT, 16},
 119         {INNER_DST_PORT, 16},
 120         {INNER_SCTP_TAG, 32},
 121 };
 122
 123 #define HNS3_BITS_PER_BYTE      8
 124 #define MAX_KEY_LENGTH          400
 125 #define MAX_200B_KEY_LENGTH     200
 126 #define MAX_META_DATA_LENGTH    16
 127 #define MAX_KEY_DWORDS  DIV_ROUND_UP(MAX_KEY_LENGTH / HNS3_BITS_PER_BYTE, 4)
 128 #define MAX_KEY_BYTES   (MAX_KEY_DWORDS * 4)
 129
 130 enum HNS3_FD_PACKET_TYPE {
 131         NIC_PACKET,
 132         ROCE_PACKET,
 133 };
 134
 135 /* For each bit of TCAM entry, it uses a pair of 'x' and
 136  * 'y' to indicate which value to match, like below:
 137  * ----------------------------------
 138  * | bit x | bit y |  search value  |
 139  * ----------------------------------
 140  * |   0   |   0   |   always hit   |
 141  * ----------------------------------
 142  * |   1   |   0   |   match '0'    |
 143  * ----------------------------------
 144  * |   0   |   1   |   match '1'    |
 145  * ----------------------------------
 146  * |   1   |   1   |   invalid      |
 147  * ----------------------------------
 148  * Then for input key(k) and mask(v), we can calculate the value by
 149  * the formulae:
 150  *      x = (~k) & v
 151  *      y = k & v
 152  */
 153 #define calc_x(x, k, v) ((x) = (~(k) & (v)))
 154 #define calc_y(y, k, v) ((y) = ((k) & (v)))
 155
 156 struct hns3_fd_tcam_config_1_cmd {
 157         uint8_t stage;
 158         uint8_t xy_sel;
 159         uint8_t port_info;
 160         uint8_t rsv1[1];
 161         rte_le32_t index;
 162         uint8_t entry_vld;
 163         uint8_t rsv2[7];
 164         uint8_t tcam_data[8];
 165 };
 166
 167 struct hns3_fd_tcam_config_2_cmd {
 168         uint8_t tcam_data[24];
 169 };
 170
 171 struct hns3_fd_tcam_config_3_cmd {
 172         uint8_t tcam_data[20];
 173         uint8_t rsv[4];
 174 };
 175
 176 struct hns3_get_fd_mode_cmd {
 177         uint8_t mode;
 178         uint8_t enable;
 179         uint8_t rsv[22];
 180 };
 181
 182 struct hns3_get_fd_allocation_cmd {
 183         rte_le32_t stage1_entry_num;
 184         rte_le32_t stage2_entry_num;
 185         rte_le16_t stage1_counter_num;
 186         rte_le16_t stage2_counter_num;
 187         uint8_t rsv[12];
 188 };
 189
 190 struct hns3_set_fd_key_config_cmd {
 191         uint8_t stage;
 192         uint8_t key_select;
 193         uint8_t inner_sipv6_word_en;
 194         uint8_t inner_dipv6_word_en;
 195         uint8_t outer_sipv6_word_en;
 196         uint8_t outer_dipv6_word_en;
 197         uint8_t rsv1[2];
 198         rte_le32_t tuple_mask;
 199         rte_le32_t meta_data_mask;
 200         uint8_t rsv2[8];
 201 };
 202
 203 struct hns3_fd_ad_config_cmd {
 204         uint8_t stage;
 205         uint8_t rsv1[3];
 206         rte_le32_t index;
 207         rte_le64_t ad_data;
 208         uint8_t rsv2[8];
 209 };
 210
 211 struct hns3_fd_get_cnt_cmd {
 212         uint8_t stage;
 213         uint8_t rsv1[3];
 214         rte_le16_t index;
 215         uint8_t rsv2[2];
 216         rte_le64_t value;
 217         uint8_t rsv3[8];
 218 };
 219
 220 static int hns3_get_fd_mode(struct hns3_hw *hw, uint8_t *fd_mode)
 221 {
 222         struct hns3_get_fd_mode_cmd *req;
 223         struct hns3_cmd_desc desc;
 224         int ret;
 225
 226         hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_FD_MODE_CTRL, true);
 227
 228         req = (struct hns3_get_fd_mode_cmd *)desc.data;
 229
 230         ret = hns3_cmd_send(hw, &desc, 1);
 231         if (ret) {
 232                 hns3_err(hw, "Get fd mode fail, ret=%d", ret);
 233                 return ret;
 234         }
 235
 236         *fd_mode = req->mode;
 237
 238         return ret;
 239 }
 240
 241 static int hns3_get_fd_allocation(struct hns3_hw *hw,
 242                                   uint32_t *stage1_entry_num,
 243                                   uint32_t *stage2_entry_num,
 244                                   uint16_t *stage1_counter_num,
 245                                   uint16_t *stage2_counter_num)
 246 {
 247         struct hns3_get_fd_allocation_cmd *req;
 248         struct hns3_cmd_desc desc;
 249         int ret;
 250
 251         hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_FD_GET_ALLOCATION, true);
 252
 253         req = (struct hns3_get_fd_allocation_cmd *)desc.data;
 254
 255         ret = hns3_cmd_send(hw, &desc, 1);
 256         if (ret) {
 257                 hns3_err(hw, "Query fd allocation fail, ret=%d", ret);
 258                 return ret;
 259         }
 260
 261         *stage1_entry_num = rte_le_to_cpu_32(req->stage1_entry_num);
 262         *stage2_entry_num = rte_le_to_cpu_32(req->stage2_entry_num);
 263         *stage1_counter_num = rte_le_to_cpu_16(req->stage1_counter_num);
 264         *stage2_counter_num = rte_le_to_cpu_16(req->stage2_counter_num);
 265
 266         return ret;
 267 }
 268
 269 static int hns3_set_fd_key_config(struct hns3_adapter *hns)
 270 {
 271         struct hns3_set_fd_key_config_cmd *req;
 272         struct hns3_fd_key_cfg *key_cfg;
 273         struct hns3_pf *pf = &hns->pf;
 274         struct hns3_hw *hw = &hns->hw;
 275         struct hns3_cmd_desc desc;
 276         int ret;
 277
 278         hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_FD_KEY_CONFIG, false);
 279
 280         req = (struct hns3_set_fd_key_config_cmd *)desc.data;
 281         key_cfg = &pf->fdir.fd_cfg.key_cfg[HNS3_FD_STAGE_1];
 282         req->stage = HNS3_FD_STAGE_1;
 283         req->key_select = key_cfg->key_sel;
 284         req->inner_sipv6_word_en = key_cfg->inner_sipv6_word_en;
 285         req->inner_dipv6_word_en = key_cfg->inner_dipv6_word_en;
 286         req->outer_sipv6_word_en = key_cfg->outer_sipv6_word_en;
 287         req->outer_dipv6_word_en = key_cfg->outer_dipv6_word_en;
 288         req->tuple_mask = rte_cpu_to_le_32(~key_cfg->tuple_active);
 289         req->meta_data_mask = rte_cpu_to_le_32(~key_cfg->meta_data_active);
 290
 291         ret = hns3_cmd_send(hw, &desc, 1);
 292         if (ret)
 293                 hns3_err(hw, "Set fd key fail, ret=%d", ret);
 294
 295         return ret;
 296 }
 297
 298 int hns3_init_fd_config(struct hns3_adapter *hns)
 299 {
 300         struct hns3_pf *pf = &hns->pf;
 301         struct hns3_hw *hw = &hns->hw;
 302         struct hns3_fd_key_cfg *key_cfg;
 303         int ret;
 304
 305         ret = hns3_get_fd_mode(hw, &pf->fdir.fd_cfg.fd_mode);
 306         if (ret)
 307                 return ret;
 308
 309         switch (pf->fdir.fd_cfg.fd_mode) {
 310         case HNS3_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1:
 311                 pf->fdir.fd_cfg.max_key_length = MAX_KEY_LENGTH;
 312                 break;
 313         case HNS3_FD_MODE_DEPTH_4K_WIDTH_200B_STAGE_1:
 314                 pf->fdir.fd_cfg.max_key_length = MAX_200B_KEY_LENGTH;
 315                 hns3_warn(hw, "Unsupported tunnel filter in 4K*200Bit");
 316                 break;
 317         default:
 318                 hns3_err(hw, "Unsupported flow director mode %d",
 319                             pf->fdir.fd_cfg.fd_mode);
 320                 return -EOPNOTSUPP;
 321         }
 322
 323         key_cfg = &pf->fdir.fd_cfg.key_cfg[HNS3_FD_STAGE_1];
 324         key_cfg->key_sel = HNS3_FD_KEY_BASE_ON_TUPLE;
 325         key_cfg->inner_sipv6_word_en = IPV6_ADDR_WORD_MASK;
 326         key_cfg->inner_dipv6_word_en = IPV6_ADDR_WORD_MASK;
 327         key_cfg->outer_sipv6_word_en = 0;
 328         key_cfg->outer_dipv6_word_en = 0;
 329
 330         key_cfg->tuple_active = BIT(INNER_VLAN_TAG1) | BIT(INNER_ETH_TYPE) |
 331             BIT(INNER_IP_PROTO) | BIT(INNER_IP_TOS) |
 332             BIT(INNER_SRC_IP) | BIT(INNER_DST_IP) |
 333             BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT);
 334
 335         /* If use max 400bit key, we can support tuples for ether type */
 336         if (pf->fdir.fd_cfg.max_key_length == MAX_KEY_LENGTH) {
 337                 key_cfg->tuple_active |=
 338                     BIT(INNER_DST_MAC) | BIT(INNER_SRC_MAC) |
 339                     BIT(OUTER_SRC_PORT) | BIT(INNER_SCTP_TAG) |
 340                     BIT(OUTER_DST_PORT) | BIT(INNER_VLAN_TAG2) |
 341                     BIT(OUTER_TUN_VNI) | BIT(OUTER_TUN_FLOW_ID) |
 342                     BIT(OUTER_ETH_TYPE) | BIT(OUTER_IP_PROTO);
 343         }
 344
 345         /* roce_type is used to filter roce frames
 346          * dst_vport is used to specify the rule
 347          */
 348         key_cfg->meta_data_active = BIT(DST_VPORT) | BIT(TUNNEL_PACKET) |
 349             BIT(VLAN_NUMBER);
 350
 351         ret = hns3_get_fd_allocation(hw,
 352                                      &pf->fdir.fd_cfg.rule_num[HNS3_FD_STAGE_1],
 353                                      &pf->fdir.fd_cfg.rule_num[HNS3_FD_STAGE_2],
 354                                      &pf->fdir.fd_cfg.cnt_num[HNS3_FD_STAGE_1],
 355                                      &pf->fdir.fd_cfg.cnt_num[HNS3_FD_STAGE_2]);
 356         if (ret)
 357                 return ret;
 358
 359         return hns3_set_fd_key_config(hns);
 360 }
 361
 362 static int hns3_fd_tcam_config(struct hns3_hw *hw, bool sel_x, int loc,
 363                                uint8_t *key, bool is_add)
 364 {
 365 #define FD_TCAM_CMD_NUM 3
 366         struct hns3_fd_tcam_config_1_cmd *req1;
 367         struct hns3_fd_tcam_config_2_cmd *req2;
 368         struct hns3_fd_tcam_config_3_cmd *req3;
 369         struct hns3_cmd_desc desc[FD_TCAM_CMD_NUM];
 370         int len;
 371         int ret;
 372
 373         hns3_cmd_setup_basic_desc(&desc[0], HNS3_OPC_FD_TCAM_OP, false);
 374         desc[0].flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_NEXT);
 375         hns3_cmd_setup_basic_desc(&desc[1], HNS3_OPC_FD_TCAM_OP, false);
 376         desc[1].flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_NEXT);
 377         hns3_cmd_setup_basic_desc(&desc[2], HNS3_OPC_FD_TCAM_OP, false);
 378
 379         req1 = (struct hns3_fd_tcam_config_1_cmd *)desc[0].data;
 380         req2 = (struct hns3_fd_tcam_config_2_cmd *)desc[1].data;
 381         req3 = (struct hns3_fd_tcam_config_3_cmd *)desc[2].data;
 382
 383         req1->stage = HNS3_FD_STAGE_1;
 384         req1->xy_sel = sel_x ? 1 : 0;
 385         hns3_set_bit(req1->port_info, HNS3_FD_EPORT_SW_EN_B, 0);
 386         req1->index = rte_cpu_to_le_32(loc);
 387         req1->entry_vld = sel_x ? is_add : 0;
 388
 389         if (key) {
 390                 len = sizeof(req1->tcam_data);
 391                 memcpy(req1->tcam_data, key, len);
 392                 key += len;
 393
 394                 len = sizeof(req2->tcam_data);
 395                 memcpy(req2->tcam_data, key, len);
 396                 key += len;
 397
 398                 len = sizeof(req3->tcam_data);
 399                 memcpy(req3->tcam_data, key, len);
 400         }
 401
 402         ret = hns3_cmd_send(hw, desc, FD_TCAM_CMD_NUM);
 403         if (ret)
 404                 hns3_err(hw, "Config tcam key fail, ret=%d loc=%d add=%d",
 405                             ret, loc, is_add);
 406         return ret;
 407 }
 408
 409 static int hns3_fd_ad_config(struct hns3_hw *hw, int loc,
 410                              struct hns3_fd_ad_data *action)
 411 {
 412         struct hns3_fd_ad_config_cmd *req;
 413         struct hns3_cmd_desc desc;
 414         uint64_t ad_data = 0;
 415         int ret;
 416
 417         hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_FD_AD_OP, false);
 418
 419         req = (struct hns3_fd_ad_config_cmd *)desc.data;
 420         req->index = rte_cpu_to_le_32(loc);
 421         req->stage = HNS3_FD_STAGE_1;
 422
 423         hns3_set_bit(ad_data, HNS3_FD_AD_WR_RULE_ID_B,
 424                      action->write_rule_id_to_bd);
 425         hns3_set_field(ad_data, HNS3_FD_AD_RULE_ID_M, HNS3_FD_AD_RULE_ID_S,
 426                        action->rule_id);
 427         ad_data <<= HNS3_FD_AD_DATA_S;
 428         hns3_set_bit(ad_data, HNS3_FD_AD_DROP_B, action->drop_packet);
 429         hns3_set_bit(ad_data, HNS3_FD_AD_DIRECT_QID_B,
 430                      action->forward_to_direct_queue);
 431         hns3_set_field(ad_data, HNS3_FD_AD_QID_M, HNS3_FD_AD_QID_S,
 432                        action->queue_id);
 433         hns3_set_bit(ad_data, HNS3_FD_AD_USE_COUNTER_B, action->use_counter);
 434         hns3_set_field(ad_data, HNS3_FD_AD_COUNTER_NUM_M,
 435                        HNS3_FD_AD_COUNTER_NUM_S, action->counter_id);
 436         hns3_set_bit(ad_data, HNS3_FD_AD_NXT_STEP_B, action->use_next_stage);
 437         hns3_set_field(ad_data, HNS3_FD_AD_NXT_KEY_M, HNS3_FD_AD_NXT_KEY_S,
 438                        action->counter_id);
 439
 440         req->ad_data = rte_cpu_to_le_64(ad_data);
 441         ret = hns3_cmd_send(hw, &desc, 1);
 442         if (ret)
 443                 hns3_err(hw, "Config fd ad fail, ret=%d loc=%d", ret, loc);
 444
 445         return ret;
 446 }
 447
 448 static inline void hns3_fd_convert_mac(uint8_t *key, uint8_t *mask,
 449                                        uint8_t *mac_x, uint8_t *mac_y)
 450 {
 451         uint8_t tmp;
 452         int i;
 453
 454         for (i = 0; i < RTE_ETHER_ADDR_LEN; i++) {
 455                 tmp = RTE_ETHER_ADDR_LEN - 1 - i;
 456                 calc_x(mac_x[tmp], key[i], mask[i]);
 457                 calc_y(mac_y[tmp], key[i], mask[i]);
 458         }
 459 }
 460
 461 static void hns3_fd_convert_int16(uint32_t tuple, struct hns3_fdir_rule *rule,
 462                                   uint8_t *val_x, uint8_t *val_y)
 463 {
 464         uint16_t tmp_x_s;
 465         uint16_t tmp_y_s;
 466         uint16_t mask;
 467         uint16_t key;
 468
 469         switch (tuple) {
 470         case OUTER_SRC_PORT:
 471                 key = rule->key_conf.spec.outer_src_port;
 472                 mask = rule->key_conf.mask.outer_src_port;
 473                 break;
 474         case OUTER_DST_PORT:
 475                 key = rule->key_conf.spec.tunnel_type;
 476                 mask = rule->key_conf.mask.tunnel_type;
 477                 break;
 478         case OUTER_ETH_TYPE:
 479                 key = rule->key_conf.spec.outer_ether_type;
 480                 mask = rule->key_conf.mask.outer_ether_type;
 481                 break;
 482         case INNER_SRC_PORT:
 483                 key = rule->key_conf.spec.src_port;
 484                 mask = rule->key_conf.mask.src_port;
 485                 break;
 486         case INNER_DST_PORT:
 487                 key = rule->key_conf.spec.dst_port;
 488                 mask = rule->key_conf.mask.dst_port;
 489                 break;
 490         case INNER_VLAN_TAG1:
 491                 key = rule->key_conf.spec.vlan_tag1;
 492                 mask = rule->key_conf.mask.vlan_tag1;
 493                 break;
 494         case INNER_VLAN_TAG2:
 495                 key = rule->key_conf.spec.vlan_tag2;
 496                 mask = rule->key_conf.mask.vlan_tag2;
 497                 break;
 498         default:
 499                 /*  INNER_ETH_TYPE: */
 500                 key = rule->key_conf.spec.ether_type;
 501                 mask = rule->key_conf.mask.ether_type;
 502                 break;
 503         }
 504         calc_x(tmp_x_s, key, mask);
 505         calc_y(tmp_y_s, key, mask);
 506         val_x[0] = rte_cpu_to_le_16(tmp_x_s) & 0xFF;
 507         val_x[1] = rte_cpu_to_le_16(tmp_x_s) >> HNS3_BITS_PER_BYTE;
 508         val_y[0] = rte_cpu_to_le_16(tmp_y_s) & 0xFF;
 509         val_y[1] = rte_cpu_to_le_16(tmp_y_s) >> HNS3_BITS_PER_BYTE;
 510 }
 511
 512 static inline void hns3_fd_convert_int32(uint32_t key, uint32_t mask,
 513                                          uint8_t *val_x, uint8_t *val_y)
 514 {
 515         uint32_t tmp_x_l;
 516         uint32_t tmp_y_l;
 517
 518         calc_x(tmp_x_l, key, mask);
 519         calc_y(tmp_y_l, key, mask);
 520         memcpy(val_x, &tmp_x_l, sizeof(tmp_x_l));
 521         memcpy(val_y, &tmp_y_l, sizeof(tmp_y_l));
 522 }
 523
 524 static bool hns3_fd_convert_tuple(uint32_t tuple, uint8_t *key_x,
 525                                   uint8_t *key_y, struct hns3_fdir_rule *rule)
 526 {
 527         struct hns3_fdir_key_conf *key_conf;
 528         int tmp;
 529         int i;
 530
 531         if ((rule->input_set & BIT(tuple)) == 0)
 532                 return true;
 533
 534         key_conf = &rule->key_conf;
 535         switch (tuple) {
 536         case INNER_DST_MAC:
 537                 hns3_fd_convert_mac(key_conf->spec.dst_mac,
 538                                     key_conf->mask.dst_mac, key_x, key_y);
 539                 break;
 540         case INNER_SRC_MAC:
 541                 hns3_fd_convert_mac(key_conf->spec.src_mac,
 542                                     key_conf->mask.src_mac, key_x, key_y);
 543                 break;
 544         case OUTER_SRC_PORT:
 545         case OUTER_DST_PORT:
 546         case OUTER_ETH_TYPE:
 547         case INNER_SRC_PORT:
 548         case INNER_DST_PORT:
 549         case INNER_VLAN_TAG1:
 550         case INNER_VLAN_TAG2:
 551         case INNER_ETH_TYPE:
 552                 hns3_fd_convert_int16(tuple, rule, key_x, key_y);
 553                 break;
 554         case INNER_SRC_IP:
 555                 hns3_fd_convert_int32(key_conf->spec.src_ip[IP_ADDR_KEY_ID],
 556                                       key_conf->mask.src_ip[IP_ADDR_KEY_ID],
 557                                       key_x, key_y);
 558                 break;
 559         case INNER_DST_IP:
 560                 hns3_fd_convert_int32(key_conf->spec.dst_ip[IP_ADDR_KEY_ID],
 561                                       key_conf->mask.dst_ip[IP_ADDR_KEY_ID],
 562                                       key_x, key_y);
 563                 break;
 564         case INNER_SCTP_TAG:
 565                 hns3_fd_convert_int32(key_conf->spec.sctp_tag,
 566                                       key_conf->mask.sctp_tag, key_x, key_y);
 567                 break;
 568         case OUTER_TUN_VNI:
 569                 for (i = 0; i < VNI_OR_TNI_LEN; i++) {
 570                         tmp = VNI_OR_TNI_LEN - 1 - i;
 571                         calc_x(key_x[tmp],
 572                                key_conf->spec.outer_tun_vni[i],
 573                                key_conf->mask.outer_tun_vni[i]);
 574                         calc_y(key_y[tmp],
 575                                key_conf->spec.outer_tun_vni[i],
 576                                key_conf->mask.outer_tun_vni[i]);
 577                 }
 578                 break;
 579         case OUTER_TUN_FLOW_ID:
 580                 calc_x(*key_x, key_conf->spec.outer_tun_flow_id,
 581                        key_conf->mask.outer_tun_flow_id);
 582                 calc_y(*key_y, key_conf->spec.outer_tun_flow_id,
 583                        key_conf->mask.outer_tun_flow_id);
 584                 break;
 585         case INNER_IP_TOS:
 586                 calc_x(*key_x, key_conf->spec.ip_tos, key_conf->mask.ip_tos);
 587                 calc_y(*key_y, key_conf->spec.ip_tos, key_conf->mask.ip_tos);
 588                 break;
 589         case OUTER_IP_PROTO:
 590                 calc_x(*key_x, key_conf->spec.outer_proto,
 591                        key_conf->mask.outer_proto);
 592                 calc_y(*key_y, key_conf->spec.outer_proto,
 593                        key_conf->mask.outer_proto);
 594                 break;
 595         case INNER_IP_PROTO:
 596                 calc_x(*key_x, key_conf->spec.ip_proto,
 597                        key_conf->mask.ip_proto);
 598                 calc_y(*key_y, key_conf->spec.ip_proto,
 599                        key_conf->mask.ip_proto);
 600                 break;
 601         }
 602         return true;
 603 }
 604
 605 static uint32_t hns3_get_port_number(uint8_t pf_id, uint8_t vf_id)
 606 {
 607         uint32_t port_number = 0;
 608
 609         hns3_set_field(port_number, HNS3_PF_ID_M, HNS3_PF_ID_S, pf_id);
 610         hns3_set_field(port_number, HNS3_VF_ID_M, HNS3_VF_ID_S, vf_id);
 611         hns3_set_bit(port_number, HNS3_PORT_TYPE_B, HOST_PORT);
 612
 613         return port_number;
 614 }
 615
 616 static void hns3_fd_convert_meta_data(struct hns3_fd_key_cfg *cfg,
 617                                       uint8_t vf_id,
 618                                       struct hns3_fdir_rule *rule,
 619                                       uint8_t *key_x, uint8_t *key_y)
 620 {
 621         uint16_t meta_data = 0;
 622         uint32_t port_number;
 623         uint8_t cur_pos = 0;
 624         uint8_t tuple_size;
 625         uint8_t shift_bits;
 626         uint32_t tmp_x;
 627         uint32_t tmp_y;
 628         uint8_t i;
 629
 630         for (i = 0; i < MAX_META_DATA; i++) {
 631                 if ((cfg->meta_data_active & BIT(i)) == 0)
 632                         continue;
 633
 634                 tuple_size = meta_data_key_info[i].key_length;
 635                 if (i == TUNNEL_PACKET) {
 636                         hns3_set_bit(meta_data, cur_pos,
 637                                      rule->key_conf.spec.tunnel_type ? 1 : 0);
 638                         cur_pos += tuple_size;
 639                 } else if (i == VLAN_NUMBER) {
 640                         uint32_t vlan_tag;
 641                         uint8_t vlan_num;
 642                         if (rule->key_conf.spec.tunnel_type == 0)
 643                                 vlan_num = rule->key_conf.vlan_num;
 644                         else
 645                                 vlan_num = rule->key_conf.outer_vlan_num;
 646                         if (vlan_num == 1)
 647                                 vlan_tag = HNS3_VLAN_TAG_TYPE_TAG1;
 648                         else if (vlan_num == VLAN_TAG_NUM_MAX)
 649                                 vlan_tag = HNS3_VLAN_TAG_TYPE_TAG1_2;
 650                         else
 651                                 vlan_tag = HNS3_VLAN_TAG_TYPE_NONE;
 652                         hns3_set_field(meta_data,
 653                                        GENMASK(cur_pos + tuple_size,
 654                                                cur_pos), cur_pos, vlan_tag);
 655                         cur_pos += tuple_size;
 656                 } else if (i == DST_VPORT) {
 657                         port_number = hns3_get_port_number(0, vf_id);
 658                         hns3_set_field(meta_data,
 659                                        GENMASK(cur_pos + tuple_size, cur_pos),
 660                                        cur_pos, port_number);
 661                         cur_pos += tuple_size;
 662                 }
 663         }
 664
 665         calc_x(tmp_x, meta_data, 0xFFFF);
 666         calc_y(tmp_y, meta_data, 0xFFFF);
 667         shift_bits = sizeof(meta_data) * HNS3_BITS_PER_BYTE - cur_pos;
 668
 669         tmp_x = rte_cpu_to_le_32(tmp_x << shift_bits);
 670         tmp_y = rte_cpu_to_le_32(tmp_y << shift_bits);
 671         key_x[0] = tmp_x & 0xFF;
 672         key_x[1] = (tmp_x >> HNS3_BITS_PER_BYTE) & 0xFF;
 673         key_y[0] = tmp_y & 0xFF;
 674         key_y[1] = (tmp_y >> HNS3_BITS_PER_BYTE) & 0xFF;
 675 }
 676
 677 /* A complete key is combined with meta data key and tuple key.
 678  * Meta data key is stored at the MSB region, and tuple key is stored at
 679  * the LSB region, unused bits will be filled 0.
 680  */
 681 static int hns3_config_key(struct hns3_adapter *hns,
 682                            struct hns3_fdir_rule *rule)
 683 {
 684         struct hns3_pf *pf = &hns->pf;
 685         struct hns3_hw *hw = &hns->hw;
 686         struct hns3_fd_key_cfg *key_cfg;
 687         uint8_t *cur_key_x;
 688         uint8_t *cur_key_y;
 689         uint8_t key_x[MAX_KEY_BYTES] __rte_aligned(4);
 690         uint8_t key_y[MAX_KEY_BYTES] __rte_aligned(4);
 691         uint8_t vf_id = rule->vf_id;
 692         uint8_t meta_data_region;
 693         uint8_t tuple_size;
 694         uint8_t i;
 695         int ret;
 696
 697         memset(key_x, 0, sizeof(key_x));
 698         memset(key_y, 0, sizeof(key_y));
 699         cur_key_x = key_x;
 700         cur_key_y = key_y;
 701
 702         key_cfg = &pf->fdir.fd_cfg.key_cfg[HNS3_FD_STAGE_1];
 703         for (i = 0; i < MAX_TUPLE; i++) {
 704                 bool tuple_valid;
 705
 706                 tuple_size = tuple_key_info[i].key_length / HNS3_BITS_PER_BYTE;
 707                 if (key_cfg->tuple_active & BIT(i)) {
 708                         tuple_valid = hns3_fd_convert_tuple(i, cur_key_x,
 709                                                             cur_key_y, rule);
 710                         if (tuple_valid) {
 711                                 cur_key_x += tuple_size;
 712                                 cur_key_y += tuple_size;
 713                         }
 714                 }
 715         }
 716
 717         meta_data_region = pf->fdir.fd_cfg.max_key_length / HNS3_BITS_PER_BYTE -
 718             MAX_META_DATA_LENGTH / HNS3_BITS_PER_BYTE;
 719
 720         hns3_fd_convert_meta_data(key_cfg, vf_id, rule,
 721                                   key_x + meta_data_region,
 722                                   key_y + meta_data_region);
 723
 724         ret = hns3_fd_tcam_config(hw, false, rule->location, key_y, true);
 725         if (ret) {
 726                 hns3_err(hw, "Config fd key_y fail, loc=%d, ret=%d",
 727                             rule->queue_id, ret);
 728                 return ret;
 729         }
 730
 731         ret = hns3_fd_tcam_config(hw, true, rule->location, key_x, true);
 732         if (ret)
 733                 hns3_err(hw, "Config fd key_x fail, loc=%d, ret=%d",
 734                             rule->queue_id, ret);
 735         return ret;
 736 }
 737
 738 static int hns3_config_action(struct hns3_hw *hw, struct hns3_fdir_rule *rule)
 739 {
 740         struct hns3_fd_ad_data ad_data;
 741
 742         ad_data.ad_id = rule->location;
 743
 744         if (rule->action == HNS3_FD_ACTION_DROP_PACKET) {
 745                 ad_data.drop_packet = true;
 746                 ad_data.forward_to_direct_queue = false;
 747                 ad_data.queue_id = 0;
 748         } else {
 749                 ad_data.drop_packet = false;
 750                 ad_data.forward_to_direct_queue = true;
 751                 ad_data.queue_id = rule->queue_id;
 752         }
 753
 754         if (unlikely(rule->flags & HNS3_RULE_FLAG_COUNTER)) {
 755                 ad_data.use_counter = true;
 756                 ad_data.counter_id = rule->act_cnt.id;
 757         } else {
 758                 ad_data.use_counter = false;
 759                 ad_data.counter_id = 0;
 760         }
 761
 762         if (unlikely(rule->flags & HNS3_RULE_FLAG_FDID))
 763                 ad_data.rule_id = rule->fd_id;
 764         else
 765                 ad_data.rule_id = rule->location;
 766
 767         ad_data.use_next_stage = false;
 768         ad_data.next_input_key = 0;
 769
 770         ad_data.write_rule_id_to_bd = true;
 771
 772         return hns3_fd_ad_config(hw, ad_data.ad_id, &ad_data);
 773 }
 774
 775 static int hns3_fd_clear_all_rules(struct hns3_hw *hw, uint32_t rule_num)
 776 {
 777         uint32_t i;
 778         int ret;
 779
 780         for (i = 0; i < rule_num; i++) {
 781                 ret = hns3_fd_tcam_config(hw, true, i, NULL, false);
 782                 if (ret)
 783                         return ret;
 784         }
 785
 786         return 0;
 787 }
 788
 789 int hns3_fdir_filter_init(struct hns3_adapter *hns)
 790 {
 791         struct hns3_pf *pf = &hns->pf;
 792         struct hns3_fdir_info *fdir_info = &pf->fdir;
 793         uint32_t rule_num = fdir_info->fd_cfg.rule_num[HNS3_FD_STAGE_1];
 794         char fdir_hash_name[RTE_HASH_NAMESIZE];
 795         struct rte_hash_parameters fdir_hash_params = {
 796                 .name = fdir_hash_name,
 797                 .entries = rule_num,
 798                 .key_len = sizeof(struct hns3_fdir_key_conf),
 799                 .hash_func = rte_hash_crc,
 800                 .hash_func_init_val = 0,
 801         };
 802         int ret;
 803
 804         ret = hns3_fd_clear_all_rules(&hns->hw, rule_num);
 805         if (ret) {
 806                 PMD_INIT_LOG(ERR, "Clear all fd rules fail! ret = %d", ret);
 807                 return ret;
 808         }
 809
 810         fdir_hash_params.socket_id = rte_socket_id();
 811         TAILQ_INIT(&fdir_info->fdir_list);
 812         rte_spinlock_init(&fdir_info->flows_lock);
 813         snprintf(fdir_hash_name, RTE_HASH_NAMESIZE, "%s", hns->hw.data->name);
 814         fdir_info->hash_handle = rte_hash_create(&fdir_hash_params);
 815         if (fdir_info->hash_handle == NULL) {
 816                 PMD_INIT_LOG(ERR, "Create FDIR hash handle fail!");
 817                 return -EINVAL;
 818         }
 819         fdir_info->hash_map = rte_zmalloc("hns3 FDIR hash",
 820                                           rule_num *
 821                                           sizeof(struct hns3_fdir_rule_ele *),
 822                                           0);
 823         if (fdir_info->hash_map == NULL) {
 824                 PMD_INIT_LOG(ERR, "Allocate memory for FDIR hash map fail!");
 825                 rte_hash_free(fdir_info->hash_handle);
 826                 return -ENOMEM;
 827         }
 828
 829         return 0;
 830 }
 831
 832 void hns3_fdir_filter_uninit(struct hns3_adapter *hns)
 833 {
 834         struct hns3_pf *pf = &hns->pf;
 835         struct hns3_fdir_info *fdir_info = &pf->fdir;
 836         struct hns3_fdir_rule_ele *fdir_filter;
 837
 838         rte_spinlock_lock(&fdir_info->flows_lock);
 839         if (fdir_info->hash_map) {
 840                 rte_free(fdir_info->hash_map);
 841                 fdir_info->hash_map = NULL;
 842         }
 843         if (fdir_info->hash_handle) {
 844                 rte_hash_free(fdir_info->hash_handle);
 845                 fdir_info->hash_handle = NULL;
 846         }
 847         rte_spinlock_unlock(&fdir_info->flows_lock);
 848
 849         fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list);
 850         while (fdir_filter) {
 851                 TAILQ_REMOVE(&fdir_info->fdir_list, fdir_filter, entries);
 852                 hns3_fd_tcam_config(&hns->hw, true,
 853                                     fdir_filter->fdir_conf.location, NULL,
 854                                     false);
 855                 rte_free(fdir_filter);
 856                 fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list);
 857         }
 858 }
 859
 860 /*
 861  * Find a key in the hash table.
 862  * @return
 863  *   - Zero and positive values are key location.
 864  *   - -EINVAL if the parameters are invalid.
 865  *   - -ENOENT if the key is not found.
 866  */
 867 static int hns3_fdir_filter_lookup(struct hns3_fdir_info *fdir_info,
 868                                     struct hns3_fdir_key_conf *key)
 869 {
 870         hash_sig_t sig;
 871         int ret;
 872
 873         rte_spinlock_lock(&fdir_info->flows_lock);
 874         sig = rte_hash_crc(key, sizeof(*key), 0);
 875         ret = rte_hash_lookup_with_hash(fdir_info->hash_handle, key, sig);
 876         rte_spinlock_unlock(&fdir_info->flows_lock);
 877
 878         return ret;
 879 }
 880
 881 static int hns3_insert_fdir_filter(struct hns3_hw *hw,
 882                                    struct hns3_fdir_info *fdir_info,
 883                                    struct hns3_fdir_rule_ele *fdir_filter)
 884 {
 885         struct hns3_fdir_key_conf *key;
 886         hash_sig_t sig;
 887         int ret;
 888
 889         key = &fdir_filter->fdir_conf.key_conf;
 890         rte_spinlock_lock(&fdir_info->flows_lock);
 891         sig = rte_hash_crc(key, sizeof(*key), 0);
 892         ret = rte_hash_add_key_with_hash(fdir_info->hash_handle, key, sig);
 893         if (ret < 0) {
 894                 rte_spinlock_unlock(&fdir_info->flows_lock);
 895                 hns3_err(hw, "Hash table full? err:%d(%s)!", ret,
 896                          strerror(-ret));
 897                 return ret;
 898         }
 899
 900         fdir_info->hash_map[ret] = fdir_filter;
 901         TAILQ_INSERT_TAIL(&fdir_info->fdir_list, fdir_filter, entries);
 902         rte_spinlock_unlock(&fdir_info->flows_lock);
 903
 904         return ret;
 905 }
 906
 907 static int hns3_remove_fdir_filter(struct hns3_hw *hw,
 908                                    struct hns3_fdir_info *fdir_info,
 909                                    struct hns3_fdir_key_conf *key)
 910 {
 911         struct hns3_fdir_rule_ele *fdir_filter;
 912         hash_sig_t sig;
 913         int ret;
 914
 915         rte_spinlock_lock(&fdir_info->flows_lock);
 916         sig = rte_hash_crc(key, sizeof(*key), 0);
 917         ret = rte_hash_del_key_with_hash(fdir_info->hash_handle, key, sig);
 918         if (ret < 0) {
 919                 rte_spinlock_unlock(&fdir_info->flows_lock);
 920                 hns3_err(hw, "Delete hash key fail ret=%d", ret);
 921                 return ret;
 922         }
 923
 924         fdir_filter = fdir_info->hash_map[ret];
 925         fdir_info->hash_map[ret] = NULL;
 926         TAILQ_REMOVE(&fdir_info->fdir_list, fdir_filter, entries);
 927         rte_spinlock_unlock(&fdir_info->flows_lock);
 928
 929         rte_free(fdir_filter);
 930
 931         return 0;
 932 }
 933
 934 int hns3_fdir_filter_program(struct hns3_adapter *hns,
 935                              struct hns3_fdir_rule *rule, bool del)
 936 {
 937         struct hns3_pf *pf = &hns->pf;
 938         struct hns3_fdir_info *fdir_info = &pf->fdir;
 939         struct hns3_fdir_rule_ele *node;
 940         struct hns3_hw *hw = &hns->hw;
 941         int ret;
 942
 943         if (del) {
 944                 ret = hns3_fd_tcam_config(hw, true, rule->location, NULL,
 945                                           false);
 946                 if (ret)
 947                         hns3_err(hw, "Failed to delete fdir: %d src_ip:%x "
 948                                  "dst_ip:%x src_port:%d dst_port:%d ret = %d",
 949                                  rule->location,
 950                                  rule->key_conf.spec.src_ip[IP_ADDR_KEY_ID],
 951                                  rule->key_conf.spec.dst_ip[IP_ADDR_KEY_ID],
 952                                  rule->key_conf.spec.src_port,
 953                                  rule->key_conf.spec.dst_port, ret);
 954                 else
 955                         hns3_remove_fdir_filter(hw, fdir_info, &rule->key_conf);
 956
 957                 return ret;
 958         }
 959
 960         ret = hns3_fdir_filter_lookup(fdir_info, &rule->key_conf);
 961         if (ret >= 0) {
 962                 hns3_err(hw, "Conflict with existing fdir loc: %d", ret);
 963                 return -EINVAL;
 964         }
 965
 966         node = rte_zmalloc("hns3 fdir rule", sizeof(struct hns3_fdir_rule_ele),
 967                            0);
 968         if (node == NULL) {
 969                 hns3_err(hw, "Failed to allocate fdir_rule memory");
 970                 return -ENOMEM;
 971         }
 972
 973         rte_memcpy(&node->fdir_conf, rule, sizeof(struct hns3_fdir_rule));
 974         ret = hns3_insert_fdir_filter(hw, fdir_info, node);
 975         if (ret < 0) {
 976                 rte_free(node);
 977                 return ret;
 978         }
 979         rule->location = ret;
 980         node->fdir_conf.location = ret;
 981
 982         rte_spinlock_lock(&fdir_info->flows_lock);
 983         ret = hns3_config_action(hw, rule);
 984         if (!ret)
 985                 ret = hns3_config_key(hns, rule);
 986         rte_spinlock_unlock(&fdir_info->flows_lock);
 987         if (ret) {
 988                 hns3_err(hw, "Failed to config fdir: %d src_ip:%x dst_ip:%x "
 989                          "src_port:%d dst_port:%d ret = %d",
 990                          rule->location,
 991                          rule->key_conf.spec.src_ip[IP_ADDR_KEY_ID],
 992                          rule->key_conf.spec.dst_ip[IP_ADDR_KEY_ID],
 993                          rule->key_conf.spec.src_port,
 994                          rule->key_conf.spec.dst_port, ret);
 995                 (void)hns3_remove_fdir_filter(hw, fdir_info, &rule->key_conf);
 996         }
 997
 998         return ret;
 999 }
1000
1001 /* remove all the flow director filters */
1002 int hns3_clear_all_fdir_filter(struct hns3_adapter *hns)
1003 {
1004         struct hns3_pf *pf = &hns->pf;
1005         struct hns3_fdir_info *fdir_info = &pf->fdir;
1006         struct hns3_fdir_rule_ele *fdir_filter;
1007         struct hns3_hw *hw = &hns->hw;
1008         int ret = 0;
1009
1010         /* flush flow director */
1011         rte_spinlock_lock(&fdir_info->flows_lock);
1012         rte_hash_reset(fdir_info->hash_handle);
1013         rte_spinlock_unlock(&fdir_info->flows_lock);
1014
1015         fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list);
1016         while (fdir_filter) {
1017                 TAILQ_REMOVE(&fdir_info->fdir_list, fdir_filter, entries);
1018                 ret += hns3_fd_tcam_config(hw, true,
1019                                            fdir_filter->fdir_conf.location,
1020                                            NULL, false);
1021                 rte_free(fdir_filter);
1022                 fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list);
1023         }
1024
1025         if (ret) {
1026                 hns3_err(hw, "Fail to delete FDIR filter, ret = %d", ret);
1027                 ret = -EIO;
1028         }
1029         return ret;
1030 }
1031
1032 int hns3_restore_all_fdir_filter(struct hns3_adapter *hns)
1033 {
1034         struct hns3_pf *pf = &hns->pf;
1035         struct hns3_fdir_info *fdir_info = &pf->fdir;
1036         struct hns3_fdir_rule_ele *fdir_filter;
1037         struct hns3_hw *hw = &hns->hw;
1038         bool err = false;
1039         int ret;
1040
1041         TAILQ_FOREACH(fdir_filter, &fdir_info->fdir_list, entries) {
1042                 ret = hns3_config_action(hw, &fdir_filter->fdir_conf);
1043                 if (!ret)
1044                         ret = hns3_config_key(hns, &fdir_filter->fdir_conf);
1045                 if (ret) {
1046                         err = true;
1047                         if (ret == -EBUSY)
1048                                 break;
1049                 }
1050         }
1051
1052         if (err) {
1053                 hns3_err(hw, "Fail to restore FDIR filter, ret = %d", ret);
1054                 return -EIO;
1055         }
1056         return 0;
1057 }
1058
1059 int hns3_get_count(struct hns3_hw *hw, uint32_t id, uint64_t *value)
1060 {
1061         struct hns3_fd_get_cnt_cmd *req;
1062         struct hns3_cmd_desc desc;
1063         int ret;
1064
1065         hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_FD_COUNTER_OP, true);
1066
1067         req = (struct hns3_fd_get_cnt_cmd *)desc.data;
1068         req->stage = HNS3_FD_STAGE_1;
1069         req->index = rte_cpu_to_le_32(id);
1070
1071         ret = hns3_cmd_send(hw, &desc, 1);
1072         if (ret) {
1073                 hns3_err(hw, "Read counter fail, ret=%d", ret);
1074                 return ret;
1075         }
1076
1077         *value = req->value;
1078
1079         return ret;
1080 }