drivers/net/hns3/hns3_cmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2019 Hisilicon Limited.
   3  */
   4
   5 #include <errno.h>
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8 #include <stdio.h>
   9 #include <string.h>
  10 #include <sys/queue.h>
  11 #include <inttypes.h>
  12 #include <unistd.h>
  13 #include <rte_bus_pci.h>
  14 #include <rte_common.h>
  15 #include <rte_cycles.h>
  16 #include <rte_dev.h>
  17 #include <rte_eal.h>
  18 #include <rte_ether.h>
  19 #include <rte_ethdev_driver.h>
  20 #include <rte_ethdev_pci.h>
  21 #include <rte_io.h>
  22
  23 #include "hns3_ethdev.h"
  24 #include "hns3_regs.h"
  25 #include "hns3_logs.h"
  26
  27 #define hns3_is_csq(ring) ((ring)->flag & HNS3_TYPE_CSQ)
  28
  29 #define cmq_ring_to_dev(ring)   (&(ring)->dev->pdev->dev)
  30
  31 static int
  32 hns3_ring_space(struct hns3_cmq_ring *ring)
  33 {
  34         int ntu = ring->next_to_use;
  35         int ntc = ring->next_to_clean;
  36         int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
  37
  38         return ring->desc_num - used - 1;
  39 }
  40
  41 static bool
  42 is_valid_csq_clean_head(struct hns3_cmq_ring *ring, int head)
  43 {
  44         int ntu = ring->next_to_use;
  45         int ntc = ring->next_to_clean;
  46
  47         if (ntu > ntc)
  48                 return head >= ntc && head <= ntu;
  49
  50         return head >= ntc || head <= ntu;
  51 }
  52
  53 /*
  54  * hns3_allocate_dma_mem - Specific memory alloc for command function.
  55  * Malloc a memzone, which is a contiguous portion of physical memory identified
  56  * by a name.
  57  * @ring: pointer to the ring structure
  58  * @size: size of memory requested
  59  * @alignment: what to align the allocation to
  60  */
  61 static int
  62 hns3_allocate_dma_mem(struct hns3_hw *hw, struct hns3_cmq_ring *ring,
  63                       uint64_t size, uint32_t alignment)
  64 {
  65         const struct rte_memzone *mz = NULL;
  66         char z_name[RTE_MEMZONE_NAMESIZE];
  67
  68         snprintf(z_name, sizeof(z_name), "hns3_dma_%" PRIu64, rte_rand());
  69         mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY,
  70                                          RTE_MEMZONE_IOVA_CONTIG, alignment,
  71                                          RTE_PGSIZE_2M);
  72         if (mz == NULL)
  73                 return -ENOMEM;
  74
  75         ring->buf_size = size;
  76         ring->desc = mz->addr;
  77         ring->desc_dma_addr = mz->iova;
  78         ring->zone = (const void *)mz;
  79         hns3_dbg(hw, "memzone %s allocated with physical address: %" PRIu64,
  80                  mz->name, ring->desc_dma_addr);
  81
  82         return 0;
  83 }
  84
  85 static void
  86 hns3_free_dma_mem(struct hns3_hw *hw, struct hns3_cmq_ring *ring)
  87 {
  88         hns3_dbg(hw, "memzone %s to be freed with physical address: %" PRIu64,
  89                  ((const struct rte_memzone *)ring->zone)->name,
  90                  ring->desc_dma_addr);
  91         rte_memzone_free((const struct rte_memzone *)ring->zone);
  92         ring->buf_size = 0;
  93         ring->desc = NULL;
  94         ring->desc_dma_addr = 0;
  95         ring->zone = NULL;
  96 }
  97
  98 static int
  99 hns3_alloc_cmd_desc(struct hns3_hw *hw, struct hns3_cmq_ring *ring)
 100 {
 101         int size  = ring->desc_num * sizeof(struct hns3_cmd_desc);
 102
 103         if (hns3_allocate_dma_mem(hw, ring, size, HNS3_CMD_DESC_ALIGNMENT)) {
 104                 hns3_err(hw, "allocate dma mem failed");
 105                 return -ENOMEM;
 106         }
 107
 108         return 0;
 109 }
 110
 111 static void
 112 hns3_free_cmd_desc(struct hns3_hw *hw, struct hns3_cmq_ring *ring)
 113 {
 114         if (ring->desc)
 115                 hns3_free_dma_mem(hw, ring);
 116 }
 117
 118 static int
 119 hns3_alloc_cmd_queue(struct hns3_hw *hw, int ring_type)
 120 {
 121         struct hns3_cmq_ring *ring =
 122                 (ring_type == HNS3_TYPE_CSQ) ? &hw->cmq.csq : &hw->cmq.crq;
 123         int ret;
 124
 125         ring->ring_type = ring_type;
 126         ring->hw = hw;
 127
 128         ret = hns3_alloc_cmd_desc(hw, ring);
 129         if (ret)
 130                 hns3_err(hw, "descriptor %s alloc error %d",
 131                             (ring_type == HNS3_TYPE_CSQ) ? "CSQ" : "CRQ", ret);
 132
 133         return ret;
 134 }
 135
 136 void
 137 hns3_cmd_reuse_desc(struct hns3_cmd_desc *desc, bool is_read)
 138 {
 139         desc->flag = rte_cpu_to_le_16(HNS3_CMD_FLAG_NO_INTR | HNS3_CMD_FLAG_IN);
 140         if (is_read)
 141                 desc->flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_WR);
 142         else
 143                 desc->flag &= rte_cpu_to_le_16(~HNS3_CMD_FLAG_WR);
 144 }
 145
 146 void
 147 hns3_cmd_setup_basic_desc(struct hns3_cmd_desc *desc,
 148                           enum hns3_opcode_type opcode, bool is_read)
 149 {
 150         memset((void *)desc, 0, sizeof(struct hns3_cmd_desc));
 151         desc->opcode = rte_cpu_to_le_16(opcode);
 152         desc->flag = rte_cpu_to_le_16(HNS3_CMD_FLAG_NO_INTR | HNS3_CMD_FLAG_IN);
 153
 154         if (is_read)
 155                 desc->flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_WR);
 156 }
 157
 158 static void
 159 hns3_cmd_clear_regs(struct hns3_hw *hw)
 160 {
 161         hns3_write_dev(hw, HNS3_CMDQ_TX_ADDR_L_REG, 0);
 162         hns3_write_dev(hw, HNS3_CMDQ_TX_ADDR_H_REG, 0);
 163         hns3_write_dev(hw, HNS3_CMDQ_TX_DEPTH_REG, 0);
 164         hns3_write_dev(hw, HNS3_CMDQ_TX_HEAD_REG, 0);
 165         hns3_write_dev(hw, HNS3_CMDQ_TX_TAIL_REG, 0);
 166         hns3_write_dev(hw, HNS3_CMDQ_RX_ADDR_L_REG, 0);
 167         hns3_write_dev(hw, HNS3_CMDQ_RX_ADDR_H_REG, 0);
 168         hns3_write_dev(hw, HNS3_CMDQ_RX_DEPTH_REG, 0);
 169         hns3_write_dev(hw, HNS3_CMDQ_RX_HEAD_REG, 0);
 170         hns3_write_dev(hw, HNS3_CMDQ_RX_TAIL_REG, 0);
 171 }
 172
 173 static void
 174 hns3_cmd_config_regs(struct hns3_cmq_ring *ring)
 175 {
 176         uint64_t dma = ring->desc_dma_addr;
 177
 178         if (ring->ring_type == HNS3_TYPE_CSQ) {
 179                 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_ADDR_L_REG,
 180                                lower_32_bits(dma));
 181                 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_ADDR_H_REG,
 182                                upper_32_bits(dma));
 183                 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_DEPTH_REG,
 184                                ring->desc_num >> HNS3_NIC_CMQ_DESC_NUM_S |
 185                                HNS3_NIC_SW_RST_RDY);
 186                 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_HEAD_REG, 0);
 187                 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_TAIL_REG, 0);
 188         } else {
 189                 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_ADDR_L_REG,
 190                                lower_32_bits(dma));
 191                 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_ADDR_H_REG,
 192                                upper_32_bits(dma));
 193                 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_DEPTH_REG,
 194                                ring->desc_num >> HNS3_NIC_CMQ_DESC_NUM_S);
 195                 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_HEAD_REG, 0);
 196                 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_TAIL_REG, 0);
 197         }
 198 }
 199
 200 static void
 201 hns3_cmd_init_regs(struct hns3_hw *hw)
 202 {
 203         hns3_cmd_config_regs(&hw->cmq.csq);
 204         hns3_cmd_config_regs(&hw->cmq.crq);
 205 }
 206
 207 static int
 208 hns3_cmd_csq_clean(struct hns3_hw *hw)
 209 {
 210         struct hns3_cmq_ring *csq = &hw->cmq.csq;
 211         uint32_t head;
 212         int clean;
 213
 214         head = hns3_read_dev(hw, HNS3_CMDQ_TX_HEAD_REG);
 215
 216         if (!is_valid_csq_clean_head(csq, head)) {
 217                 hns3_err(hw, "wrong cmd head (%u, %u-%u)", head,
 218                             csq->next_to_use, csq->next_to_clean);
 219                 rte_atomic16_set(&hw->reset.disable_cmd, 1);
 220                 return -EIO;
 221         }
 222
 223         clean = (head - csq->next_to_clean + csq->desc_num) % csq->desc_num;
 224         csq->next_to_clean = head;
 225         return clean;
 226 }
 227
 228 static int
 229 hns3_cmd_csq_done(struct hns3_hw *hw)
 230 {
 231         uint32_t head = hns3_read_dev(hw, HNS3_CMDQ_TX_HEAD_REG);
 232
 233         return head == hw->cmq.csq.next_to_use;
 234 }
 235
 236 static bool
 237 hns3_is_special_opcode(uint16_t opcode)
 238 {
 239         /*
 240          * These commands have several descriptors,
 241          * and use the first one to save opcode and return value.
 242          */
 243         uint16_t spec_opcode[] = {HNS3_OPC_STATS_64_BIT,
 244                                   HNS3_OPC_STATS_32_BIT,
 245                                   HNS3_OPC_STATS_MAC,
 246                                   HNS3_OPC_STATS_MAC_ALL,
 247                                   HNS3_OPC_QUERY_32_BIT_REG,
 248                                   HNS3_OPC_QUERY_64_BIT_REG};
 249         uint32_t i;
 250
 251         for (i = 0; i < ARRAY_SIZE(spec_opcode); i++)
 252                 if (spec_opcode[i] == opcode)
 253                         return true;
 254
 255         return false;
 256 }
 257
 258 static int
 259 hns3_cmd_convert_err_code(uint16_t desc_ret)
 260 {
 261         switch (desc_ret) {
 262         case HNS3_CMD_EXEC_SUCCESS:
 263                 return 0;
 264         case HNS3_CMD_NO_AUTH:
 265                 return -EPERM;
 266         case HNS3_CMD_NOT_SUPPORTED:
 267                 return -EOPNOTSUPP;
 268         case HNS3_CMD_QUEUE_FULL:
 269                 return -EXFULL;
 270         case HNS3_CMD_NEXT_ERR:
 271                 return -ENOSR;
 272         case HNS3_CMD_UNEXE_ERR:
 273                 return -ENOTBLK;
 274         case HNS3_CMD_PARA_ERR:
 275                 return -EINVAL;
 276         case HNS3_CMD_RESULT_ERR:
 277                 return -ERANGE;
 278         case HNS3_CMD_TIMEOUT:
 279                 return -ETIME;
 280         case HNS3_CMD_HILINK_ERR:
 281                 return -ENOLINK;
 282         case HNS3_CMD_QUEUE_ILLEGAL:
 283                 return -ENXIO;
 284         case HNS3_CMD_INVALID:
 285                 return -EBADR;
 286         default:
 287                 return -EIO;
 288         }
 289 }
 290
 291 static int
 292 hns3_cmd_get_hardware_reply(struct hns3_hw *hw,
 293                             struct hns3_cmd_desc *desc, int num, int ntc)
 294 {
 295         uint16_t opcode, desc_ret;
 296         int current_ntc = ntc;
 297         int handle;
 298
 299         opcode = rte_le_to_cpu_16(desc[0].opcode);
 300         for (handle = 0; handle < num; handle++) {
 301                 /* Get the result of hardware write back */
 302                 desc[handle] = hw->cmq.csq.desc[current_ntc];
 303
 304                 current_ntc++;
 305                 if (current_ntc == hw->cmq.csq.desc_num)
 306                         current_ntc = 0;
 307         }
 308
 309         if (likely(!hns3_is_special_opcode(opcode)))
 310                 desc_ret = rte_le_to_cpu_16(desc[num - 1].retval);
 311         else
 312                 desc_ret = rte_le_to_cpu_16(desc[0].retval);
 313
 314         hw->cmq.last_status = desc_ret;
 315         return hns3_cmd_convert_err_code(desc_ret);
 316 }
 317
 318 static int hns3_cmd_poll_reply(struct hns3_hw *hw)
 319 {
 320         uint32_t timeout = 0;
 321
 322         do {
 323                 if (hns3_cmd_csq_done(hw))
 324                         return 0;
 325
 326                 if (rte_atomic16_read(&hw->reset.disable_cmd)) {
 327                         hns3_err(hw,
 328                                  "Don't wait for reply because of disable_cmd");
 329                         return -EBUSY;
 330                 }
 331
 332                 rte_delay_us(1);
 333                 timeout++;
 334         } while (timeout < hw->cmq.tx_timeout);
 335         hns3_err(hw, "Wait for reply timeout");
 336         return -EBADE;
 337 }
 338
 339 /*
 340  * hns3_cmd_send - send command to command queue
 341  * @hw: pointer to the hw struct
 342  * @desc: prefilled descriptor for describing the command
 343  * @num : the number of descriptors to be sent
 344  *
 345  * This is the main send command for command queue, it
 346  * sends the queue, cleans the queue, etc
 347  */
 348 int
 349 hns3_cmd_send(struct hns3_hw *hw, struct hns3_cmd_desc *desc, int num)
 350 {
 351         struct hns3_cmd_desc *desc_to_use;
 352         int handle = 0;
 353         int retval;
 354         uint32_t ntc;
 355
 356         if (rte_atomic16_read(&hw->reset.disable_cmd))
 357                 return -EBUSY;
 358
 359         rte_spinlock_lock(&hw->cmq.csq.lock);
 360
 361         /* Clean the command send queue */
 362         retval = hns3_cmd_csq_clean(hw);
 363         if (retval < 0) {
 364                 rte_spinlock_unlock(&hw->cmq.csq.lock);
 365                 return retval;
 366         }
 367
 368         if (num > hns3_ring_space(&hw->cmq.csq)) {
 369                 rte_spinlock_unlock(&hw->cmq.csq.lock);
 370                 return -ENOMEM;
 371         }
 372
 373         /*
 374          * Record the location of desc in the ring for this time
 375          * which will be use for hardware to write back
 376          */
 377         ntc = hw->cmq.csq.next_to_use;
 378
 379         while (handle < num) {
 380                 desc_to_use = &hw->cmq.csq.desc[hw->cmq.csq.next_to_use];
 381                 *desc_to_use = desc[handle];
 382                 (hw->cmq.csq.next_to_use)++;
 383                 if (hw->cmq.csq.next_to_use == hw->cmq.csq.desc_num)
 384                         hw->cmq.csq.next_to_use = 0;
 385                 handle++;
 386         }
 387
 388         /* Write to hardware */
 389         hns3_write_dev(hw, HNS3_CMDQ_TX_TAIL_REG, hw->cmq.csq.next_to_use);
 390
 391         /*
 392          * If the command is sync, wait for the firmware to write back,
 393          * if multi descriptors to be sent, use the first one to check.
 394          */
 395         if (HNS3_CMD_SEND_SYNC(rte_le_to_cpu_16(desc->flag))) {
 396                 retval = hns3_cmd_poll_reply(hw);
 397                 if (!retval)
 398                         retval = hns3_cmd_get_hardware_reply(hw, desc, num,
 399                                                              ntc);
 400         }
 401
 402         rte_spinlock_unlock(&hw->cmq.csq.lock);
 403         return retval;
 404 }
 405
 406 static enum hns3_cmd_status
 407 hns3_cmd_query_firmware_version(struct hns3_hw *hw, uint32_t *version)
 408 {
 409         struct hns3_query_version_cmd *resp;
 410         struct hns3_cmd_desc desc;
 411         int ret;
 412
 413         hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_QUERY_FW_VER, 1);
 414         resp = (struct hns3_query_version_cmd *)desc.data;
 415
 416         /* Initialize the cmd function */
 417         ret = hns3_cmd_send(hw, &desc, 1);
 418         if (ret == 0)
 419                 *version = rte_le_to_cpu_32(resp->firmware);
 420
 421         return ret;
 422 }
 423
 424 int
 425 hns3_cmd_init_queue(struct hns3_hw *hw)
 426 {
 427         int ret;
 428
 429         /* Setup the lock for command queue */
 430         rte_spinlock_init(&hw->cmq.csq.lock);
 431         rte_spinlock_init(&hw->cmq.crq.lock);
 432
 433         /*
 434          * Clear up all command register,
 435          * in case there are some residual values
 436          */
 437         hns3_cmd_clear_regs(hw);
 438
 439         /* Setup the queue entries for use cmd queue */
 440         hw->cmq.csq.desc_num = HNS3_NIC_CMQ_DESC_NUM;
 441         hw->cmq.crq.desc_num = HNS3_NIC_CMQ_DESC_NUM;
 442
 443         /* Setup Tx write back timeout */
 444         hw->cmq.tx_timeout = HNS3_CMDQ_TX_TIMEOUT;
 445
 446         /* Setup queue rings */
 447         ret = hns3_alloc_cmd_queue(hw, HNS3_TYPE_CSQ);
 448         if (ret) {
 449                 PMD_INIT_LOG(ERR, "CSQ ring setup error %d", ret);
 450                 return ret;
 451         }
 452
 453         ret = hns3_alloc_cmd_queue(hw, HNS3_TYPE_CRQ);
 454         if (ret) {
 455                 PMD_INIT_LOG(ERR, "CRQ ring setup error %d", ret);
 456                 goto err_crq;
 457         }
 458
 459         return 0;
 460
 461 err_crq:
 462         hns3_free_cmd_desc(hw, &hw->cmq.csq);
 463
 464         return ret;
 465 }
 466
 467 int
 468 hns3_cmd_init(struct hns3_hw *hw)
 469 {
 470         int ret;
 471
 472         rte_spinlock_lock(&hw->cmq.csq.lock);
 473         rte_spinlock_lock(&hw->cmq.crq.lock);
 474
 475         hw->cmq.csq.next_to_clean = 0;
 476         hw->cmq.csq.next_to_use = 0;
 477         hw->cmq.crq.next_to_clean = 0;
 478         hw->cmq.crq.next_to_use = 0;
 479         hns3_cmd_init_regs(hw);
 480
 481         rte_spinlock_unlock(&hw->cmq.crq.lock);
 482         rte_spinlock_unlock(&hw->cmq.csq.lock);
 483
 484         rte_atomic16_clear(&hw->reset.disable_cmd);
 485
 486         ret = hns3_cmd_query_firmware_version(hw, &hw->fw_version);
 487         if (ret) {
 488                 PMD_INIT_LOG(ERR, "firmware version query failed %d", ret);
 489                 goto err_cmd_init;
 490         }
 491
 492         PMD_INIT_LOG(INFO, "The firmware version is %08x", hw->fw_version);
 493
 494         return 0;
 495
 496 err_cmd_init:
 497         hns3_cmd_uninit(hw);
 498         return ret;
 499 }
 500
 501 static void
 502 hns3_destroy_queue(struct hns3_hw *hw, struct hns3_cmq_ring *ring)
 503 {
 504         rte_spinlock_lock(&ring->lock);
 505
 506         hns3_free_cmd_desc(hw, ring);
 507
 508         rte_spinlock_unlock(&ring->lock);
 509 }
 510
 511 void
 512 hns3_cmd_destroy_queue(struct hns3_hw *hw)
 513 {
 514         hns3_destroy_queue(hw, &hw->cmq.csq);
 515         hns3_destroy_queue(hw, &hw->cmq.crq);
 516 }
 517
 518 void
 519 hns3_cmd_uninit(struct hns3_hw *hw)
 520 {
 521         rte_spinlock_lock(&hw->cmq.csq.lock);
 522         rte_spinlock_lock(&hw->cmq.crq.lock);
 523         rte_atomic16_set(&hw->reset.disable_cmd, 1);
 524         hns3_cmd_clear_regs(hw);
 525         rte_spinlock_unlock(&hw->cmq.crq.lock);
 526         rte_spinlock_unlock(&hw->cmq.csq.lock);
 527 }