1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018-2019 Hisilicon Limited.
5 #include <rte_ethdev_pci.h>
8 #include "hns3_ethdev.h"
10 #include "hns3_intr.h"
11 #include "hns3_logs.h"
13 #define hns3_is_csq(ring) ((ring)->flag & HNS3_TYPE_CSQ)
15 #define cmq_ring_to_dev(ring) (&(ring)->dev->pdev->dev)
18 hns3_ring_space(struct hns3_cmq_ring *ring)
20 int ntu = ring->next_to_use;
21 int ntc = ring->next_to_clean;
22 int used = (ntu - ntc + ring->desc_num) % ring->desc_num;
24 return ring->desc_num - used - 1;
28 is_valid_csq_clean_head(struct hns3_cmq_ring *ring, int head)
30 int ntu = ring->next_to_use;
31 int ntc = ring->next_to_clean;
34 return head >= ntc && head <= ntu;
36 return head >= ntc || head <= ntu;
40 * hns3_allocate_dma_mem - Specific memory alloc for command function.
41 * Malloc a memzone, which is a contiguous portion of physical memory identified
43 * @ring: pointer to the ring structure
44 * @size: size of memory requested
45 * @alignment: what to align the allocation to
48 hns3_allocate_dma_mem(struct hns3_hw *hw, struct hns3_cmq_ring *ring,
49 uint64_t size, uint32_t alignment)
51 const struct rte_memzone *mz = NULL;
52 char z_name[RTE_MEMZONE_NAMESIZE];
54 snprintf(z_name, sizeof(z_name), "hns3_dma_%" PRIu64, rte_rand());
55 mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY,
56 RTE_MEMZONE_IOVA_CONTIG, alignment,
61 ring->buf_size = size;
62 ring->desc = mz->addr;
63 ring->desc_dma_addr = mz->iova;
64 ring->zone = (const void *)mz;
65 hns3_dbg(hw, "memzone %s allocated with physical address: %" PRIu64,
66 mz->name, ring->desc_dma_addr);
72 hns3_free_dma_mem(struct hns3_hw *hw, struct hns3_cmq_ring *ring)
74 hns3_dbg(hw, "memzone %s to be freed with physical address: %" PRIu64,
75 ((const struct rte_memzone *)ring->zone)->name,
77 rte_memzone_free((const struct rte_memzone *)ring->zone);
80 ring->desc_dma_addr = 0;
85 hns3_alloc_cmd_desc(struct hns3_hw *hw, struct hns3_cmq_ring *ring)
87 int size = ring->desc_num * sizeof(struct hns3_cmd_desc);
89 if (hns3_allocate_dma_mem(hw, ring, size, HNS3_CMD_DESC_ALIGNMENT)) {
90 hns3_err(hw, "allocate dma mem failed");
98 hns3_free_cmd_desc(struct hns3_hw *hw, struct hns3_cmq_ring *ring)
101 hns3_free_dma_mem(hw, ring);
105 hns3_alloc_cmd_queue(struct hns3_hw *hw, int ring_type)
107 struct hns3_cmq_ring *ring =
108 (ring_type == HNS3_TYPE_CSQ) ? &hw->cmq.csq : &hw->cmq.crq;
111 ring->ring_type = ring_type;
114 ret = hns3_alloc_cmd_desc(hw, ring);
116 hns3_err(hw, "descriptor %s alloc error %d",
117 (ring_type == HNS3_TYPE_CSQ) ? "CSQ" : "CRQ", ret);
123 hns3_cmd_reuse_desc(struct hns3_cmd_desc *desc, bool is_read)
125 desc->flag = rte_cpu_to_le_16(HNS3_CMD_FLAG_NO_INTR | HNS3_CMD_FLAG_IN);
127 desc->flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_WR);
129 desc->flag &= rte_cpu_to_le_16(~HNS3_CMD_FLAG_WR);
133 hns3_cmd_setup_basic_desc(struct hns3_cmd_desc *desc,
134 enum hns3_opcode_type opcode, bool is_read)
136 memset((void *)desc, 0, sizeof(struct hns3_cmd_desc));
137 desc->opcode = rte_cpu_to_le_16(opcode);
138 desc->flag = rte_cpu_to_le_16(HNS3_CMD_FLAG_NO_INTR | HNS3_CMD_FLAG_IN);
141 desc->flag |= rte_cpu_to_le_16(HNS3_CMD_FLAG_WR);
145 hns3_cmd_clear_regs(struct hns3_hw *hw)
147 hns3_write_dev(hw, HNS3_CMDQ_TX_ADDR_L_REG, 0);
148 hns3_write_dev(hw, HNS3_CMDQ_TX_ADDR_H_REG, 0);
149 hns3_write_dev(hw, HNS3_CMDQ_TX_DEPTH_REG, 0);
150 hns3_write_dev(hw, HNS3_CMDQ_TX_HEAD_REG, 0);
151 hns3_write_dev(hw, HNS3_CMDQ_TX_TAIL_REG, 0);
152 hns3_write_dev(hw, HNS3_CMDQ_RX_ADDR_L_REG, 0);
153 hns3_write_dev(hw, HNS3_CMDQ_RX_ADDR_H_REG, 0);
154 hns3_write_dev(hw, HNS3_CMDQ_RX_DEPTH_REG, 0);
155 hns3_write_dev(hw, HNS3_CMDQ_RX_HEAD_REG, 0);
156 hns3_write_dev(hw, HNS3_CMDQ_RX_TAIL_REG, 0);
160 hns3_cmd_config_regs(struct hns3_cmq_ring *ring)
162 uint64_t dma = ring->desc_dma_addr;
164 if (ring->ring_type == HNS3_TYPE_CSQ) {
165 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_ADDR_L_REG,
167 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_ADDR_H_REG,
169 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_DEPTH_REG,
170 ring->desc_num >> HNS3_NIC_CMQ_DESC_NUM_S |
171 HNS3_NIC_SW_RST_RDY);
172 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_HEAD_REG, 0);
173 hns3_write_dev(ring->hw, HNS3_CMDQ_TX_TAIL_REG, 0);
175 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_ADDR_L_REG,
177 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_ADDR_H_REG,
179 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_DEPTH_REG,
180 ring->desc_num >> HNS3_NIC_CMQ_DESC_NUM_S);
181 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_HEAD_REG, 0);
182 hns3_write_dev(ring->hw, HNS3_CMDQ_RX_TAIL_REG, 0);
187 hns3_cmd_init_regs(struct hns3_hw *hw)
189 hns3_cmd_config_regs(&hw->cmq.csq);
190 hns3_cmd_config_regs(&hw->cmq.crq);
194 hns3_cmd_csq_clean(struct hns3_hw *hw)
196 struct hns3_cmq_ring *csq = &hw->cmq.csq;
200 head = hns3_read_dev(hw, HNS3_CMDQ_TX_HEAD_REG);
201 if (!is_valid_csq_clean_head(csq, head)) {
202 hns3_err(hw, "wrong cmd head (%u, %u-%u)", head,
203 csq->next_to_use, csq->next_to_clean);
204 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
205 rte_atomic16_set(&hw->reset.disable_cmd, 1);
206 hns3_schedule_delayed_reset(HNS3_DEV_HW_TO_ADAPTER(hw));
212 clean = (head - csq->next_to_clean + csq->desc_num) % csq->desc_num;
213 csq->next_to_clean = head;
218 hns3_cmd_csq_done(struct hns3_hw *hw)
220 uint32_t head = hns3_read_dev(hw, HNS3_CMDQ_TX_HEAD_REG);
222 return head == hw->cmq.csq.next_to_use;
226 hns3_is_special_opcode(uint16_t opcode)
229 * These commands have several descriptors,
230 * and use the first one to save opcode and return value.
232 uint16_t spec_opcode[] = {HNS3_OPC_STATS_64_BIT,
233 HNS3_OPC_STATS_32_BIT,
235 HNS3_OPC_STATS_MAC_ALL,
236 HNS3_OPC_QUERY_32_BIT_REG,
237 HNS3_OPC_QUERY_64_BIT_REG};
240 for (i = 0; i < ARRAY_SIZE(spec_opcode); i++)
241 if (spec_opcode[i] == opcode)
248 hns3_cmd_convert_err_code(uint16_t desc_ret)
251 case HNS3_CMD_EXEC_SUCCESS:
253 case HNS3_CMD_NO_AUTH:
255 case HNS3_CMD_NOT_SUPPORTED:
257 case HNS3_CMD_QUEUE_FULL:
259 case HNS3_CMD_NEXT_ERR:
261 case HNS3_CMD_UNEXE_ERR:
263 case HNS3_CMD_PARA_ERR:
265 case HNS3_CMD_RESULT_ERR:
267 case HNS3_CMD_TIMEOUT:
269 case HNS3_CMD_HILINK_ERR:
271 case HNS3_CMD_QUEUE_ILLEGAL:
273 case HNS3_CMD_INVALID:
281 hns3_cmd_get_hardware_reply(struct hns3_hw *hw,
282 struct hns3_cmd_desc *desc, int num, int ntc)
284 uint16_t opcode, desc_ret;
285 int current_ntc = ntc;
288 opcode = rte_le_to_cpu_16(desc[0].opcode);
289 for (handle = 0; handle < num; handle++) {
290 /* Get the result of hardware write back */
291 desc[handle] = hw->cmq.csq.desc[current_ntc];
294 if (current_ntc == hw->cmq.csq.desc_num)
298 if (likely(!hns3_is_special_opcode(opcode)))
299 desc_ret = rte_le_to_cpu_16(desc[num - 1].retval);
301 desc_ret = rte_le_to_cpu_16(desc[0].retval);
303 hw->cmq.last_status = desc_ret;
304 return hns3_cmd_convert_err_code(desc_ret);
307 static int hns3_cmd_poll_reply(struct hns3_hw *hw)
309 struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw);
310 uint32_t timeout = 0;
313 if (hns3_cmd_csq_done(hw))
316 if (rte_atomic16_read(&hw->reset.disable_cmd)) {
318 "Don't wait for reply because of disable_cmd");
322 if (is_reset_pending(hns)) {
323 hns3_err(hw, "Don't wait for reply because of reset pending");
329 } while (timeout < hw->cmq.tx_timeout);
330 hns3_err(hw, "Wait for reply timeout");
335 * hns3_cmd_send - send command to command queue
338 * pointer to the hw struct
340 * prefilled descriptor for describing the command
342 * the number of descriptors to be sent
344 * - -EBUSY if detect device is in resetting
345 * - -EIO if detect cmd csq corrupted (due to reset) or
346 * there is reset pending
347 * - -ENOMEM/-ETIME/...(Non-Zero) if other error case
348 * - Zero if operation completed successfully
350 * Note -BUSY/-EIO only used in reset case
352 * Note this is the main send command for command queue, it
353 * sends the queue, cleans the queue, etc
356 hns3_cmd_send(struct hns3_hw *hw, struct hns3_cmd_desc *desc, int num)
358 struct hns3_cmd_desc *desc_to_use;
363 if (rte_atomic16_read(&hw->reset.disable_cmd))
366 rte_spinlock_lock(&hw->cmq.csq.lock);
368 /* Clean the command send queue */
369 retval = hns3_cmd_csq_clean(hw);
371 rte_spinlock_unlock(&hw->cmq.csq.lock);
375 if (num > hns3_ring_space(&hw->cmq.csq)) {
376 rte_spinlock_unlock(&hw->cmq.csq.lock);
381 * Record the location of desc in the ring for this time
382 * which will be use for hardware to write back
384 ntc = hw->cmq.csq.next_to_use;
386 while (handle < num) {
387 desc_to_use = &hw->cmq.csq.desc[hw->cmq.csq.next_to_use];
388 *desc_to_use = desc[handle];
389 (hw->cmq.csq.next_to_use)++;
390 if (hw->cmq.csq.next_to_use == hw->cmq.csq.desc_num)
391 hw->cmq.csq.next_to_use = 0;
395 /* Write to hardware */
396 hns3_write_dev(hw, HNS3_CMDQ_TX_TAIL_REG, hw->cmq.csq.next_to_use);
399 * If the command is sync, wait for the firmware to write back,
400 * if multi descriptors to be sent, use the first one to check.
402 if (HNS3_CMD_SEND_SYNC(rte_le_to_cpu_16(desc->flag))) {
403 retval = hns3_cmd_poll_reply(hw);
405 retval = hns3_cmd_get_hardware_reply(hw, desc, num,
409 rte_spinlock_unlock(&hw->cmq.csq.lock);
413 static void hns3_parse_capability(struct hns3_hw *hw,
414 struct hns3_query_version_cmd *cmd)
416 uint32_t caps = rte_le_to_cpu_32(cmd->caps[0]);
418 if (hns3_get_bit(caps, HNS3_CAPS_UDP_GSO_B))
419 hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_UDP_GSO_B, 1);
420 if (hns3_get_bit(caps, HNS3_CAPS_FD_QUEUE_REGION_B))
421 hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_FD_QUEUE_REGION_B,
423 if (hns3_get_bit(caps, HNS3_CAPS_PTP_B))
424 hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_PTP_B, 1);
425 if (hns3_get_bit(caps, HNS3_CAPS_TX_PUSH_B))
426 hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_TX_PUSH_B, 1);
427 if (hns3_get_bit(caps, HNS3_CAPS_PHY_IMP_B))
428 hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_COPPER_B, 1);
429 if (hns3_get_bit(caps, HNS3_CAPS_TQP_TXRX_INDEP_B))
430 hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_INDEP_TXRX_B, 1);
431 if (hns3_get_bit(caps, HNS3_CAPS_STASH_B))
432 hns3_set_bit(hw->capability, HNS3_DEV_SUPPORT_STASH_B, 1);
435 static enum hns3_cmd_status
436 hns3_cmd_query_firmware_version_and_capability(struct hns3_hw *hw)
438 struct hns3_query_version_cmd *resp;
439 struct hns3_cmd_desc desc;
442 hns3_cmd_setup_basic_desc(&desc, HNS3_OPC_QUERY_FW_VER, 1);
443 resp = (struct hns3_query_version_cmd *)desc.data;
445 /* Initialize the cmd function */
446 ret = hns3_cmd_send(hw, &desc, 1);
450 hw->fw_version = rte_le_to_cpu_32(resp->firmware);
451 hns3_parse_capability(hw, resp);
457 hns3_cmd_init_queue(struct hns3_hw *hw)
461 /* Setup the lock for command queue */
462 rte_spinlock_init(&hw->cmq.csq.lock);
463 rte_spinlock_init(&hw->cmq.crq.lock);
466 * Clear up all command register,
467 * in case there are some residual values
469 hns3_cmd_clear_regs(hw);
471 /* Setup the queue entries for use cmd queue */
472 hw->cmq.csq.desc_num = HNS3_NIC_CMQ_DESC_NUM;
473 hw->cmq.crq.desc_num = HNS3_NIC_CMQ_DESC_NUM;
475 /* Setup Tx write back timeout */
476 hw->cmq.tx_timeout = HNS3_CMDQ_TX_TIMEOUT;
478 /* Setup queue rings */
479 ret = hns3_alloc_cmd_queue(hw, HNS3_TYPE_CSQ);
481 PMD_INIT_LOG(ERR, "CSQ ring setup error %d", ret);
485 ret = hns3_alloc_cmd_queue(hw, HNS3_TYPE_CRQ);
487 PMD_INIT_LOG(ERR, "CRQ ring setup error %d", ret);
494 hns3_free_cmd_desc(hw, &hw->cmq.csq);
500 hns3_cmd_init(struct hns3_hw *hw)
505 rte_spinlock_lock(&hw->cmq.csq.lock);
506 rte_spinlock_lock(&hw->cmq.crq.lock);
508 hw->cmq.csq.next_to_clean = 0;
509 hw->cmq.csq.next_to_use = 0;
510 hw->cmq.crq.next_to_clean = 0;
511 hw->cmq.crq.next_to_use = 0;
512 hw->mbx_resp.head = 0;
513 hw->mbx_resp.tail = 0;
514 hw->mbx_resp.lost = 0;
515 hns3_cmd_init_regs(hw);
517 rte_spinlock_unlock(&hw->cmq.crq.lock);
518 rte_spinlock_unlock(&hw->cmq.csq.lock);
521 * Check if there is new reset pending, because the higher level
522 * reset may happen when lower level reset is being processed.
524 if (is_reset_pending(HNS3_DEV_HW_TO_ADAPTER(hw))) {
525 PMD_INIT_LOG(ERR, "New reset pending, keep disable cmd");
529 rte_atomic16_clear(&hw->reset.disable_cmd);
531 ret = hns3_cmd_query_firmware_version_and_capability(hw);
533 PMD_INIT_LOG(ERR, "firmware version query failed %d", ret);
537 version = hw->fw_version;
538 PMD_INIT_LOG(INFO, "The firmware version is %lu.%lu.%lu.%lu",
539 hns3_get_field(version, HNS3_FW_VERSION_BYTE3_M,
540 HNS3_FW_VERSION_BYTE3_S),
541 hns3_get_field(version, HNS3_FW_VERSION_BYTE2_M,
542 HNS3_FW_VERSION_BYTE2_S),
543 hns3_get_field(version, HNS3_FW_VERSION_BYTE1_M,
544 HNS3_FW_VERSION_BYTE1_S),
545 hns3_get_field(version, HNS3_FW_VERSION_BYTE0_M,
546 HNS3_FW_VERSION_BYTE0_S));
551 rte_atomic16_set(&hw->reset.disable_cmd, 1);
556 hns3_destroy_queue(struct hns3_hw *hw, struct hns3_cmq_ring *ring)
558 rte_spinlock_lock(&ring->lock);
560 hns3_free_cmd_desc(hw, ring);
562 rte_spinlock_unlock(&ring->lock);
566 hns3_cmd_destroy_queue(struct hns3_hw *hw)
568 hns3_destroy_queue(hw, &hw->cmq.csq);
569 hns3_destroy_queue(hw, &hw->cmq.crq);
573 hns3_cmd_uninit(struct hns3_hw *hw)
575 rte_spinlock_lock(&hw->cmq.csq.lock);
576 rte_spinlock_lock(&hw->cmq.crq.lock);
577 rte_atomic16_set(&hw->reset.disable_cmd, 1);
578 hns3_cmd_clear_regs(hw);
579 rte_spinlock_unlock(&hw->cmq.crq.lock);
580 rte_spinlock_unlock(&hw->cmq.csq.lock);