1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
13 #include <rte_bus_pci.h>
14 #include <rte_atomic.h>
15 #include <rte_byteorder.h>
18 #include <rte_spinlock.h>
19 #include <rte_crypto_sym.h>
20 #include <rte_cryptodev.h>
23 #define MAX_HW_QUEUES 5
25 /**< CCP Register Mappings */
26 #define Q_MASK_REG 0x000
27 #define TRNG_OUT_REG 0x00c
29 /* CCP Version 5 Specifics */
30 #define CMD_QUEUE_MASK_OFFSET 0x00
31 #define CMD_QUEUE_PRIO_OFFSET 0x04
32 #define CMD_REQID_CONFIG_OFFSET 0x08
33 #define CMD_CMD_TIMEOUT_OFFSET 0x10
34 #define LSB_PUBLIC_MASK_LO_OFFSET 0x18
35 #define LSB_PUBLIC_MASK_HI_OFFSET 0x1C
36 #define LSB_PRIVATE_MASK_LO_OFFSET 0x20
37 #define LSB_PRIVATE_MASK_HI_OFFSET 0x24
39 #define CMD_Q_CONTROL_BASE 0x0000
40 #define CMD_Q_TAIL_LO_BASE 0x0004
41 #define CMD_Q_HEAD_LO_BASE 0x0008
42 #define CMD_Q_INT_ENABLE_BASE 0x000C
43 #define CMD_Q_INTERRUPT_STATUS_BASE 0x0010
45 #define CMD_Q_STATUS_BASE 0x0100
46 #define CMD_Q_INT_STATUS_BASE 0x0104
48 #define CMD_CONFIG_0_OFFSET 0x6000
49 #define CMD_TRNG_CTL_OFFSET 0x6008
50 #define CMD_AES_MASK_OFFSET 0x6010
51 #define CMD_CLK_GATE_CTL_OFFSET 0x603C
53 /* Address offset between two virtual queue registers */
54 #define CMD_Q_STATUS_INCR 0x1000
58 #define CMD_Q_SIZE 0x1F
60 #define COMMANDS_PER_QUEUE 2048
62 #define QUEUE_SIZE_VAL ((ffs(COMMANDS_PER_QUEUE) - 2) & \
64 #define Q_DESC_SIZE sizeof(struct ccp_desc)
65 #define Q_SIZE(n) (COMMANDS_PER_QUEUE*(n))
67 #define INT_COMPLETION 0x1
69 #define INT_QUEUE_STOPPED 0x4
70 #define ALL_INTERRUPTS (INT_COMPLETION| \
74 #define LSB_REGION_WIDTH 5
78 #define LSB_ITEM_SIZE 32
79 #define SLSB_MAP_SIZE (MAX_LSB_CNT * LSB_SIZE)
81 /* General CCP Defines */
83 #define CCP_SB_BYTES 32
87 BITS_PER_WORD = sizeof(unsigned long) * CHAR_BIT
90 #define WORD_OFFSET(b) ((b) / BITS_PER_WORD)
91 #define BIT_OFFSET(b) ((b) % BITS_PER_WORD)
93 #define CCP_DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
94 #define CCP_BITMAP_SIZE(nr) \
95 CCP_DIV_ROUND_UP(nr, CHAR_BIT * sizeof(unsigned long))
97 #define CCP_BITMAP_FIRST_WORD_MASK(start) \
98 (~0UL << ((start) & (BITS_PER_WORD - 1)))
99 #define CCP_BITMAP_LAST_WORD_MASK(nbits) \
100 (~0UL >> (-(nbits) & (BITS_PER_WORD - 1)))
102 #define __ccp_round_mask(x, y) ((typeof(x))((y)-1))
103 #define ccp_round_down(x, y) ((x) & ~__ccp_round_mask(x, y))
105 /** CCP registers Write/Read */
107 static inline void ccp_pci_reg_write(void *base, int offset,
110 volatile void *reg_addr = ((uint8_t *)base + offset);
112 rte_write32((rte_cpu_to_le_32(value)), reg_addr);
115 static inline uint32_t ccp_pci_reg_read(void *base, int offset)
117 volatile void *reg_addr = ((uint8_t *)base + offset);
119 return rte_le_to_cpu_32(rte_read32(reg_addr));
122 #define CCP_READ_REG(hw_addr, reg_offset) \
123 ccp_pci_reg_read(hw_addr, reg_offset)
125 #define CCP_WRITE_REG(hw_addr, reg_offset, value) \
126 ccp_pci_reg_write(hw_addr, reg_offset, value)
128 TAILQ_HEAD(ccp_list, ccp_device);
130 extern struct ccp_list ccp_list;
135 enum ccp_device_version {
141 * A structure describing a CCP command queue.
144 struct ccp_device *dev;
145 char memz_name[RTE_MEMZONE_NAMESIZE];
147 rte_atomic64_t free_slots;
148 /**< available free slots updated from enq/deq calls */
150 /* Queue identifier */
151 uint64_t id; /**< queue id */
152 uint64_t qidx; /**< queue index */
153 uint64_t qsize; /**< queue size */
156 struct ccp_desc *qbase_desc;
158 phys_addr_t qbase_phys_addr;
159 /**< queue-page registers addr */
163 /**< queue ctrl reg */
166 /**< lsb region assigned to queue */
167 unsigned long lsbmask;
168 /**< lsb regions queue can access */
169 unsigned long lsbmap[CCP_BITMAP_SIZE(LSB_SIZE)];
170 /**< all lsb resources which queue is using */
172 /**< lsb assigned for queue */
174 /**< lsb assigned for iv */
176 /**< lsb assigned for sha ctx */
178 /**< lsb assigned for hmac ctx */
179 } ____cacheline_aligned;
182 * A structure describing a CCP device.
185 TAILQ_ENTRY(ccp_device) next;
187 /**< ccp dev id on platform */
188 struct ccp_queue cmd_q[MAX_HW_QUEUES];
191 /**< no. of ccp Queues */
192 struct rte_pci_device pci;
193 /**< ccp pci identifier */
194 unsigned long lsbmap[CCP_BITMAP_SIZE(SLSB_MAP_SIZE)];
195 /**< shared lsb mask of ccp */
196 rte_spinlock_t lsb_lock;
197 /**< protection for shared lsb region allocation */
199 /**< current queue index */
200 } __rte_cache_aligned;
202 /**< CCP H/W engine related */
204 * ccp_engine - CCP operation identifiers
206 * @CCP_ENGINE_AES: AES operation
207 * @CCP_ENGINE_XTS_AES: 128-bit XTS AES operation
208 * @CCP_ENGINE_3DES: DES/3DES operation
209 * @CCP_ENGINE_SHA: SHA operation
210 * @CCP_ENGINE_RSA: RSA operation
211 * @CCP_ENGINE_PASSTHRU: pass-through operation
212 * @CCP_ENGINE_ZLIB_DECOMPRESS: unused
213 * @CCP_ENGINE_ECC: ECC operation
217 CCP_ENGINE_XTS_AES_128,
222 CCP_ENGINE_ZLIB_DECOMPRESS,
227 /* Passthru engine */
229 * ccp_passthru_bitwise - type of bitwise passthru operation
231 * @CCP_PASSTHRU_BITWISE_NOOP: no bitwise operation performed
232 * @CCP_PASSTHRU_BITWISE_AND: perform bitwise AND of src with mask
233 * @CCP_PASSTHRU_BITWISE_OR: perform bitwise OR of src with mask
234 * @CCP_PASSTHRU_BITWISE_XOR: perform bitwise XOR of src with mask
235 * @CCP_PASSTHRU_BITWISE_MASK: overwrite with mask
237 enum ccp_passthru_bitwise {
238 CCP_PASSTHRU_BITWISE_NOOP = 0,
239 CCP_PASSTHRU_BITWISE_AND,
240 CCP_PASSTHRU_BITWISE_OR,
241 CCP_PASSTHRU_BITWISE_XOR,
242 CCP_PASSTHRU_BITWISE_MASK,
243 CCP_PASSTHRU_BITWISE__LAST,
247 * ccp_passthru_byteswap - type of byteswap passthru operation
249 * @CCP_PASSTHRU_BYTESWAP_NOOP: no byte swapping performed
250 * @CCP_PASSTHRU_BYTESWAP_32BIT: swap bytes within 32-bit words
251 * @CCP_PASSTHRU_BYTESWAP_256BIT: swap bytes within 256-bit words
253 enum ccp_passthru_byteswap {
254 CCP_PASSTHRU_BYTESWAP_NOOP = 0,
255 CCP_PASSTHRU_BYTESWAP_32BIT,
256 CCP_PASSTHRU_BYTESWAP_256BIT,
257 CCP_PASSTHRU_BYTESWAP__LAST,
263 struct ccp_passthru {
264 phys_addr_t src_addr;
265 phys_addr_t dest_addr;
266 enum ccp_passthru_bitwise bit_mod;
267 enum ccp_passthru_byteswap byte_swap;
272 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
320 * descriptor for version 5 CPP commands
322 * word 0: function; engine; control bits
323 * word 1: length of source data
324 * word 2: low 32 bits of source pointer
325 * word 3: upper 16 bits of source pointer; source memory type
326 * word 4: low 32 bits of destination pointer
327 * word 5: upper 16 bits of destination pointer; destination memory
329 * word 6: low 32 bits of key pointer
330 * word 7: upper 16 bits of key pointer; key memory type
338 uint32_t function:15;
347 uint32_t lsb_cxt_id:8;
353 uint32_t dst_lo; /* NON-SHA */
354 uint32_t sha_len_lo; /* SHA */
386 * cmd id to follow order
394 CCP_CMD_NOT_SUPPORTED,
397 static inline uint32_t
398 low32_value(unsigned long addr)
400 return ((uint64_t)addr) & 0x0ffffffff;
403 static inline uint32_t
404 high32_value(unsigned long addr)
406 return ((uint64_t)addr >> 32) & 0x00000ffff;
412 int ccp_dev_start(struct rte_cryptodev *dev);
415 * Detect ccp platform and initialize all ccp devices
417 * @param ccp_id rte_pci_id list for supported CCP devices
418 * @return no. of successfully initialized CCP devices
420 int ccp_probe_devices(const struct rte_pci_id *ccp_id);
423 * allocate a ccp command queue
425 * @dev rte crypto device
426 * @param slot_req number of required
427 * @return allotted CCP queue on success otherwise NULL
429 struct ccp_queue *ccp_allot_queue(struct rte_cryptodev *dev, int slot_req);
431 #endif /* _CCP_DEV_H_ */