1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
10 #include <sys/queue.h>
11 #include <sys/types.h>
15 #include <rte_hexdump.h>
16 #include <rte_memzone.h>
17 #include <rte_malloc.h>
18 #include <rte_memory.h>
19 #include <rte_spinlock.h>
20 #include <rte_string_fns.h>
24 #include "ccp_pmd_private.h"
26 struct ccp_list ccp_list = TAILQ_HEAD_INITIALIZER(ccp_list);
27 static int ccp_dev_id;
30 ccp_dev_start(struct rte_cryptodev *dev)
32 struct ccp_private *priv = dev->data->dev_private;
34 priv->last_dev = TAILQ_FIRST(&ccp_list);
39 ccp_allot_queue(struct rte_cryptodev *cdev, int slot_req)
42 struct ccp_device *dev;
43 struct ccp_private *priv = cdev->data->dev_private;
45 dev = TAILQ_NEXT(priv->last_dev, next);
46 if (unlikely(dev == NULL))
47 dev = TAILQ_FIRST(&ccp_list);
49 if (dev->qidx >= dev->cmd_q_count)
51 ret = rte_atomic64_read(&dev->cmd_q[dev->qidx].free_slots);
53 return &dev->cmd_q[dev->qidx];
54 for (i = 0; i < dev->cmd_q_count; i++) {
56 if (dev->qidx >= dev->cmd_q_count)
58 ret = rte_atomic64_read(&dev->cmd_q[dev->qidx].free_slots);
60 return &dev->cmd_q[dev->qidx];
65 static const struct rte_memzone *
66 ccp_queue_dma_zone_reserve(const char *queue_name,
70 const struct rte_memzone *mz;
72 mz = rte_memzone_lookup(queue_name);
74 if (((size_t)queue_size <= mz->len) &&
75 ((socket_id == SOCKET_ID_ANY) ||
76 (socket_id == mz->socket_id))) {
77 CCP_LOG_INFO("re-use memzone already "
78 "allocated for %s", queue_name);
81 CCP_LOG_ERR("Incompatible memzone already "
82 "allocated %s, size %u, socket %d. "
83 "Requested size %u, socket %u",
84 queue_name, (uint32_t)mz->len,
85 mz->socket_id, queue_size, socket_id);
89 CCP_LOG_INFO("Allocate memzone for %s, size %u on socket %u",
90 queue_name, queue_size, socket_id);
92 return rte_memzone_reserve_aligned(queue_name, queue_size,
93 socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
96 /* bitmap support apis */
98 ccp_set_bit(unsigned long *bitmap, int n)
100 __sync_fetch_and_or(&bitmap[WORD_OFFSET(n)], (1UL << BIT_OFFSET(n)));
104 ccp_clear_bit(unsigned long *bitmap, int n)
106 __sync_fetch_and_and(&bitmap[WORD_OFFSET(n)], ~(1UL << BIT_OFFSET(n)));
109 static inline uint32_t
110 ccp_get_bit(unsigned long *bitmap, int n)
112 return ((bitmap[WORD_OFFSET(n)] & (1 << BIT_OFFSET(n))) != 0);
116 static inline uint32_t
117 ccp_ffz(unsigned long word)
119 unsigned long first_zero;
121 first_zero = __builtin_ffsl(~word);
122 return first_zero ? (first_zero - 1) :
126 static inline uint32_t
127 ccp_find_first_zero_bit(unsigned long *addr, uint32_t limit)
132 nwords = (limit - 1) / BITS_PER_WORD + 1;
133 for (i = 0; i < nwords; i++) {
135 return i * BITS_PER_WORD;
136 if (addr[i] < ~(0UL))
139 return (i == nwords) ? limit : i * BITS_PER_WORD + ccp_ffz(addr[i]);
143 ccp_bitmap_set(unsigned long *map, unsigned int start, int len)
145 unsigned long *p = map + WORD_OFFSET(start);
146 const unsigned int size = start + len;
147 int bits_to_set = BITS_PER_WORD - (start % BITS_PER_WORD);
148 unsigned long mask_to_set = CCP_BITMAP_FIRST_WORD_MASK(start);
150 while (len - bits_to_set >= 0) {
153 bits_to_set = BITS_PER_WORD;
158 mask_to_set &= CCP_BITMAP_LAST_WORD_MASK(size);
164 ccp_bitmap_clear(unsigned long *map, unsigned int start, int len)
166 unsigned long *p = map + WORD_OFFSET(start);
167 const unsigned int size = start + len;
168 int bits_to_clear = BITS_PER_WORD - (start % BITS_PER_WORD);
169 unsigned long mask_to_clear = CCP_BITMAP_FIRST_WORD_MASK(start);
171 while (len - bits_to_clear >= 0) {
172 *p &= ~mask_to_clear;
173 len -= bits_to_clear;
174 bits_to_clear = BITS_PER_WORD;
175 mask_to_clear = ~0UL;
179 mask_to_clear &= CCP_BITMAP_LAST_WORD_MASK(size);
180 *p &= ~mask_to_clear;
186 _ccp_find_next_bit(const unsigned long *addr,
189 unsigned long invert)
193 if (!nbits || start >= nbits)
196 tmp = addr[start / BITS_PER_WORD] ^ invert;
198 /* Handle 1st word. */
199 tmp &= CCP_BITMAP_FIRST_WORD_MASK(start);
200 start = ccp_round_down(start, BITS_PER_WORD);
203 start += BITS_PER_WORD;
207 tmp = addr[start / BITS_PER_WORD] ^ invert;
210 return RTE_MIN(start + (ffs(tmp) - 1), nbits);
214 ccp_find_next_bit(const unsigned long *addr,
216 unsigned long offset)
218 return _ccp_find_next_bit(addr, size, offset, 0UL);
222 ccp_find_next_zero_bit(const unsigned long *addr,
224 unsigned long offset)
226 return _ccp_find_next_bit(addr, size, offset, ~0UL);
230 * bitmap_find_next_zero_area - find a contiguous aligned zero area
231 * @map: The address to base the search on
232 * @size: The bitmap size in bits
233 * @start: The bitnumber to start searching at
234 * @nr: The number of zeroed bits we're looking for
237 ccp_bitmap_find_next_zero_area(unsigned long *map,
242 unsigned long index, end, i;
245 index = ccp_find_next_zero_bit(map, size, start);
250 i = ccp_find_next_bit(map, end, index);
259 ccp_lsb_alloc(struct ccp_queue *cmd_q, unsigned int count)
261 struct ccp_device *ccp;
264 /* First look at the map for the queue */
265 if (cmd_q->lsb >= 0) {
266 start = (uint32_t)ccp_bitmap_find_next_zero_area(cmd_q->lsbmap,
269 if (start < LSB_SIZE) {
270 ccp_bitmap_set(cmd_q->lsbmap, start, count);
271 return start + cmd_q->lsb * LSB_SIZE;
275 /* try to get an entry from the shared blocks */
278 rte_spinlock_lock(&ccp->lsb_lock);
280 start = (uint32_t)ccp_bitmap_find_next_zero_area(ccp->lsbmap,
281 MAX_LSB_CNT * LSB_SIZE,
283 if (start <= MAX_LSB_CNT * LSB_SIZE) {
284 ccp_bitmap_set(ccp->lsbmap, start, count);
285 rte_spinlock_unlock(&ccp->lsb_lock);
286 return start * LSB_ITEM_SIZE;
288 CCP_LOG_ERR("NO LSBs available");
290 rte_spinlock_unlock(&ccp->lsb_lock);
295 static void __rte_unused
296 ccp_lsb_free(struct ccp_queue *cmd_q,
300 int lsbno = start / LSB_SIZE;
305 if (cmd_q->lsb == lsbno) {
306 /* An entry from the private LSB */
307 ccp_bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
309 /* From the shared LSBs */
310 struct ccp_device *ccp = cmd_q->dev;
312 rte_spinlock_lock(&ccp->lsb_lock);
313 ccp_bitmap_clear(ccp->lsbmap, start, count);
314 rte_spinlock_unlock(&ccp->lsb_lock);
319 ccp_find_lsb_regions(struct ccp_queue *cmd_q, uint64_t status)
321 int q_mask = 1 << cmd_q->id;
325 /* Build a bit mask to know which LSBs
326 * this queue has access to.
327 * Don't bother with segment 0
332 status >>= LSB_REGION_WIDTH;
333 for (j = 1; j < MAX_LSB_CNT; j++) {
335 ccp_set_bit(&cmd_q->lsbmask, j);
337 status >>= LSB_REGION_WIDTH;
340 for (j = 0; j < MAX_LSB_CNT; j++)
341 if (ccp_get_bit(&cmd_q->lsbmask, j))
344 printf("Queue %d can access %d LSB regions of mask %lu\n",
345 (int)cmd_q->id, weight, cmd_q->lsbmask);
347 return weight ? 0 : -EINVAL;
351 ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
352 int lsb_cnt, int n_lsbs,
353 unsigned long *lsb_pub)
355 unsigned long qlsb = 0;
361 * If the count of potential LSBs available to a queue matches the
362 * ordinal given to us in lsb_cnt:
363 * Copy the mask of possible LSBs for this queue into "qlsb";
364 * For each bit in qlsb, see if the corresponding bit in the
365 * aggregation mask is set; if so, we have a match.
366 * If we have a match, clear the bit in the aggregation to
367 * mark it as no longer available.
368 * If there is no match, clear the bit in qlsb and keep looking.
370 for (i = 0; i < ccp->cmd_q_count; i++) {
371 struct ccp_queue *cmd_q = &ccp->cmd_q[i];
374 for (j = 0; j < MAX_LSB_CNT; j++)
375 if (ccp_get_bit(&cmd_q->lsbmask, j))
378 if (qlsb_wgt == lsb_cnt) {
379 qlsb = cmd_q->lsbmask;
381 bitno = ffs(qlsb) - 1;
382 while (bitno < MAX_LSB_CNT) {
383 if (ccp_get_bit(lsb_pub, bitno)) {
384 /* We found an available LSB
385 * that this queue can access
388 ccp_clear_bit(lsb_pub, bitno);
391 ccp_clear_bit(&qlsb, bitno);
392 bitno = ffs(qlsb) - 1;
394 if (bitno >= MAX_LSB_CNT)
402 /* For each queue, from the most- to least-constrained:
403 * find an LSB that can be assigned to the queue. If there are N queues that
404 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
405 * dedicated LSB. Remaining LSB regions become a shared resource.
406 * If we have fewer LSBs than queues, all LSB regions become shared
410 ccp_assign_lsbs(struct ccp_device *ccp)
412 unsigned long lsb_pub = 0, qlsb = 0;
418 rte_spinlock_init(&ccp->lsb_lock);
420 /* Create an aggregate bitmap to get a total count of available LSBs */
421 for (i = 0; i < ccp->cmd_q_count; i++)
422 lsb_pub |= ccp->cmd_q[i].lsbmask;
424 for (i = 0; i < MAX_LSB_CNT; i++)
425 if (ccp_get_bit(&lsb_pub, i))
428 if (n_lsbs >= ccp->cmd_q_count) {
429 /* We have enough LSBS to give every queue a private LSB.
430 * Brute force search to start with the queues that are more
431 * constrained in LSB choice. When an LSB is privately
432 * assigned, it is removed from the public mask.
433 * This is an ugly N squared algorithm with some optimization.
435 for (lsb_cnt = 1; n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
437 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
446 /* What's left of the LSBs, according to the public mask, now become
447 * shared. Any zero bits in the lsb_pub mask represent an LSB region
448 * that can't be used as a shared resource, so mark the LSB slots for
452 bitno = ccp_find_first_zero_bit(&qlsb, MAX_LSB_CNT);
453 while (bitno < MAX_LSB_CNT) {
454 ccp_bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
455 ccp_set_bit(&qlsb, bitno);
456 bitno = ccp_find_first_zero_bit(&qlsb, MAX_LSB_CNT);
463 ccp_add_device(struct ccp_device *dev, int type)
466 uint32_t qmr, status_lo, status_hi, dma_addr_lo, dma_addr_hi;
468 struct ccp_queue *cmd_q;
469 const struct rte_memzone *q_mz;
475 dev->id = ccp_dev_id++;
477 vaddr = (void *)(dev->pci.mem_resource[2].addr);
479 if (type == CCP_VERSION_5B) {
480 CCP_WRITE_REG(vaddr, CMD_TRNG_CTL_OFFSET, 0x00012D57);
481 CCP_WRITE_REG(vaddr, CMD_CONFIG_0_OFFSET, 0x00000003);
482 for (i = 0; i < 12; i++) {
483 CCP_WRITE_REG(vaddr, CMD_AES_MASK_OFFSET,
484 CCP_READ_REG(vaddr, TRNG_OUT_REG));
486 CCP_WRITE_REG(vaddr, CMD_QUEUE_MASK_OFFSET, 0x0000001F);
487 CCP_WRITE_REG(vaddr, CMD_QUEUE_PRIO_OFFSET, 0x00005B6D);
488 CCP_WRITE_REG(vaddr, CMD_CMD_TIMEOUT_OFFSET, 0x00000000);
490 CCP_WRITE_REG(vaddr, LSB_PRIVATE_MASK_LO_OFFSET, 0x3FFFFFFF);
491 CCP_WRITE_REG(vaddr, LSB_PRIVATE_MASK_HI_OFFSET, 0x000003FF);
493 CCP_WRITE_REG(vaddr, CMD_CLK_GATE_CTL_OFFSET, 0x00108823);
495 CCP_WRITE_REG(vaddr, CMD_REQID_CONFIG_OFFSET, 0x00001249);
497 /* Copy the private LSB mask to the public registers */
498 status_lo = CCP_READ_REG(vaddr, LSB_PRIVATE_MASK_LO_OFFSET);
499 status_hi = CCP_READ_REG(vaddr, LSB_PRIVATE_MASK_HI_OFFSET);
500 CCP_WRITE_REG(vaddr, LSB_PUBLIC_MASK_LO_OFFSET, status_lo);
501 CCP_WRITE_REG(vaddr, LSB_PUBLIC_MASK_HI_OFFSET, status_hi);
502 status = ((uint64_t)status_hi<<30) | ((uint64_t)status_lo);
504 dev->cmd_q_count = 0;
505 /* Find available queues */
506 qmr = CCP_READ_REG(vaddr, Q_MASK_REG);
507 for (i = 0; i < MAX_HW_QUEUES; i++) {
508 if (!(qmr & (1 << i)))
510 cmd_q = &dev->cmd_q[dev->cmd_q_count++];
514 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
516 cmd_q->reg_base = (uint8_t *)vaddr +
517 CMD_Q_STATUS_INCR * (i + 1);
519 /* CCP queue memory */
520 snprintf(cmd_q->memz_name, sizeof(cmd_q->memz_name),
523 (int)dev->id, "queue",
524 (int)cmd_q->id, "mem");
525 q_mz = ccp_queue_dma_zone_reserve(cmd_q->memz_name,
526 cmd_q->qsize, SOCKET_ID_ANY);
527 cmd_q->qbase_addr = (void *)q_mz->addr;
528 cmd_q->qbase_desc = (void *)q_mz->addr;
529 cmd_q->qbase_phys_addr = q_mz->phys_addr;
532 /* init control reg to zero */
533 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
536 /* Disable the interrupts */
537 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_INT_ENABLE_BASE, 0x00);
538 CCP_READ_REG(cmd_q->reg_base, CMD_Q_INT_STATUS_BASE);
539 CCP_READ_REG(cmd_q->reg_base, CMD_Q_STATUS_BASE);
541 /* Clear the interrupts */
542 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_INTERRUPT_STATUS_BASE,
545 /* Configure size of each virtual queue accessible to host */
546 cmd_q->qcontrol &= ~(CMD_Q_SIZE << CMD_Q_SHIFT);
547 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD_Q_SHIFT;
549 dma_addr_lo = low32_value(cmd_q->qbase_phys_addr);
550 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE,
551 (uint32_t)dma_addr_lo);
552 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_HEAD_LO_BASE,
553 (uint32_t)dma_addr_lo);
555 dma_addr_hi = high32_value(cmd_q->qbase_phys_addr);
556 cmd_q->qcontrol |= (dma_addr_hi << 16);
557 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
560 /* create LSB Mask map */
561 if (ccp_find_lsb_regions(cmd_q, status))
562 CCP_LOG_ERR("queue doesn't have lsb regions");
565 rte_atomic64_init(&cmd_q->free_slots);
566 rte_atomic64_set(&cmd_q->free_slots, (COMMANDS_PER_QUEUE - 1));
567 /* unused slot barrier b/w H&T */
570 if (ccp_assign_lsbs(dev))
571 CCP_LOG_ERR("Unable to assign lsb region");
573 /* pre-allocate LSB slots */
574 for (i = 0; i < dev->cmd_q_count; i++) {
575 dev->cmd_q[i].sb_key =
576 ccp_lsb_alloc(&dev->cmd_q[i], 1);
577 dev->cmd_q[i].sb_iv =
578 ccp_lsb_alloc(&dev->cmd_q[i], 1);
579 dev->cmd_q[i].sb_sha =
580 ccp_lsb_alloc(&dev->cmd_q[i], 2);
581 dev->cmd_q[i].sb_hmac =
582 ccp_lsb_alloc(&dev->cmd_q[i], 2);
585 TAILQ_INSERT_TAIL(&ccp_list, dev, next);
590 ccp_remove_device(struct ccp_device *dev)
595 TAILQ_REMOVE(&ccp_list, dev, next);
599 is_ccp_device(const char *dirname,
600 const struct rte_pci_id *ccp_id,
603 char filename[PATH_MAX];
604 const struct rte_pci_id *id;
605 uint16_t vendor, device_id;
610 snprintf(filename, sizeof(filename), "%s/vendor", dirname);
611 if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
613 vendor = (uint16_t)tmp;
616 snprintf(filename, sizeof(filename), "%s/device", dirname);
617 if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
619 device_id = (uint16_t)tmp;
621 for (id = ccp_id, i = 0; id->vendor_id != 0; id++, i++) {
622 if (vendor == id->vendor_id &&
623 device_id == id->device_id) {
625 return 1; /* Matched device */
632 ccp_probe_device(const char *dirname, uint16_t domain,
633 uint8_t bus, uint8_t devid,
634 uint8_t function, int ccp_type)
636 struct ccp_device *ccp_dev = NULL;
637 struct rte_pci_device *pci;
638 char filename[PATH_MAX];
640 int uio_fd = -1, i, uio_num;
641 char uio_devname[PATH_MAX];
644 ccp_dev = rte_zmalloc("ccp_device", sizeof(*ccp_dev),
645 RTE_CACHE_LINE_SIZE);
648 pci = &(ccp_dev->pci);
650 pci->addr.domain = domain;
652 pci->addr.devid = devid;
653 pci->addr.function = function;
656 snprintf(filename, sizeof(filename), "%s/vendor", dirname);
657 if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
659 pci->id.vendor_id = (uint16_t)tmp;
662 snprintf(filename, sizeof(filename), "%s/device", dirname);
663 if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
665 pci->id.device_id = (uint16_t)tmp;
667 /* get subsystem_vendor id */
668 snprintf(filename, sizeof(filename), "%s/subsystem_vendor",
670 if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
672 pci->id.subsystem_vendor_id = (uint16_t)tmp;
674 /* get subsystem_device id */
675 snprintf(filename, sizeof(filename), "%s/subsystem_device",
677 if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
679 pci->id.subsystem_device_id = (uint16_t)tmp;
682 snprintf(filename, sizeof(filename), "%s/class",
684 if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
686 /* the least 24 bits are valid: class, subclass, program interface */
687 pci->id.class_id = (uint32_t)tmp & RTE_CLASS_ANY_ID;
689 /* parse resources */
690 snprintf(filename, sizeof(filename), "%s/resource", dirname);
691 if (ccp_pci_parse_sysfs_resource(filename, pci) < 0)
694 uio_num = ccp_find_uio_devname(dirname);
697 * It may take time for uio device to appear,
698 * wait here and try again
701 uio_num = ccp_find_uio_devname(dirname);
705 snprintf(uio_devname, sizeof(uio_devname), "/dev/uio%u", uio_num);
707 uio_fd = open(uio_devname, O_RDWR | O_NONBLOCK);
710 if (flock(uio_fd, LOCK_EX | LOCK_NB))
713 /* Map the PCI memory resource of device */
714 for (i = 0; i < PCI_MAX_RESOURCE; i++) {
716 char devname[PATH_MAX];
719 if (pci->mem_resource[i].phys_addr == 0)
721 snprintf(devname, sizeof(devname), "%s/resource%d", dirname, i);
722 res_fd = open(devname, O_RDWR);
725 map_addr = mmap(NULL, pci->mem_resource[i].len,
726 PROT_READ | PROT_WRITE,
727 MAP_SHARED, res_fd, 0);
728 if (map_addr == MAP_FAILED)
731 pci->mem_resource[i].addr = map_addr;
734 /* device is valid, add in list */
735 if (ccp_add_device(ccp_dev, ccp_type)) {
736 ccp_remove_device(ccp_dev);
742 CCP_LOG_ERR("CCP Device probe failed");
751 ccp_probe_devices(const struct rte_pci_id *ccp_id)
760 uint8_t bus, devid, function;
761 char dirname[PATH_MAX];
763 module_idx = ccp_check_pci_uio_module();
767 TAILQ_INIT(&ccp_list);
768 dir = opendir(SYSFS_PCI_DEVICES);
771 while ((d = readdir(dir)) != NULL) {
772 if (d->d_name[0] == '.')
774 if (ccp_parse_pci_addr_format(d->d_name, sizeof(d->d_name),
775 &domain, &bus, &devid, &function) != 0)
777 snprintf(dirname, sizeof(dirname), "%s/%s",
778 SYSFS_PCI_DEVICES, d->d_name);
779 if (is_ccp_device(dirname, ccp_id, &ccp_type)) {
780 printf("CCP : Detected CCP device with ID = 0x%x\n",
781 ccp_id[ccp_type].device_id);
782 ret = ccp_probe_device(dirname, domain, bus, devid,