drivers/crypto/ccp/ccp_dev.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  */
   4
   5 #include <dirent.h>
   6 #include <fcntl.h>
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <sys/mman.h>
  10 #include <sys/queue.h>
  11 #include <sys/types.h>
  12 #include <sys/file.h>
  13 #include <unistd.h>
  14
  15 #include <rte_hexdump.h>
  16 #include <rte_memzone.h>
  17 #include <rte_malloc.h>
  18 #include <rte_memory.h>
  19 #include <rte_spinlock.h>
  20 #include <rte_string_fns.h>
  21
  22 #include "ccp_dev.h"
  23 #include "ccp_pci.h"
  24 #include "ccp_pmd_private.h"
  25
  26 struct ccp_list ccp_list = TAILQ_HEAD_INITIALIZER(ccp_list);
  27 static int ccp_dev_id;
  28
  29 int
  30 ccp_dev_start(struct rte_cryptodev *dev)
  31 {
  32         struct ccp_private *priv = dev->data->dev_private;
  33
  34         priv->last_dev = TAILQ_FIRST(&ccp_list);
  35         return 0;
  36 }
  37
  38 struct ccp_queue *
  39 ccp_allot_queue(struct rte_cryptodev *cdev, int slot_req)
  40 {
  41         int i, ret = 0;
  42         struct ccp_device *dev;
  43         struct ccp_private *priv = cdev->data->dev_private;
  44
  45         dev = TAILQ_NEXT(priv->last_dev, next);
  46         if (unlikely(dev == NULL))
  47                 dev = TAILQ_FIRST(&ccp_list);
  48         priv->last_dev = dev;
  49         if (dev->qidx >= dev->cmd_q_count)
  50                 dev->qidx = 0;
  51         ret = rte_atomic64_read(&dev->cmd_q[dev->qidx].free_slots);
  52         if (ret >= slot_req)
  53                 return &dev->cmd_q[dev->qidx];
  54         for (i = 0; i < dev->cmd_q_count; i++) {
  55                 dev->qidx++;
  56                 if (dev->qidx >= dev->cmd_q_count)
  57                         dev->qidx = 0;
  58                 ret = rte_atomic64_read(&dev->cmd_q[dev->qidx].free_slots);
  59                 if (ret >= slot_req)
  60                         return &dev->cmd_q[dev->qidx];
  61         }
  62         return NULL;
  63 }
  64
  65 int
  66 ccp_read_hwrng(uint32_t *value)
  67 {
  68         struct ccp_device *dev;
  69
  70         TAILQ_FOREACH(dev, &ccp_list, next) {
  71                 void *vaddr = (void *)(dev->pci.mem_resource[2].addr);
  72
  73                 while (dev->hwrng_retries++ < CCP_MAX_TRNG_RETRIES) {
  74                         *value = CCP_READ_REG(vaddr, TRNG_OUT_REG);
  75                         if (*value) {
  76                                 dev->hwrng_retries = 0;
  77                                 return 0;
  78                         }
  79                 }
  80                 dev->hwrng_retries = 0;
  81         }
  82         return -1;
  83 }
  84
  85 static const struct rte_memzone *
  86 ccp_queue_dma_zone_reserve(const char *queue_name,
  87                            uint32_t queue_size,
  88                            int socket_id)
  89 {
  90         const struct rte_memzone *mz;
  91
  92         mz = rte_memzone_lookup(queue_name);
  93         if (mz != 0) {
  94                 if (((size_t)queue_size <= mz->len) &&
  95                     ((socket_id == SOCKET_ID_ANY) ||
  96                      (socket_id == mz->socket_id))) {
  97                         CCP_LOG_INFO("re-use memzone already "
  98                                      "allocated for %s", queue_name);
  99                         return mz;
 100                 }
 101                 CCP_LOG_ERR("Incompatible memzone already "
 102                             "allocated %s, size %u, socket %d. "
 103                             "Requested size %u, socket %u",
 104                             queue_name, (uint32_t)mz->len,
 105                             mz->socket_id, queue_size, socket_id);
 106                 return NULL;
 107         }
 108
 109         CCP_LOG_INFO("Allocate memzone for %s, size %u on socket %u",
 110                      queue_name, queue_size, socket_id);
 111
 112         return rte_memzone_reserve_aligned(queue_name, queue_size,
 113                         socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 114 }
 115
 116 /* bitmap support apis */
 117 static inline void
 118 ccp_set_bit(unsigned long *bitmap, int n)
 119 {
 120         __sync_fetch_and_or(&bitmap[WORD_OFFSET(n)], (1UL << BIT_OFFSET(n)));
 121 }
 122
 123 static inline void
 124 ccp_clear_bit(unsigned long *bitmap, int n)
 125 {
 126         __sync_fetch_and_and(&bitmap[WORD_OFFSET(n)], ~(1UL << BIT_OFFSET(n)));
 127 }
 128
 129 static inline uint32_t
 130 ccp_get_bit(unsigned long *bitmap, int n)
 131 {
 132         return ((bitmap[WORD_OFFSET(n)] & (1 << BIT_OFFSET(n))) != 0);
 133 }
 134
 135
 136 static inline uint32_t
 137 ccp_ffz(unsigned long word)
 138 {
 139         unsigned long first_zero;
 140
 141         first_zero = __builtin_ffsl(~word);
 142         return first_zero ? (first_zero - 1) :
 143                 BITS_PER_WORD;
 144 }
 145
 146 static inline uint32_t
 147 ccp_find_first_zero_bit(unsigned long *addr, uint32_t limit)
 148 {
 149         uint32_t i;
 150         uint32_t nwords = 0;
 151
 152         nwords = (limit - 1) / BITS_PER_WORD + 1;
 153         for (i = 0; i < nwords; i++) {
 154                 if (addr[i] == 0UL)
 155                         return i * BITS_PER_WORD;
 156                 if (addr[i] < ~(0UL))
 157                         break;
 158         }
 159         return (i == nwords) ? limit : i * BITS_PER_WORD + ccp_ffz(addr[i]);
 160 }
 161
 162 static void
 163 ccp_bitmap_set(unsigned long *map, unsigned int start, int len)
 164 {
 165         unsigned long *p = map + WORD_OFFSET(start);
 166         const unsigned int size = start + len;
 167         int bits_to_set = BITS_PER_WORD - (start % BITS_PER_WORD);
 168         unsigned long mask_to_set = CCP_BITMAP_FIRST_WORD_MASK(start);
 169
 170         while (len - bits_to_set >= 0) {
 171                 *p |= mask_to_set;
 172                 len -= bits_to_set;
 173                 bits_to_set = BITS_PER_WORD;
 174                 mask_to_set = ~0UL;
 175                 p++;
 176         }
 177         if (len) {
 178                 mask_to_set &= CCP_BITMAP_LAST_WORD_MASK(size);
 179                 *p |= mask_to_set;
 180         }
 181 }
 182
 183 static void
 184 ccp_bitmap_clear(unsigned long *map, unsigned int start, int len)
 185 {
 186         unsigned long *p = map + WORD_OFFSET(start);
 187         const unsigned int size = start + len;
 188         int bits_to_clear = BITS_PER_WORD - (start % BITS_PER_WORD);
 189         unsigned long mask_to_clear = CCP_BITMAP_FIRST_WORD_MASK(start);
 190
 191         while (len - bits_to_clear >= 0) {
 192                 *p &= ~mask_to_clear;
 193                 len -= bits_to_clear;
 194                 bits_to_clear = BITS_PER_WORD;
 195                 mask_to_clear = ~0UL;
 196                 p++;
 197         }
 198         if (len) {
 199                 mask_to_clear &= CCP_BITMAP_LAST_WORD_MASK(size);
 200                 *p &= ~mask_to_clear;
 201         }
 202 }
 203
 204
 205 static unsigned long
 206 _ccp_find_next_bit(const unsigned long *addr,
 207                    unsigned long nbits,
 208                    unsigned long start,
 209                    unsigned long invert)
 210 {
 211         unsigned long tmp;
 212
 213         if (!nbits || start >= nbits)
 214                 return nbits;
 215
 216         tmp = addr[start / BITS_PER_WORD] ^ invert;
 217
 218         /* Handle 1st word. */
 219         tmp &= CCP_BITMAP_FIRST_WORD_MASK(start);
 220         start = ccp_round_down(start, BITS_PER_WORD);
 221
 222         while (!tmp) {
 223                 start += BITS_PER_WORD;
 224                 if (start >= nbits)
 225                         return nbits;
 226
 227                 tmp = addr[start / BITS_PER_WORD] ^ invert;
 228         }
 229
 230         return RTE_MIN(start + (ffs(tmp) - 1), nbits);
 231 }
 232
 233 static unsigned long
 234 ccp_find_next_bit(const unsigned long *addr,
 235                   unsigned long size,
 236                   unsigned long offset)
 237 {
 238         return _ccp_find_next_bit(addr, size, offset, 0UL);
 239 }
 240
 241 static unsigned long
 242 ccp_find_next_zero_bit(const unsigned long *addr,
 243                        unsigned long size,
 244                        unsigned long offset)
 245 {
 246         return _ccp_find_next_bit(addr, size, offset, ~0UL);
 247 }
 248
 249 /**
 250  * bitmap_find_next_zero_area - find a contiguous aligned zero area
 251  * @map: The address to base the search on
 252  * @size: The bitmap size in bits
 253  * @start: The bitnumber to start searching at
 254  * @nr: The number of zeroed bits we're looking for
 255  */
 256 static unsigned long
 257 ccp_bitmap_find_next_zero_area(unsigned long *map,
 258                                unsigned long size,
 259                                unsigned long start,
 260                                unsigned int nr)
 261 {
 262         unsigned long index, end, i;
 263
 264 again:
 265         index = ccp_find_next_zero_bit(map, size, start);
 266
 267         end = index + nr;
 268         if (end > size)
 269                 return end;
 270         i = ccp_find_next_bit(map, end, index);
 271         if (i < end) {
 272                 start = i + 1;
 273                 goto again;
 274         }
 275         return index;
 276 }
 277
 278 static uint32_t
 279 ccp_lsb_alloc(struct ccp_queue *cmd_q, unsigned int count)
 280 {
 281         struct ccp_device *ccp;
 282         int start;
 283
 284         /* First look at the map for the queue */
 285         if (cmd_q->lsb >= 0) {
 286                 start = (uint32_t)ccp_bitmap_find_next_zero_area(cmd_q->lsbmap,
 287                                                                  LSB_SIZE, 0,
 288                                                                  count);
 289                 if (start < LSB_SIZE) {
 290                         ccp_bitmap_set(cmd_q->lsbmap, start, count);
 291                         return start + cmd_q->lsb * LSB_SIZE;
 292                 }
 293         }
 294
 295         /* try to get an entry from the shared blocks */
 296         ccp = cmd_q->dev;
 297
 298         rte_spinlock_lock(&ccp->lsb_lock);
 299
 300         start = (uint32_t)ccp_bitmap_find_next_zero_area(ccp->lsbmap,
 301                                                     MAX_LSB_CNT * LSB_SIZE,
 302                                                     0, count);
 303         if (start <= MAX_LSB_CNT * LSB_SIZE) {
 304                 ccp_bitmap_set(ccp->lsbmap, start, count);
 305                 rte_spinlock_unlock(&ccp->lsb_lock);
 306                 return start * LSB_ITEM_SIZE;
 307         }
 308         CCP_LOG_ERR("NO LSBs available");
 309
 310         rte_spinlock_unlock(&ccp->lsb_lock);
 311
 312         return 0;
 313 }
 314
 315 static void __rte_unused
 316 ccp_lsb_free(struct ccp_queue *cmd_q,
 317              unsigned int start,
 318              unsigned int count)
 319 {
 320         int lsbno = start / LSB_SIZE;
 321
 322         if (!start)
 323                 return;
 324
 325         if (cmd_q->lsb == lsbno) {
 326                 /* An entry from the private LSB */
 327                 ccp_bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
 328         } else {
 329                 /* From the shared LSBs */
 330                 struct ccp_device *ccp = cmd_q->dev;
 331
 332                 rte_spinlock_lock(&ccp->lsb_lock);
 333                 ccp_bitmap_clear(ccp->lsbmap, start, count);
 334                 rte_spinlock_unlock(&ccp->lsb_lock);
 335         }
 336 }
 337
 338 static int
 339 ccp_find_lsb_regions(struct ccp_queue *cmd_q, uint64_t status)
 340 {
 341         int q_mask = 1 << cmd_q->id;
 342         int weight = 0;
 343         int j;
 344
 345         /* Build a bit mask to know which LSBs
 346          * this queue has access to.
 347          * Don't bother with segment 0
 348          * as it has special
 349          * privileges.
 350          */
 351         cmd_q->lsbmask = 0;
 352         status >>= LSB_REGION_WIDTH;
 353         for (j = 1; j < MAX_LSB_CNT; j++) {
 354                 if (status & q_mask)
 355                         ccp_set_bit(&cmd_q->lsbmask, j);
 356
 357                 status >>= LSB_REGION_WIDTH;
 358         }
 359
 360         for (j = 0; j < MAX_LSB_CNT; j++)
 361                 if (ccp_get_bit(&cmd_q->lsbmask, j))
 362                         weight++;
 363
 364         printf("Queue %d can access %d LSB regions  of mask  %lu\n",
 365                (int)cmd_q->id, weight, cmd_q->lsbmask);
 366
 367         return weight ? 0 : -EINVAL;
 368 }
 369
 370 static int
 371 ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
 372                              int lsb_cnt, int n_lsbs,
 373                              unsigned long *lsb_pub)
 374 {
 375         unsigned long qlsb = 0;
 376         int bitno = 0;
 377         int qlsb_wgt = 0;
 378         int i, j;
 379
 380         /* For each queue:
 381          * If the count of potential LSBs available to a queue matches the
 382          * ordinal given to us in lsb_cnt:
 383          * Copy the mask of possible LSBs for this queue into "qlsb";
 384          * For each bit in qlsb, see if the corresponding bit in the
 385          * aggregation mask is set; if so, we have a match.
 386          *     If we have a match, clear the bit in the aggregation to
 387          *     mark it as no longer available.
 388          *     If there is no match, clear the bit in qlsb and keep looking.
 389          */
 390         for (i = 0; i < ccp->cmd_q_count; i++) {
 391                 struct ccp_queue *cmd_q = &ccp->cmd_q[i];
 392
 393                 qlsb_wgt = 0;
 394                 for (j = 0; j < MAX_LSB_CNT; j++)
 395                         if (ccp_get_bit(&cmd_q->lsbmask, j))
 396                                 qlsb_wgt++;
 397
 398                 if (qlsb_wgt == lsb_cnt) {
 399                         qlsb = cmd_q->lsbmask;
 400
 401                         bitno = ffs(qlsb) - 1;
 402                         while (bitno < MAX_LSB_CNT) {
 403                                 if (ccp_get_bit(lsb_pub, bitno)) {
 404                                         /* We found an available LSB
 405                                          * that this queue can access
 406                                          */
 407                                         cmd_q->lsb = bitno;
 408                                         ccp_clear_bit(lsb_pub, bitno);
 409                                         break;
 410                                 }
 411                                 ccp_clear_bit(&qlsb, bitno);
 412                                 bitno = ffs(qlsb) - 1;
 413                         }
 414                         if (bitno >= MAX_LSB_CNT)
 415                                 return -EINVAL;
 416                         n_lsbs--;
 417                 }
 418         }
 419         return n_lsbs;
 420 }
 421
 422 /* For each queue, from the most- to least-constrained:
 423  * find an LSB that can be assigned to the queue. If there are N queues that
 424  * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
 425  * dedicated LSB. Remaining LSB regions become a shared resource.
 426  * If we have fewer LSBs than queues, all LSB regions become shared
 427  * resources.
 428  */
 429 static int
 430 ccp_assign_lsbs(struct ccp_device *ccp)
 431 {
 432         unsigned long lsb_pub = 0, qlsb = 0;
 433         int n_lsbs = 0;
 434         int bitno;
 435         int i, lsb_cnt;
 436         int rc = 0;
 437
 438         rte_spinlock_init(&ccp->lsb_lock);
 439
 440         /* Create an aggregate bitmap to get a total count of available LSBs */
 441         for (i = 0; i < ccp->cmd_q_count; i++)
 442                 lsb_pub |= ccp->cmd_q[i].lsbmask;
 443
 444         for (i = 0; i < MAX_LSB_CNT; i++)
 445                 if (ccp_get_bit(&lsb_pub, i))
 446                         n_lsbs++;
 447
 448         if (n_lsbs >= ccp->cmd_q_count) {
 449                 /* We have enough LSBS to give every queue a private LSB.
 450                  * Brute force search to start with the queues that are more
 451                  * constrained in LSB choice. When an LSB is privately
 452                  * assigned, it is removed from the public mask.
 453                  * This is an ugly N squared algorithm with some optimization.
 454                  */
 455                 for (lsb_cnt = 1; n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
 456                      lsb_cnt++) {
 457                         rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
 458                                                           &lsb_pub);
 459                         if (rc < 0)
 460                                 return -EINVAL;
 461                         n_lsbs = rc;
 462                 }
 463         }
 464
 465         rc = 0;
 466         /* What's left of the LSBs, according to the public mask, now become
 467          * shared. Any zero bits in the lsb_pub mask represent an LSB region
 468          * that can't be used as a shared resource, so mark the LSB slots for
 469          * them as "in use".
 470          */
 471         qlsb = lsb_pub;
 472         bitno = ccp_find_first_zero_bit(&qlsb, MAX_LSB_CNT);
 473         while (bitno < MAX_LSB_CNT) {
 474                 ccp_bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
 475                 ccp_set_bit(&qlsb, bitno);
 476                 bitno = ccp_find_first_zero_bit(&qlsb, MAX_LSB_CNT);
 477         }
 478
 479         return rc;
 480 }
 481
 482 static int
 483 ccp_add_device(struct ccp_device *dev, int type)
 484 {
 485         int i;
 486         uint32_t qmr, status_lo, status_hi, dma_addr_lo, dma_addr_hi;
 487         uint64_t status;
 488         struct ccp_queue *cmd_q;
 489         const struct rte_memzone *q_mz;
 490         void *vaddr;
 491
 492         if (dev == NULL)
 493                 return -1;
 494
 495         dev->id = ccp_dev_id++;
 496         dev->qidx = 0;
 497         vaddr = (void *)(dev->pci.mem_resource[2].addr);
 498
 499         if (type == CCP_VERSION_5B) {
 500                 CCP_WRITE_REG(vaddr, CMD_TRNG_CTL_OFFSET, 0x00012D57);
 501                 CCP_WRITE_REG(vaddr, CMD_CONFIG_0_OFFSET, 0x00000003);
 502                 for (i = 0; i < 12; i++) {
 503                         CCP_WRITE_REG(vaddr, CMD_AES_MASK_OFFSET,
 504                                       CCP_READ_REG(vaddr, TRNG_OUT_REG));
 505                 }
 506                 CCP_WRITE_REG(vaddr, CMD_QUEUE_MASK_OFFSET, 0x0000001F);
 507                 CCP_WRITE_REG(vaddr, CMD_QUEUE_PRIO_OFFSET, 0x00005B6D);
 508                 CCP_WRITE_REG(vaddr, CMD_CMD_TIMEOUT_OFFSET, 0x00000000);
 509
 510                 CCP_WRITE_REG(vaddr, LSB_PRIVATE_MASK_LO_OFFSET, 0x3FFFFFFF);
 511                 CCP_WRITE_REG(vaddr, LSB_PRIVATE_MASK_HI_OFFSET, 0x000003FF);
 512
 513                 CCP_WRITE_REG(vaddr, CMD_CLK_GATE_CTL_OFFSET, 0x00108823);
 514         }
 515         CCP_WRITE_REG(vaddr, CMD_REQID_CONFIG_OFFSET, 0x00001249);
 516
 517         /* Copy the private LSB mask to the public registers */
 518         status_lo = CCP_READ_REG(vaddr, LSB_PRIVATE_MASK_LO_OFFSET);
 519         status_hi = CCP_READ_REG(vaddr, LSB_PRIVATE_MASK_HI_OFFSET);
 520         CCP_WRITE_REG(vaddr, LSB_PUBLIC_MASK_LO_OFFSET, status_lo);
 521         CCP_WRITE_REG(vaddr, LSB_PUBLIC_MASK_HI_OFFSET, status_hi);
 522         status = ((uint64_t)status_hi<<30) | ((uint64_t)status_lo);
 523
 524         dev->cmd_q_count = 0;
 525         /* Find available queues */
 526         qmr = CCP_READ_REG(vaddr, Q_MASK_REG);
 527         for (i = 0; i < MAX_HW_QUEUES; i++) {
 528                 if (!(qmr & (1 << i)))
 529                         continue;
 530                 cmd_q = &dev->cmd_q[dev->cmd_q_count++];
 531                 cmd_q->dev = dev;
 532                 cmd_q->id = i;
 533                 cmd_q->qidx = 0;
 534                 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
 535
 536                 cmd_q->reg_base = (uint8_t *)vaddr +
 537                         CMD_Q_STATUS_INCR * (i + 1);
 538
 539                 /* CCP queue memory */
 540                 snprintf(cmd_q->memz_name, sizeof(cmd_q->memz_name),
 541                          "%s_%d_%s_%d_%s",
 542                          "ccp_dev",
 543                          (int)dev->id, "queue",
 544                          (int)cmd_q->id, "mem");
 545                 q_mz = ccp_queue_dma_zone_reserve(cmd_q->memz_name,
 546                                                   cmd_q->qsize, SOCKET_ID_ANY);
 547                 cmd_q->qbase_addr = (void *)q_mz->addr;
 548                 cmd_q->qbase_desc = (void *)q_mz->addr;
 549                 cmd_q->qbase_phys_addr =  q_mz->phys_addr;
 550
 551                 cmd_q->qcontrol = 0;
 552                 /* init control reg to zero */
 553                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
 554                               cmd_q->qcontrol);
 555
 556                 /* Disable the interrupts */
 557                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_INT_ENABLE_BASE, 0x00);
 558                 CCP_READ_REG(cmd_q->reg_base, CMD_Q_INT_STATUS_BASE);
 559                 CCP_READ_REG(cmd_q->reg_base, CMD_Q_STATUS_BASE);
 560
 561                 /* Clear the interrupts */
 562                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_INTERRUPT_STATUS_BASE,
 563                               ALL_INTERRUPTS);
 564
 565                 /* Configure size of each virtual queue accessible to host */
 566                 cmd_q->qcontrol &= ~(CMD_Q_SIZE << CMD_Q_SHIFT);
 567                 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD_Q_SHIFT;
 568
 569                 dma_addr_lo = low32_value(cmd_q->qbase_phys_addr);
 570                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE,
 571                               (uint32_t)dma_addr_lo);
 572                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_HEAD_LO_BASE,
 573                               (uint32_t)dma_addr_lo);
 574
 575                 dma_addr_hi = high32_value(cmd_q->qbase_phys_addr);
 576                 cmd_q->qcontrol |= (dma_addr_hi << 16);
 577                 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
 578                               cmd_q->qcontrol);
 579
 580                 /* create LSB Mask map */
 581                 if (ccp_find_lsb_regions(cmd_q, status))
 582                         CCP_LOG_ERR("queue doesn't have lsb regions");
 583                 cmd_q->lsb = -1;
 584
 585                 rte_atomic64_init(&cmd_q->free_slots);
 586                 rte_atomic64_set(&cmd_q->free_slots, (COMMANDS_PER_QUEUE - 1));
 587                 /* unused slot barrier b/w H&T */
 588         }
 589
 590         if (ccp_assign_lsbs(dev))
 591                 CCP_LOG_ERR("Unable to assign lsb region");
 592
 593         /* pre-allocate LSB slots */
 594         for (i = 0; i < dev->cmd_q_count; i++) {
 595                 dev->cmd_q[i].sb_key =
 596                         ccp_lsb_alloc(&dev->cmd_q[i], 1);
 597                 dev->cmd_q[i].sb_iv =
 598                         ccp_lsb_alloc(&dev->cmd_q[i], 1);
 599                 dev->cmd_q[i].sb_sha =
 600                         ccp_lsb_alloc(&dev->cmd_q[i], 2);
 601                 dev->cmd_q[i].sb_hmac =
 602                         ccp_lsb_alloc(&dev->cmd_q[i], 2);
 603         }
 604
 605         TAILQ_INSERT_TAIL(&ccp_list, dev, next);
 606         return 0;
 607 }
 608
 609 static void
 610 ccp_remove_device(struct ccp_device *dev)
 611 {
 612         if (dev == NULL)
 613                 return;
 614
 615         TAILQ_REMOVE(&ccp_list, dev, next);
 616 }
 617
 618 static int
 619 is_ccp_device(const char *dirname,
 620               const struct rte_pci_id *ccp_id,
 621               int *type)
 622 {
 623         char filename[PATH_MAX];
 624         const struct rte_pci_id *id;
 625         uint16_t vendor, device_id;
 626         int i;
 627         unsigned long tmp;
 628
 629         /* get vendor id */
 630         snprintf(filename, sizeof(filename), "%s/vendor", dirname);
 631         if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 632                 return 0;
 633         vendor = (uint16_t)tmp;
 634
 635         /* get device id */
 636         snprintf(filename, sizeof(filename), "%s/device", dirname);
 637         if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 638                 return 0;
 639         device_id = (uint16_t)tmp;
 640
 641         for (id = ccp_id, i = 0; id->vendor_id != 0; id++, i++) {
 642                 if (vendor == id->vendor_id &&
 643                     device_id == id->device_id) {
 644                         *type = i;
 645                         return 1; /* Matched device */
 646                 }
 647         }
 648         return 0;
 649 }
 650
 651 static int
 652 ccp_probe_device(const char *dirname, uint16_t domain,
 653                  uint8_t bus, uint8_t devid,
 654                  uint8_t function, int ccp_type)
 655 {
 656         struct ccp_device *ccp_dev = NULL;
 657         struct rte_pci_device *pci;
 658         char filename[PATH_MAX];
 659         unsigned long tmp;
 660         int uio_fd = -1, i, uio_num;
 661         char uio_devname[PATH_MAX];
 662         void *map_addr;
 663
 664         ccp_dev = rte_zmalloc("ccp_device", sizeof(*ccp_dev),
 665                               RTE_CACHE_LINE_SIZE);
 666         if (ccp_dev == NULL)
 667                 goto fail;
 668         pci = &(ccp_dev->pci);
 669
 670         pci->addr.domain = domain;
 671         pci->addr.bus = bus;
 672         pci->addr.devid = devid;
 673         pci->addr.function = function;
 674
 675         /* get vendor id */
 676         snprintf(filename, sizeof(filename), "%s/vendor", dirname);
 677         if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 678                 goto fail;
 679         pci->id.vendor_id = (uint16_t)tmp;
 680
 681         /* get device id */
 682         snprintf(filename, sizeof(filename), "%s/device", dirname);
 683         if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 684                 goto fail;
 685         pci->id.device_id = (uint16_t)tmp;
 686
 687         /* get subsystem_vendor id */
 688         snprintf(filename, sizeof(filename), "%s/subsystem_vendor",
 689                         dirname);
 690         if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 691                 goto fail;
 692         pci->id.subsystem_vendor_id = (uint16_t)tmp;
 693
 694         /* get subsystem_device id */
 695         snprintf(filename, sizeof(filename), "%s/subsystem_device",
 696                         dirname);
 697         if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 698                 goto fail;
 699         pci->id.subsystem_device_id = (uint16_t)tmp;
 700
 701         /* get class_id */
 702         snprintf(filename, sizeof(filename), "%s/class",
 703                         dirname);
 704         if (ccp_pci_parse_sysfs_value(filename, &tmp) < 0)
 705                 goto fail;
 706         /* the least 24 bits are valid: class, subclass, program interface */
 707         pci->id.class_id = (uint32_t)tmp & RTE_CLASS_ANY_ID;
 708
 709         /* parse resources */
 710         snprintf(filename, sizeof(filename), "%s/resource", dirname);
 711         if (ccp_pci_parse_sysfs_resource(filename, pci) < 0)
 712                 goto fail;
 713
 714         uio_num = ccp_find_uio_devname(dirname);
 715         if (uio_num < 0) {
 716                 /*
 717                  * It may take time for uio device to appear,
 718                  * wait  here and try again
 719                  */
 720                 usleep(100000);
 721                 uio_num = ccp_find_uio_devname(dirname);
 722                 if (uio_num < 0)
 723                         goto fail;
 724         }
 725         snprintf(uio_devname, sizeof(uio_devname), "/dev/uio%u", uio_num);
 726
 727         uio_fd = open(uio_devname, O_RDWR | O_NONBLOCK);
 728         if (uio_fd < 0)
 729                 goto fail;
 730         if (flock(uio_fd, LOCK_EX | LOCK_NB))
 731                 goto fail;
 732
 733         /* Map the PCI memory resource of device */
 734         for (i = 0; i < PCI_MAX_RESOURCE; i++) {
 735
 736                 char devname[PATH_MAX];
 737                 int res_fd;
 738
 739                 if (pci->mem_resource[i].phys_addr == 0)
 740                         continue;
 741                 snprintf(devname, sizeof(devname), "%s/resource%d", dirname, i);
 742                 res_fd = open(devname, O_RDWR);
 743                 if (res_fd < 0)
 744                         goto fail;
 745                 map_addr = mmap(NULL, pci->mem_resource[i].len,
 746                                 PROT_READ | PROT_WRITE,
 747                                 MAP_SHARED, res_fd, 0);
 748                 if (map_addr == MAP_FAILED)
 749                         goto fail;
 750
 751                 pci->mem_resource[i].addr = map_addr;
 752         }
 753
 754         /* device is valid, add in list */
 755         if (ccp_add_device(ccp_dev, ccp_type)) {
 756                 ccp_remove_device(ccp_dev);
 757                 goto fail;
 758         }
 759
 760         return 0;
 761 fail:
 762         CCP_LOG_ERR("CCP Device probe failed");
 763         if (uio_fd > 0)
 764                 close(uio_fd);
 765         if (ccp_dev)
 766                 rte_free(ccp_dev);
 767         return -1;
 768 }
 769
 770 int
 771 ccp_probe_devices(const struct rte_pci_id *ccp_id)
 772 {
 773         int dev_cnt = 0;
 774         int ccp_type = 0;
 775         struct dirent *d;
 776         DIR *dir;
 777         int ret = 0;
 778         int module_idx = 0;
 779         uint16_t domain;
 780         uint8_t bus, devid, function;
 781         char dirname[PATH_MAX];
 782
 783         module_idx = ccp_check_pci_uio_module();
 784         if (module_idx < 0)
 785                 return -1;
 786
 787         TAILQ_INIT(&ccp_list);
 788         dir = opendir(SYSFS_PCI_DEVICES);
 789         if (dir == NULL)
 790                 return -1;
 791         while ((d = readdir(dir)) != NULL) {
 792                 if (d->d_name[0] == '.')
 793                         continue;
 794                 if (ccp_parse_pci_addr_format(d->d_name, sizeof(d->d_name),
 795                                         &domain, &bus, &devid, &function) != 0)
 796                         continue;
 797                 snprintf(dirname, sizeof(dirname), "%s/%s",
 798                              SYSFS_PCI_DEVICES, d->d_name);
 799                 if (is_ccp_device(dirname, ccp_id, &ccp_type)) {
 800                         printf("CCP : Detected CCP device with ID = 0x%x\n",
 801                                ccp_id[ccp_type].device_id);
 802                         ret = ccp_probe_device(dirname, domain, bus, devid,
 803                                                function, ccp_type);
 804                         if (ret == 0)
 805                                 dev_cnt++;
 806                 }
 807         }
 808         closedir(dir);
 809         return dev_cnt;
 810 }