drivers/common/cnxk/roc_bphy_irq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4 #include <fcntl.h>
   5 #include <pthread.h>
   6 #include <sys/ioctl.h>
   7 #include <sys/mman.h>
   8 #include <sys/queue.h>
   9 #include <unistd.h>
  10
  11 #include "roc_api.h"
  12 #include "roc_bphy_irq.h"
  13
  14 #define roc_cpuset_t cpu_set_t
  15
  16 struct roc_bphy_irq_usr_data {
  17         uint64_t isr_base;
  18         uint64_t sp;
  19         uint64_t cpu;
  20         uint64_t irq_num;
  21 };
  22
  23 struct roc_bphy_irq_stack {
  24         STAILQ_ENTRY(roc_bphy_irq_stack) entries;
  25         void *sp_buffer;
  26         int cpu;
  27         int inuse;
  28 };
  29
  30 #define ROC_BPHY_MEMZONE_NAME "roc_bphy_mz"
  31 #define ROC_BPHY_CTR_DEV_PATH "/dev/otx-bphy-ctr"
  32
  33 #define ROC_BPHY_IOC_MAGIC 0xF3
  34 #define ROC_BPHY_IOC_SET_BPHY_HANDLER                                          \
  35         _IOW(ROC_BPHY_IOC_MAGIC, 1, struct roc_bphy_irq_usr_data)
  36 #define ROC_BPHY_IOC_CLR_BPHY_HANDLER   _IO(ROC_BPHY_IOC_MAGIC, 2)
  37 #define ROC_BPHY_IOC_GET_BPHY_MAX_IRQ   _IOR(ROC_BPHY_IOC_MAGIC, 3, uint64_t)
  38 #define ROC_BPHY_IOC_GET_BPHY_BMASK_IRQ _IOR(ROC_BPHY_IOC_MAGIC, 4, uint64_t)
  39
  40 static STAILQ_HEAD(slisthead, roc_bphy_irq_stack)
  41         irq_stacks = STAILQ_HEAD_INITIALIZER(irq_stacks);
  42
  43 /* Note: it is assumed that as for now there is no multiprocess support */
  44 static pthread_mutex_t stacks_mutex = PTHREAD_MUTEX_INITIALIZER;
  45
  46 struct roc_bphy_irq_chip *
  47 roc_bphy_intr_init(void)
  48 {
  49         struct roc_bphy_irq_chip *irq_chip;
  50         uint64_t max_irq, i, avail_irqs;
  51         int fd, ret;
  52
  53         fd = open(ROC_BPHY_CTR_DEV_PATH, O_RDWR | O_SYNC);
  54         if (fd < 0) {
  55                 plt_err("Failed to open %s", ROC_BPHY_CTR_DEV_PATH);
  56                 return NULL;
  57         }
  58
  59         ret = ioctl(fd, ROC_BPHY_IOC_GET_BPHY_MAX_IRQ, &max_irq);
  60         if (ret < 0) {
  61                 plt_err("Failed to get max irq number via ioctl");
  62                 goto err_ioctl;
  63         }
  64
  65         ret = ioctl(fd, ROC_BPHY_IOC_GET_BPHY_BMASK_IRQ, &avail_irqs);
  66         if (ret < 0) {
  67                 plt_err("Failed to get available irqs bitmask via ioctl");
  68                 goto err_ioctl;
  69         }
  70
  71         irq_chip = plt_zmalloc(sizeof(*irq_chip), 0);
  72         if (irq_chip == NULL) {
  73                 plt_err("Failed to alloc irq_chip");
  74                 goto err_alloc_chip;
  75         }
  76
  77         irq_chip->intfd = fd;
  78         irq_chip->max_irq = max_irq;
  79         irq_chip->avail_irq_bmask = avail_irqs;
  80         irq_chip->irq_vecs =
  81                 plt_zmalloc(irq_chip->max_irq * sizeof(*irq_chip->irq_vecs), 0);
  82         if (irq_chip->irq_vecs == NULL) {
  83                 plt_err("Failed to alloc irq_chip irq_vecs");
  84                 goto err_alloc_irq;
  85         }
  86
  87         irq_chip->mz_name = plt_zmalloc(strlen(ROC_BPHY_MEMZONE_NAME) + 1, 0);
  88         if (irq_chip->mz_name == NULL) {
  89                 plt_err("Failed to alloc irq_chip name");
  90                 goto err_alloc_name;
  91         }
  92         plt_strlcpy(irq_chip->mz_name, ROC_BPHY_MEMZONE_NAME,
  93                     strlen(ROC_BPHY_MEMZONE_NAME) + 1);
  94
  95         for (i = 0; i < irq_chip->max_irq; i++) {
  96                 irq_chip->irq_vecs[i].fd = -1;
  97                 irq_chip->irq_vecs[i].handler_cpu = -1;
  98         }
  99
 100         return irq_chip;
 101
 102 err_alloc_name:
 103         plt_free(irq_chip->irq_vecs);
 104
 105 err_alloc_irq:
 106         plt_free(irq_chip);
 107
 108 err_ioctl:
 109 err_alloc_chip:
 110         close(fd);
 111         return NULL;
 112 }
 113
 114 void
 115 roc_bphy_intr_fini(struct roc_bphy_irq_chip *irq_chip)
 116 {
 117         if (irq_chip == NULL)
 118                 return;
 119
 120         close(irq_chip->intfd);
 121         plt_free(irq_chip->mz_name);
 122         plt_free(irq_chip->irq_vecs);
 123         plt_free(irq_chip);
 124 }
 125
 126 static void
 127 roc_bphy_irq_stack_remove(int cpu)
 128 {
 129         struct roc_bphy_irq_stack *curr_stack;
 130
 131         if (pthread_mutex_lock(&stacks_mutex))
 132                 return;
 133
 134         STAILQ_FOREACH(curr_stack, &irq_stacks, entries) {
 135                 if (curr_stack->cpu == cpu)
 136                         break;
 137         }
 138
 139         if (curr_stack == NULL)
 140                 goto leave;
 141
 142         if (curr_stack->inuse > 0)
 143                 curr_stack->inuse--;
 144
 145         if (curr_stack->inuse == 0) {
 146                 STAILQ_REMOVE(&irq_stacks, curr_stack, roc_bphy_irq_stack,
 147                               entries);
 148                 plt_free(curr_stack->sp_buffer);
 149                 plt_free(curr_stack);
 150         }
 151
 152 leave:
 153         pthread_mutex_unlock(&stacks_mutex);
 154 }
 155
 156 static void *
 157 roc_bphy_irq_stack_get(int cpu)
 158 {
 159 #define ARM_STACK_ALIGNMENT (2 * sizeof(void *))
 160 #define IRQ_ISR_STACK_SIZE  0x200000
 161
 162         struct roc_bphy_irq_stack *curr_stack;
 163         void *retval = NULL;
 164
 165         if (pthread_mutex_lock(&stacks_mutex))
 166                 return NULL;
 167
 168         STAILQ_FOREACH(curr_stack, &irq_stacks, entries) {
 169                 if (curr_stack->cpu == cpu) {
 170                         curr_stack->inuse++;
 171                         retval = ((char *)curr_stack->sp_buffer) +
 172                                  IRQ_ISR_STACK_SIZE;
 173                         goto found_stack;
 174                 }
 175         }
 176
 177         curr_stack = plt_zmalloc(sizeof(struct roc_bphy_irq_stack), 0);
 178         if (curr_stack == NULL)
 179                 goto err_stack;
 180
 181         curr_stack->sp_buffer =
 182                 plt_zmalloc(IRQ_ISR_STACK_SIZE * 2, ARM_STACK_ALIGNMENT);
 183         if (curr_stack->sp_buffer == NULL)
 184                 goto err_buffer;
 185
 186         curr_stack->cpu = cpu;
 187         curr_stack->inuse = 0;
 188         STAILQ_INSERT_TAIL(&irq_stacks, curr_stack, entries);
 189         retval = ((char *)curr_stack->sp_buffer) + IRQ_ISR_STACK_SIZE;
 190
 191 found_stack:
 192         pthread_mutex_unlock(&stacks_mutex);
 193         return retval;
 194
 195 err_buffer:
 196         plt_free(curr_stack);
 197
 198 err_stack:
 199         pthread_mutex_unlock(&stacks_mutex);
 200         return NULL;
 201 }
 202
 203 void
 204 roc_bphy_intr_handler(unsigned int irq_num)
 205 {
 206         struct roc_bphy_irq_chip *irq_chip;
 207         const struct plt_memzone *mz;
 208
 209         mz = plt_memzone_lookup(ROC_BPHY_MEMZONE_NAME);
 210         if (mz == NULL)
 211                 return;
 212
 213         irq_chip = *(struct roc_bphy_irq_chip **)mz->addr;
 214         if (irq_chip == NULL)
 215                 return;
 216
 217         if (irq_chip->irq_vecs[irq_num].handler != NULL)
 218                 irq_chip->irq_vecs[irq_num].handler(
 219                         (int)irq_num, irq_chip->irq_vecs[irq_num].isr_data);
 220
 221         roc_atf_ret();
 222 }
 223
 224 static int
 225 roc_bphy_irq_handler_set(struct roc_bphy_irq_chip *chip, int irq_num,
 226                          void (*isr)(int irq_num, void *isr_data),
 227                          void *isr_data)
 228 {
 229         roc_cpuset_t orig_cpuset, intr_cpuset;
 230         struct roc_bphy_irq_usr_data irq_usr;
 231         const struct plt_memzone *mz;
 232         int i, retval, curr_cpu, rc;
 233         char *env;
 234
 235         mz = plt_memzone_lookup(chip->mz_name);
 236         if (mz == NULL) {
 237                 /* what we want is just a pointer to chip, not object itself */
 238                 mz = plt_memzone_reserve_cache_align(chip->mz_name,
 239                                                      sizeof(chip));
 240                 if (mz == NULL)
 241                         return -ENOMEM;
 242         }
 243
 244         if (chip->irq_vecs[irq_num].handler != NULL)
 245                 return -EINVAL;
 246
 247         rc = pthread_getaffinity_np(pthread_self(), sizeof(orig_cpuset),
 248                                     &orig_cpuset);
 249         if (rc < 0) {
 250                 plt_err("Failed to get affinity mask");
 251                 return rc;
 252         }
 253
 254         for (curr_cpu = -1, i = 0; i < CPU_SETSIZE; i++)
 255                 if (CPU_ISSET(i, &orig_cpuset))
 256                         curr_cpu = i;
 257         if (curr_cpu < 0)
 258                 return -ENOENT;
 259
 260         CPU_ZERO(&intr_cpuset);
 261         CPU_SET(curr_cpu, &intr_cpuset);
 262         retval = pthread_setaffinity_np(pthread_self(), sizeof(intr_cpuset),
 263                                         &intr_cpuset);
 264         if (rc < 0) {
 265                 plt_err("Failed to set affinity mask");
 266                 return rc;
 267         }
 268
 269         irq_usr.isr_base = (uint64_t)roc_bphy_intr_handler;
 270         irq_usr.sp = (uint64_t)roc_bphy_irq_stack_get(curr_cpu);
 271         irq_usr.cpu = curr_cpu;
 272         if (irq_usr.sp == 0) {
 273                 rc = pthread_setaffinity_np(pthread_self(), sizeof(orig_cpuset),
 274                                             &orig_cpuset);
 275                 if (rc < 0)
 276                         plt_err("Failed to restore affinity mask");
 277                 return rc;
 278         }
 279
 280         /* On simulator memory locking operation takes much time. We want
 281          * to skip this when running in such an environment.
 282          */
 283         env = getenv("BPHY_INTR_MLOCK_DISABLE");
 284         if (env == NULL) {
 285                 rc = mlockall(MCL_CURRENT | MCL_FUTURE);
 286                 if (rc < 0)
 287                         plt_warn("Failed to lock memory into RAM");
 288         }
 289
 290         *((struct roc_bphy_irq_chip **)(mz->addr)) = chip;
 291         irq_usr.irq_num = irq_num;
 292         chip->irq_vecs[irq_num].handler_cpu = curr_cpu;
 293         chip->irq_vecs[irq_num].handler = isr;
 294         chip->irq_vecs[irq_num].isr_data = isr_data;
 295         retval = ioctl(chip->intfd, ROC_BPHY_IOC_SET_BPHY_HANDLER, &irq_usr);
 296         if (retval != 0) {
 297                 roc_bphy_irq_stack_remove(curr_cpu);
 298                 chip->irq_vecs[irq_num].handler = NULL;
 299                 chip->irq_vecs[irq_num].handler_cpu = -1;
 300         } else {
 301                 chip->n_handlers++;
 302         }
 303
 304         rc = pthread_setaffinity_np(pthread_self(), sizeof(orig_cpuset),
 305                                     &orig_cpuset);
 306         if (rc < 0)
 307                 plt_warn("Failed to restore affinity mask");
 308
 309         return retval;
 310 }
 311
 312 bool
 313 roc_bphy_intr_available(struct roc_bphy_irq_chip *irq_chip, int irq_num)
 314 {
 315         if (irq_num < 0 || (uint64_t)irq_num >= irq_chip->max_irq)
 316                 return false;
 317
 318         return irq_chip->avail_irq_bmask & BIT(irq_num);
 319 }
 320
 321 int
 322 roc_bphy_intr_clear(struct roc_bphy_irq_chip *chip, int irq_num)
 323 {
 324         roc_cpuset_t orig_cpuset, intr_cpuset;
 325         const struct plt_memzone *mz;
 326         int retval;
 327
 328         if (chip == NULL)
 329                 return -EINVAL;
 330         if ((uint64_t)irq_num >= chip->max_irq || irq_num < 0)
 331                 return -EINVAL;
 332         if (!roc_bphy_intr_available(chip, irq_num))
 333                 return -ENOTSUP;
 334         if (chip->irq_vecs[irq_num].handler == NULL)
 335                 return -EINVAL;
 336         mz = plt_memzone_lookup(chip->mz_name);
 337         if (mz == NULL)
 338                 return -ENXIO;
 339
 340         retval = pthread_getaffinity_np(pthread_self(), sizeof(orig_cpuset),
 341                                         &orig_cpuset);
 342         if (retval < 0) {
 343                 plt_warn("Failed to get affinity mask");
 344                 CPU_ZERO(&orig_cpuset);
 345                 CPU_SET(0, &orig_cpuset);
 346         }
 347
 348         CPU_ZERO(&intr_cpuset);
 349         CPU_SET(chip->irq_vecs[irq_num].handler_cpu, &intr_cpuset);
 350         retval = pthread_setaffinity_np(pthread_self(), sizeof(intr_cpuset),
 351                                         &intr_cpuset);
 352         if (retval < 0) {
 353                 plt_warn("Failed to set affinity mask");
 354                 CPU_ZERO(&orig_cpuset);
 355                 CPU_SET(0, &orig_cpuset);
 356         }
 357
 358         retval = ioctl(chip->intfd, ROC_BPHY_IOC_CLR_BPHY_HANDLER, irq_num);
 359         if (retval == 0) {
 360                 roc_bphy_irq_stack_remove(chip->irq_vecs[irq_num].handler_cpu);
 361                 chip->n_handlers--;
 362                 chip->irq_vecs[irq_num].isr_data = NULL;
 363                 chip->irq_vecs[irq_num].handler = NULL;
 364                 chip->irq_vecs[irq_num].handler_cpu = -1;
 365                 if (chip->n_handlers == 0) {
 366                         retval = plt_memzone_free(mz);
 367                         if (retval < 0)
 368                                 plt_err("Failed to free memzone: irq %d",
 369                                         irq_num);
 370                 }
 371         } else {
 372                 plt_err("Failed to clear bphy interrupt handler");
 373         }
 374
 375         retval = pthread_setaffinity_np(pthread_self(), sizeof(orig_cpuset),
 376                                         &orig_cpuset);
 377         if (retval < 0) {
 378                 plt_warn("Failed to restore affinity mask");
 379                 CPU_ZERO(&orig_cpuset);
 380                 CPU_SET(0, &orig_cpuset);
 381         }
 382
 383         return retval;
 384 }
 385
 386 int
 387 roc_bphy_intr_register(struct roc_bphy_irq_chip *irq_chip,
 388                        struct roc_bphy_intr *intr)
 389 {
 390         roc_cpuset_t orig_cpuset, intr_cpuset;
 391         int retval;
 392         int ret;
 393
 394         if (!roc_bphy_intr_available(irq_chip, intr->irq_num))
 395                 return -ENOTSUP;
 396
 397         retval = pthread_getaffinity_np(pthread_self(), sizeof(orig_cpuset),
 398                                         &orig_cpuset);
 399         if (retval < 0) {
 400                 plt_err("Failed to get affinity mask");
 401                 return retval;
 402         }
 403
 404         CPU_ZERO(&intr_cpuset);
 405         CPU_SET(intr->cpu, &intr_cpuset);
 406         retval = pthread_setaffinity_np(pthread_self(), sizeof(intr_cpuset),
 407                                         &intr_cpuset);
 408         if (retval < 0) {
 409                 plt_err("Failed to set affinity mask");
 410                 return retval;
 411         }
 412
 413         ret = roc_bphy_irq_handler_set(irq_chip, intr->irq_num,
 414                                        intr->intr_handler, intr->isr_data);
 415
 416         retval = pthread_setaffinity_np(pthread_self(), sizeof(orig_cpuset),
 417                                         &orig_cpuset);
 418         if (retval < 0)
 419                 plt_warn("Failed to restore affinity mask");
 420
 421         return ret;
 422 }