kernel/freebsd/contigmem/contigmem.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <sys/cdefs.h>
   6 __FBSDID("$FreeBSD$");
   7
   8 #include <sys/param.h>
   9 #include <sys/bio.h>
  10 #include <sys/bus.h>
  11 #include <sys/conf.h>
  12 #include <sys/kernel.h>
  13 #include <sys/malloc.h>
  14 #include <sys/module.h>
  15 #include <sys/proc.h>
  16 #include <sys/rwlock.h>
  17 #include <sys/systm.h>
  18 #include <sys/sysctl.h>
  19 #include <sys/vmmeter.h>
  20
  21 #include <machine/bus.h>
  22
  23 #include <vm/vm.h>
  24 #include <vm/pmap.h>
  25 #include <vm/vm_param.h>
  26 #include <vm/vm_object.h>
  27 #include <vm/vm_page.h>
  28 #include <vm/vm_pager.h>
  29 #include <vm/vm_phys.h>
  30
  31 struct contigmem_buffer {
  32         void           *addr;
  33         int             refcnt;
  34         struct mtx      mtx;
  35 };
  36
  37 struct contigmem_vm_handle {
  38         int             buffer_index;
  39 };
  40
  41 static int              contigmem_load(void);
  42 static int              contigmem_unload(void);
  43 static int              contigmem_physaddr(SYSCTL_HANDLER_ARGS);
  44
  45 static d_mmap_single_t  contigmem_mmap_single;
  46 static d_open_t         contigmem_open;
  47 static d_close_t        contigmem_close;
  48
  49 static int              contigmem_num_buffers = RTE_CONTIGMEM_DEFAULT_NUM_BUFS;
  50 static int64_t          contigmem_buffer_size = RTE_CONTIGMEM_DEFAULT_BUF_SIZE;
  51
  52 static eventhandler_tag contigmem_eh_tag;
  53 static struct contigmem_buffer contigmem_buffers[RTE_CONTIGMEM_MAX_NUM_BUFS];
  54 static struct cdev     *contigmem_cdev = NULL;
  55 static int              contigmem_refcnt;
  56
  57 TUNABLE_INT("hw.contigmem.num_buffers", &contigmem_num_buffers);
  58 TUNABLE_QUAD("hw.contigmem.buffer_size", &contigmem_buffer_size);
  59
  60 static SYSCTL_NODE(_hw, OID_AUTO, contigmem, CTLFLAG_RD, 0, "contigmem");
  61
  62 SYSCTL_INT(_hw_contigmem, OID_AUTO, num_buffers, CTLFLAG_RD,
  63         &contigmem_num_buffers, 0, "Number of contigmem buffers allocated");
  64 SYSCTL_QUAD(_hw_contigmem, OID_AUTO, buffer_size, CTLFLAG_RD,
  65         &contigmem_buffer_size, 0, "Size of each contiguous buffer");
  66 SYSCTL_INT(_hw_contigmem, OID_AUTO, num_references, CTLFLAG_RD,
  67         &contigmem_refcnt, 0, "Number of references to contigmem");
  68
  69 static SYSCTL_NODE(_hw_contigmem, OID_AUTO, physaddr, CTLFLAG_RD, 0,
  70         "physaddr");
  71
  72 MALLOC_DEFINE(M_CONTIGMEM, "contigmem", "contigmem(4) allocations");
  73
  74 static int contigmem_modevent(module_t mod, int type, void *arg)
  75 {
  76         int error = 0;
  77
  78         switch (type) {
  79         case MOD_LOAD:
  80                 error = contigmem_load();
  81                 break;
  82         case MOD_UNLOAD:
  83                 error = contigmem_unload();
  84                 break;
  85         default:
  86                 break;
  87         }
  88
  89         return error;
  90 }
  91
  92 moduledata_t contigmem_mod = {
  93         "contigmem",
  94         (modeventhand_t)contigmem_modevent,
  95         0
  96 };
  97
  98 DECLARE_MODULE(contigmem, contigmem_mod, SI_SUB_DRIVERS, SI_ORDER_ANY);
  99 MODULE_VERSION(contigmem, 1);
 100
 101 static struct cdevsw contigmem_ops = {
 102         .d_name         = "contigmem",
 103         .d_version      = D_VERSION,
 104         .d_flags        = D_TRACKCLOSE,
 105         .d_mmap_single  = contigmem_mmap_single,
 106         .d_open         = contigmem_open,
 107         .d_close        = contigmem_close,
 108 };
 109
 110 static int
 111 contigmem_load()
 112 {
 113         char index_string[8], description[32];
 114         int  i, error = 0;
 115         void *addr;
 116
 117         if (contigmem_num_buffers > RTE_CONTIGMEM_MAX_NUM_BUFS) {
 118                 printf("%d buffers requested is greater than %d allowed\n",
 119                                 contigmem_num_buffers, RTE_CONTIGMEM_MAX_NUM_BUFS);
 120                 error = EINVAL;
 121                 goto error;
 122         }
 123
 124         if (contigmem_buffer_size < PAGE_SIZE ||
 125                         (contigmem_buffer_size & (contigmem_buffer_size - 1)) != 0) {
 126                 printf("buffer size 0x%lx is not greater than PAGE_SIZE and "
 127                                 "power of two\n", contigmem_buffer_size);
 128                 error = EINVAL;
 129                 goto error;
 130         }
 131
 132         for (i = 0; i < contigmem_num_buffers; i++) {
 133                 addr = contigmalloc(contigmem_buffer_size, M_CONTIGMEM, M_ZERO,
 134                         0, BUS_SPACE_MAXADDR, contigmem_buffer_size, 0);
 135                 if (addr == NULL) {
 136                         printf("contigmalloc failed for buffer %d\n", i);
 137                         error = ENOMEM;
 138                         goto error;
 139                 }
 140
 141                 printf("%2u: virt=%p phys=%p\n", i, addr,
 142                         (void *)pmap_kextract((vm_offset_t)addr));
 143
 144                 mtx_init(&contigmem_buffers[i].mtx, "contigmem", NULL, MTX_DEF);
 145                 contigmem_buffers[i].addr = addr;
 146                 contigmem_buffers[i].refcnt = 0;
 147
 148                 snprintf(index_string, sizeof(index_string), "%d", i);
 149                 snprintf(description, sizeof(description),
 150                                 "phys addr for buffer %d", i);
 151                 SYSCTL_ADD_PROC(NULL,
 152                                 &SYSCTL_NODE_CHILDREN(_hw_contigmem, physaddr), OID_AUTO,
 153                                 index_string, CTLTYPE_U64 | CTLFLAG_RD,
 154                                 (void *)(uintptr_t)i, 0, contigmem_physaddr, "LU",
 155                                 description);
 156         }
 157
 158         contigmem_cdev = make_dev_credf(0, &contigmem_ops, 0, NULL, UID_ROOT,
 159                         GID_WHEEL, 0600, "contigmem");
 160
 161         return 0;
 162
 163 error:
 164         for (i = 0; i < contigmem_num_buffers; i++) {
 165                 if (contigmem_buffers[i].addr != NULL)
 166                         contigfree(contigmem_buffers[i].addr,
 167                                 contigmem_buffer_size, M_CONTIGMEM);
 168                 if (mtx_initialized(&contigmem_buffers[i].mtx))
 169                         mtx_destroy(&contigmem_buffers[i].mtx);
 170         }
 171
 172         return error;
 173 }
 174
 175 static int
 176 contigmem_unload()
 177 {
 178         int i;
 179
 180         if (contigmem_refcnt > 0)
 181                 return EBUSY;
 182
 183         if (contigmem_cdev != NULL)
 184                 destroy_dev(contigmem_cdev);
 185
 186         if (contigmem_eh_tag != NULL)
 187                 EVENTHANDLER_DEREGISTER(process_exit, contigmem_eh_tag);
 188
 189         for (i = 0; i < RTE_CONTIGMEM_MAX_NUM_BUFS; i++) {
 190                 if (contigmem_buffers[i].addr != NULL)
 191                         contigfree(contigmem_buffers[i].addr,
 192                                 contigmem_buffer_size, M_CONTIGMEM);
 193                 if (mtx_initialized(&contigmem_buffers[i].mtx))
 194                         mtx_destroy(&contigmem_buffers[i].mtx);
 195         }
 196
 197         return 0;
 198 }
 199
 200 static int
 201 contigmem_physaddr(SYSCTL_HANDLER_ARGS)
 202 {
 203         uint64_t        physaddr;
 204         int             index = (int)(uintptr_t)arg1;
 205
 206         physaddr = (uint64_t)vtophys(contigmem_buffers[index].addr);
 207         return sysctl_handle_64(oidp, &physaddr, 0, req);
 208 }
 209
 210 static int
 211 contigmem_open(struct cdev *cdev, int fflags, int devtype,
 212                 struct thread *td)
 213 {
 214
 215         atomic_add_int(&contigmem_refcnt, 1);
 216
 217         return 0;
 218 }
 219
 220 static int
 221 contigmem_close(struct cdev *cdev, int fflags, int devtype,
 222                 struct thread *td)
 223 {
 224
 225         atomic_subtract_int(&contigmem_refcnt, 1);
 226
 227         return 0;
 228 }
 229
 230 static int
 231 contigmem_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
 232                 vm_ooffset_t foff, struct ucred *cred, u_short *color)
 233 {
 234         struct contigmem_vm_handle *vmh = handle;
 235         struct contigmem_buffer *buf;
 236
 237         buf = &contigmem_buffers[vmh->buffer_index];
 238
 239         atomic_add_int(&contigmem_refcnt, 1);
 240
 241         mtx_lock(&buf->mtx);
 242         if (buf->refcnt == 0)
 243                 memset(buf->addr, 0, contigmem_buffer_size);
 244         buf->refcnt++;
 245         mtx_unlock(&buf->mtx);
 246
 247         return 0;
 248 }
 249
 250 static void
 251 contigmem_cdev_pager_dtor(void *handle)
 252 {
 253         struct contigmem_vm_handle *vmh = handle;
 254         struct contigmem_buffer *buf;
 255
 256         buf = &contigmem_buffers[vmh->buffer_index];
 257
 258         mtx_lock(&buf->mtx);
 259         buf->refcnt--;
 260         mtx_unlock(&buf->mtx);
 261
 262         free(vmh, M_CONTIGMEM);
 263
 264         atomic_subtract_int(&contigmem_refcnt, 1);
 265 }
 266
 267 static int
 268 contigmem_cdev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot,
 269                 vm_page_t *mres)
 270 {
 271         vm_paddr_t paddr;
 272         vm_page_t m_paddr, page;
 273         vm_memattr_t memattr, memattr1;
 274
 275         memattr = object->memattr;
 276
 277         VM_OBJECT_WUNLOCK(object);
 278
 279         paddr = offset;
 280
 281         m_paddr = vm_phys_paddr_to_vm_page(paddr);
 282         if (m_paddr != NULL) {
 283                 memattr1 = pmap_page_get_memattr(m_paddr);
 284                 if (memattr1 != memattr)
 285                         memattr = memattr1;
 286         }
 287
 288         if (((*mres)->flags & PG_FICTITIOUS) != 0) {
 289                 /*
 290                  * If the passed in result page is a fake page, update it with
 291                  * the new physical address.
 292                  */
 293                 page = *mres;
 294                 VM_OBJECT_WLOCK(object);
 295                 vm_page_updatefake(page, paddr, memattr);
 296         } else {
 297                 vm_page_t mret;
 298                 /*
 299                  * Replace the passed in reqpage page with our own fake page and
 300                  * free up the original page.
 301                  */
 302                 page = vm_page_getfake(paddr, memattr);
 303                 VM_OBJECT_WLOCK(object);
 304                 mret = vm_page_replace(page, object, (*mres)->pindex);
 305                 KASSERT(mret == *mres,
 306                     ("invalid page replacement, old=%p, ret=%p", *mres, mret));
 307                 vm_page_lock(mret);
 308                 vm_page_free(mret);
 309                 vm_page_unlock(mret);
 310                 *mres = page;
 311         }
 312
 313         page->valid = VM_PAGE_BITS_ALL;
 314
 315         return VM_PAGER_OK;
 316 }
 317
 318 static struct cdev_pager_ops contigmem_cdev_pager_ops = {
 319         .cdev_pg_ctor = contigmem_cdev_pager_ctor,
 320         .cdev_pg_dtor = contigmem_cdev_pager_dtor,
 321         .cdev_pg_fault = contigmem_cdev_pager_fault,
 322 };
 323
 324 static int
 325 contigmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size,
 326                 struct vm_object **obj, int nprot)
 327 {
 328         struct contigmem_vm_handle *vmh;
 329         uint64_t buffer_index;
 330
 331         /*
 332          * The buffer index is encoded in the offset.  Divide the offset by
 333          *  PAGE_SIZE to get the index of the buffer requested by the user
 334          *  app.
 335          */
 336         buffer_index = *offset / PAGE_SIZE;
 337         if (buffer_index >= contigmem_num_buffers)
 338                 return EINVAL;
 339
 340         if (size > contigmem_buffer_size)
 341                 return EINVAL;
 342
 343         vmh = malloc(sizeof(*vmh), M_CONTIGMEM, M_NOWAIT | M_ZERO);
 344         if (vmh == NULL)
 345                 return ENOMEM;
 346         vmh->buffer_index = buffer_index;
 347
 348         *offset = (vm_ooffset_t)vtophys(contigmem_buffers[buffer_index].addr);
 349         *obj = cdev_pager_allocate(vmh, OBJT_DEVICE, &contigmem_cdev_pager_ops,
 350                         size, nprot, *offset, curthread->td_ucred);
 351
 352         return 0;
 353 }