drivers/mempool/dpaa/dpaa_mempool.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 NXP.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of NXP nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 /* System headers */
  34 #include <stdio.h>
  35 #include <inttypes.h>
  36 #include <unistd.h>
  37 #include <limits.h>
  38 #include <sched.h>
  39 #include <signal.h>
  40 #include <pthread.h>
  41 #include <sys/types.h>
  42 #include <sys/syscall.h>
  43
  44 #include <rte_byteorder.h>
  45 #include <rte_common.h>
  46 #include <rte_log.h>
  47 #include <rte_debug.h>
  48 #include <rte_memory.h>
  49 #include <rte_memzone.h>
  50 #include <rte_tailq.h>
  51 #include <rte_eal.h>
  52 #include <rte_malloc.h>
  53 #include <rte_ring.h>
  54
  55 #include <dpaa_mempool.h>
  56
  57 struct dpaa_bp_info rte_dpaa_bpid_info[DPAA_MAX_BPOOLS];
  58
  59 static int
  60 dpaa_mbuf_create_pool(struct rte_mempool *mp)
  61 {
  62         struct bman_pool *bp;
  63         struct bm_buffer bufs[8];
  64         struct dpaa_bp_info *bp_info;
  65         uint8_t bpid;
  66         int num_bufs = 0, ret = 0;
  67         struct bman_pool_params params = {
  68                 .flags = BMAN_POOL_FLAG_DYNAMIC_BPID
  69         };
  70
  71         MEMPOOL_INIT_FUNC_TRACE();
  72
  73         bp = bman_new_pool(&params);
  74         if (!bp) {
  75                 DPAA_MEMPOOL_ERR("bman_new_pool() failed");
  76                 return -ENODEV;
  77         }
  78         bpid = bman_get_params(bp)->bpid;
  79
  80         /* Drain the pool of anything already in it. */
  81         do {
  82                 /* Acquire is all-or-nothing, so we drain in 8s,
  83                  * then in 1s for the remainder.
  84                  */
  85                 if (ret != 1)
  86                         ret = bman_acquire(bp, bufs, 8, 0);
  87                 if (ret < 8)
  88                         ret = bman_acquire(bp, bufs, 1, 0);
  89                 if (ret > 0)
  90                         num_bufs += ret;
  91         } while (ret > 0);
  92         if (num_bufs)
  93                 DPAA_MEMPOOL_WARN("drained %u bufs from BPID %d",
  94                                   num_bufs, bpid);
  95
  96         rte_dpaa_bpid_info[bpid].mp = mp;
  97         rte_dpaa_bpid_info[bpid].bpid = bpid;
  98         rte_dpaa_bpid_info[bpid].size = mp->elt_size;
  99         rte_dpaa_bpid_info[bpid].bp = bp;
 100         rte_dpaa_bpid_info[bpid].meta_data_size =
 101                 sizeof(struct rte_mbuf) + rte_pktmbuf_priv_size(mp);
 102         rte_dpaa_bpid_info[bpid].dpaa_ops_index = mp->ops_index;
 103
 104         bp_info = rte_malloc(NULL,
 105                              sizeof(struct dpaa_bp_info),
 106                              RTE_CACHE_LINE_SIZE);
 107         if (!bp_info) {
 108                 DPAA_MEMPOOL_WARN("Memory allocation failed for bp_info");
 109                 bman_free_pool(bp);
 110                 return -ENOMEM;
 111         }
 112
 113         rte_memcpy(bp_info, (void *)&rte_dpaa_bpid_info[bpid],
 114                    sizeof(struct dpaa_bp_info));
 115         mp->pool_data = (void *)bp_info;
 116
 117         DPAA_MEMPOOL_INFO("BMAN pool created for bpid =%d", bpid);
 118         return 0;
 119 }
 120
 121 static void
 122 dpaa_mbuf_free_pool(struct rte_mempool *mp)
 123 {
 124         struct dpaa_bp_info *bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp);
 125
 126         MEMPOOL_INIT_FUNC_TRACE();
 127
 128         if (bp_info) {
 129                 bman_free_pool(bp_info->bp);
 130                 DPAA_MEMPOOL_INFO("BMAN pool freed for bpid =%d",
 131                                   bp_info->bpid);
 132                 rte_free(mp->pool_data);
 133                 mp->pool_data = NULL;
 134         }
 135 }
 136
 137 static void
 138 dpaa_buf_free(struct dpaa_bp_info *bp_info, uint64_t addr)
 139 {
 140         struct bm_buffer buf;
 141         int ret;
 142
 143         DPAA_MEMPOOL_DEBUG("Free 0x%lx to bpid: %d", addr, bp_info->bpid);
 144
 145         bm_buffer_set64(&buf, addr);
 146 retry:
 147         ret = bman_release(bp_info->bp, &buf, 1, 0);
 148         if (ret) {
 149                 DPAA_MEMPOOL_DEBUG("BMAN busy. Retrying...");
 150                 cpu_spin(CPU_SPIN_BACKOFF_CYCLES);
 151                 goto retry;
 152         }
 153 }
 154
 155 static int
 156 dpaa_mbuf_free_bulk(struct rte_mempool *pool,
 157                     void *const *obj_table,
 158                     unsigned int n)
 159 {
 160         struct dpaa_bp_info *bp_info = DPAA_MEMPOOL_TO_POOL_INFO(pool);
 161         int ret;
 162         unsigned int i = 0;
 163
 164         DPAA_MEMPOOL_DPDEBUG("Request to free %d buffers in bpid = %d",
 165                              n, bp_info->bpid);
 166
 167         ret = rte_dpaa_portal_init((void *)0);
 168         if (ret) {
 169                 DPAA_MEMPOOL_ERR("rte_dpaa_portal_init failed with ret: %d",
 170                                  ret);
 171                 return 0;
 172         }
 173
 174         while (i < n) {
 175                 dpaa_buf_free(bp_info,
 176                               (uint64_t)rte_mempool_virt2phy(pool,
 177                               obj_table[i]) + bp_info->meta_data_size);
 178                 i = i + 1;
 179         }
 180
 181         DPAA_MEMPOOL_DPDEBUG("freed %d buffers in bpid =%d",
 182                              n, bp_info->bpid);
 183
 184         return 0;
 185 }
 186
 187 static int
 188 dpaa_mbuf_alloc_bulk(struct rte_mempool *pool,
 189                      void **obj_table,
 190                      unsigned int count)
 191 {
 192         struct rte_mbuf **m = (struct rte_mbuf **)obj_table;
 193         struct bm_buffer bufs[DPAA_MBUF_MAX_ACQ_REL];
 194         struct dpaa_bp_info *bp_info;
 195         void *bufaddr;
 196         int i, ret;
 197         unsigned int n = 0;
 198
 199         bp_info = DPAA_MEMPOOL_TO_POOL_INFO(pool);
 200
 201         DPAA_MEMPOOL_DPDEBUG("Request to alloc %d buffers in bpid = %d",
 202                              count, bp_info->bpid);
 203
 204         if (unlikely(count >= (RTE_MEMPOOL_CACHE_MAX_SIZE * 2))) {
 205                 DPAA_MEMPOOL_ERR("Unable to allocate requested (%u) buffers",
 206                                  count);
 207                 return -1;
 208         }
 209
 210         ret = rte_dpaa_portal_init((void *)0);
 211         if (ret) {
 212                 DPAA_MEMPOOL_ERR("rte_dpaa_portal_init failed with ret: %d",
 213                                  ret);
 214                 return -1;
 215         }
 216
 217         while (n < count) {
 218                 /* Acquire is all-or-nothing, so we drain in 7s,
 219                  * then the remainder.
 220                  */
 221                 if ((count - n) > DPAA_MBUF_MAX_ACQ_REL) {
 222                         ret = bman_acquire(bp_info->bp, bufs,
 223                                            DPAA_MBUF_MAX_ACQ_REL, 0);
 224                 } else {
 225                         ret = bman_acquire(bp_info->bp, bufs, count - n, 0);
 226                 }
 227                 /* In case of less than requested number of buffers available
 228                  * in pool, qbman_swp_acquire returns 0
 229                  */
 230                 if (ret <= 0) {
 231                         DPAA_MEMPOOL_DPDEBUG("Buffer acquire failed (%d)",
 232                                              ret);
 233                         /* The API expect the exact number of requested
 234                          * buffers. Releasing all buffers allocated
 235                          */
 236                         dpaa_mbuf_free_bulk(pool, obj_table, n);
 237                         return -ENOBUFS;
 238                 }
 239                 /* assigning mbuf from the acquired objects */
 240                 for (i = 0; (i < ret) && bufs[i].addr; i++) {
 241                         /* TODO-errata - objerved that bufs may be null
 242                          * i.e. first buffer is valid, remaining 6 buffers
 243                          * may be null.
 244                          */
 245                         bufaddr = (void *)rte_dpaa_mem_ptov(bufs[i].addr);
 246                         m[n] = (struct rte_mbuf *)((char *)bufaddr
 247                                                 - bp_info->meta_data_size);
 248                         DPAA_MEMPOOL_DPDEBUG("Paddr (%p), FD (%p) from BMAN",
 249                                              (void *)bufaddr, (void *)m[n]);
 250                         n++;
 251                 }
 252         }
 253
 254         DPAA_MEMPOOL_DPDEBUG("Allocated %d buffers from bpid=%d",
 255                              n, bp_info->bpid);
 256         return 0;
 257 }
 258
 259 static unsigned int
 260 dpaa_mbuf_get_count(const struct rte_mempool *mp)
 261 {
 262         struct dpaa_bp_info *bp_info;
 263
 264         MEMPOOL_INIT_FUNC_TRACE();
 265
 266         if (!mp || !mp->pool_data) {
 267                 DPAA_MEMPOOL_ERR("Invalid mempool provided\n");
 268                 return 0;
 269         }
 270
 271         bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp);
 272
 273         return bman_query_free_buffers(bp_info->bp);
 274 }
 275
 276 struct rte_mempool_ops dpaa_mpool_ops = {
 277         .name = "dpaa",
 278         .alloc = dpaa_mbuf_create_pool,
 279         .free = dpaa_mbuf_free_pool,
 280         .enqueue = dpaa_mbuf_free_bulk,
 281         .dequeue = dpaa_mbuf_alloc_bulk,
 282         .get_count = dpaa_mbuf_get_count,
 283 };
 284
 285 MEMPOOL_REGISTER_OPS(dpaa_mpool_ops);