ring: fix unaligned memory access on aarch32

[dpdk.git] / lib / librte_ring / rte_ring_elem.h

diff --git a/lib/librte_ring/rte_ring_elem.h b/lib/librte_ring/rte_ring_elem.h
index 15d79bf..663addc 100644 (file)
--- a/lib/librte_ring/rte_ring_elem.h
+++ b/lib/librte_ring/rte_ring_elem.h
@@ -110,8 +110,8 @@ struct rte_ring *rte_ring_create_elem(const char *name, unsigned int esize,
                        unsigned int count, int socket_id, unsigned int flags);
 
 static __rte_always_inline void
-enqueue_elems_32(struct rte_ring *r, const uint32_t size, uint32_t idx,
-               const void *obj_table, uint32_t n)
+__rte_ring_enqueue_elems_32(struct rte_ring *r, const uint32_t size,
+               uint32_t idx, const void *obj_table, uint32_t n)
 {
        unsigned int i;
        uint32_t *ring = (uint32_t *)&r[1];
@@ -153,14 +153,14 @@ enqueue_elems_32(struct rte_ring *r, const uint32_t size, uint32_t idx,
 }
 
 static __rte_always_inline void
-enqueue_elems_64(struct rte_ring *r, uint32_t prod_head,
+__rte_ring_enqueue_elems_64(struct rte_ring *r, uint32_t prod_head,
                const void *obj_table, uint32_t n)
 {
        unsigned int i;
        const uint32_t size = r->size;
        uint32_t idx = prod_head & r->mask;
        uint64_t *ring = (uint64_t *)&r[1];
-       const uint64_t *obj = (const uint64_t *)obj_table;
+       const unaligned_uint64_t *obj = (const unaligned_uint64_t *)obj_table;
        if (likely(idx + n < size)) {
                for (i = 0; i < (n & ~0x3); i += 4, idx += 4) {
                        ring[idx] = obj[i];
@@ -186,7 +186,7 @@ enqueue_elems_64(struct rte_ring *r, uint32_t prod_head,
 }
 
 static __rte_always_inline void
-enqueue_elems_128(struct rte_ring *r, uint32_t prod_head,
+__rte_ring_enqueue_elems_128(struct rte_ring *r, uint32_t prod_head,
                const void *obj_table, uint32_t n)
 {
        unsigned int i;
@@ -219,16 +219,16 @@ enqueue_elems_128(struct rte_ring *r, uint32_t prod_head,
  * single and multi producer enqueue functions.
  */
 static __rte_always_inline void
-enqueue_elems(struct rte_ring *r, uint32_t prod_head, const void *obj_table,
-               uint32_t esize, uint32_t num)
+__rte_ring_enqueue_elems(struct rte_ring *r, uint32_t prod_head,
+               const void *obj_table, uint32_t esize, uint32_t num)
 {
        /* 8B and 16B copies implemented individually to retain
         * the current performance.
         */
        if (esize == 8)
-               enqueue_elems_64(r, prod_head, obj_table, num);
+               __rte_ring_enqueue_elems_64(r, prod_head, obj_table, num);
        else if (esize == 16)
-               enqueue_elems_128(r, prod_head, obj_table, num);
+               __rte_ring_enqueue_elems_128(r, prod_head, obj_table, num);
        else {
                uint32_t idx, scale, nr_idx, nr_num, nr_size;
 
@@ -238,13 +238,14 @@ enqueue_elems(struct rte_ring *r, uint32_t prod_head, const void *obj_table,
                idx = prod_head & r->mask;
                nr_idx = idx * scale;
                nr_size = r->size * scale;
-               enqueue_elems_32(r, nr_size, nr_idx, obj_table, nr_num);
+               __rte_ring_enqueue_elems_32(r, nr_size, nr_idx,
+                               obj_table, nr_num);
        }
 }
 
 static __rte_always_inline void
-dequeue_elems_32(struct rte_ring *r, const uint32_t size, uint32_t idx,
-               void *obj_table, uint32_t n)
+__rte_ring_dequeue_elems_32(struct rte_ring *r, const uint32_t size,
+               uint32_t idx, void *obj_table, uint32_t n)
 {
        unsigned int i;
        uint32_t *ring = (uint32_t *)&r[1];
@@ -286,14 +287,14 @@ dequeue_elems_32(struct rte_ring *r, const uint32_t size, uint32_t idx,
 }
 
 static __rte_always_inline void
-dequeue_elems_64(struct rte_ring *r, uint32_t prod_head,
+__rte_ring_dequeue_elems_64(struct rte_ring *r, uint32_t prod_head,
                void *obj_table, uint32_t n)
 {
        unsigned int i;
        const uint32_t size = r->size;
        uint32_t idx = prod_head & r->mask;
        uint64_t *ring = (uint64_t *)&r[1];
-       uint64_t *obj = (uint64_t *)obj_table;
+       unaligned_uint64_t *obj = (unaligned_uint64_t *)obj_table;
        if (likely(idx + n < size)) {
                for (i = 0; i < (n & ~0x3); i += 4, idx += 4) {
                        obj[i] = ring[idx];
@@ -319,7 +320,7 @@ dequeue_elems_64(struct rte_ring *r, uint32_t prod_head,
 }
 
 static __rte_always_inline void
-dequeue_elems_128(struct rte_ring *r, uint32_t prod_head,
+__rte_ring_dequeue_elems_128(struct rte_ring *r, uint32_t prod_head,
                void *obj_table, uint32_t n)
 {
        unsigned int i;
@@ -348,16 +349,16 @@ dequeue_elems_128(struct rte_ring *r, uint32_t prod_head,
  * single and multi producer enqueue functions.
  */
 static __rte_always_inline void
-dequeue_elems(struct rte_ring *r, uint32_t cons_head, void *obj_table,
-               uint32_t esize, uint32_t num)
+__rte_ring_dequeue_elems(struct rte_ring *r, uint32_t cons_head,
+               void *obj_table, uint32_t esize, uint32_t num)
 {
        /* 8B and 16B copies implemented individually to retain
         * the current performance.
         */
        if (esize == 8)
-               dequeue_elems_64(r, cons_head, obj_table, num);
+               __rte_ring_dequeue_elems_64(r, cons_head, obj_table, num);
        else if (esize == 16)
-               dequeue_elems_128(r, cons_head, obj_table, num);
+               __rte_ring_dequeue_elems_128(r, cons_head, obj_table, num);
        else {
                uint32_t idx, scale, nr_idx, nr_num, nr_size;
 
@@ -367,7 +368,8 @@ dequeue_elems(struct rte_ring *r, uint32_t cons_head, void *obj_table,
                idx = cons_head & r->mask;
                nr_idx = idx * scale;
                nr_size = r->size * scale;
-               dequeue_elems_32(r, nr_size, nr_idx, obj_table, nr_num);
+               __rte_ring_dequeue_elems_32(r, nr_size, nr_idx,
+                               obj_table, nr_num);
        }
 }
 
@@ -392,7 +394,7 @@ dequeue_elems(struct rte_ring *r, uint32_t cons_head, void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects).
+ *   A pointer to a table of objects.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -424,7 +426,7 @@ __rte_ring_do_enqueue_elem(struct rte_ring *r, const void *obj_table,
        if (n == 0)
                goto end;
 
-       enqueue_elems(r, prod_head, obj_table, esize, n);
+       __rte_ring_enqueue_elems(r, prod_head, obj_table, esize, n);
 
        update_tail(&r->prod, prod_head, prod_next, is_sp, 1);
 end:
@@ -439,7 +441,7 @@ end:
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects).
+ *   A pointer to a table of objects.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -471,7 +473,7 @@ __rte_ring_do_dequeue_elem(struct rte_ring *r, void *obj_table,
        if (n == 0)
                goto end;
 
-       dequeue_elems(r, cons_head, obj_table, esize, n);
+       __rte_ring_dequeue_elems(r, cons_head, obj_table, esize, n);
 
        update_tail(&r->cons, cons_head, cons_next, is_sc, 0);
 
@@ -490,7 +492,7 @@ end:
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects).
+ *   A pointer to a table of objects.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -519,7 +521,7 @@ rte_ring_mp_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects).
+ *   A pointer to a table of objects.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -550,7 +552,7 @@ rte_ring_sp_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects).
+ *   A pointer to a table of objects.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -655,7 +657,7 @@ rte_ring_enqueue_elem(struct rte_ring *r, void *obj, unsigned int esize)
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects) that will be filled.
+ *   A pointer to a table of objects that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -682,7 +684,7 @@ rte_ring_mc_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects) that will be filled.
+ *   A pointer to a table of objects that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -714,7 +716,7 @@ rte_ring_sc_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects) that will be filled.
+ *   A pointer to a table of objects that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -744,7 +746,7 @@ rte_ring_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_p
- *   A pointer to a void * pointer (object) that will be filled.
+ *   A pointer to the object that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -768,7 +770,7 @@ rte_ring_mc_dequeue_elem(struct rte_ring *r, void *obj_p,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_p
- *   A pointer to a void * pointer (object) that will be filled.
+ *   A pointer to the object that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -796,7 +798,7 @@ rte_ring_sc_dequeue_elem(struct rte_ring *r, void *obj_p,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_p
- *   A pointer to a void * pointer (object) that will be filled.
+ *   A pointer to the object that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -822,7 +824,7 @@ rte_ring_dequeue_elem(struct rte_ring *r, void *obj_p, unsigned int esize)
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects).
+ *   A pointer to a table of objects.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -851,7 +853,7 @@ rte_ring_mp_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects).
+ *   A pointer to a table of objects.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -882,7 +884,7 @@ rte_ring_sp_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects).
+ *   A pointer to a table of objects.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -914,7 +916,7 @@ rte_ring_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects) that will be filled.
+ *   A pointer to a table of objects that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -943,7 +945,7 @@ rte_ring_mc_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects) that will be filled.
+ *   A pointer to a table of objects that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise
@@ -974,7 +976,7 @@ rte_ring_sc_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
  * @param r
  *   A pointer to the ring structure.
  * @param obj_table
- *   A pointer to a table of void * pointers (objects) that will be filled.
+ *   A pointer to a table of objects that will be filled.
  * @param esize
  *   The size of ring element, in bytes. It must be a multiple of 4.
  *   This must be the same value used while creating the ring. Otherwise