From fd5f33323e5a698941db03908d5bd1fa91016b7e Mon Sep 17 00:00:00 2001 From: Phil Yang Date: Mon, 8 Oct 2018 17:11:46 +0800 Subject: [PATCH] kni: introduce C11 atomic into FIFO synchronization Syncing the values by adding c11 atomic memory barriers to make sure the values being synced before updating fifo_write and fifo_read. Signed-off-by: Phil Yang Reviewed-by: Honnappa Nagarahalli Reviewed-by: Gavin Hu Reviewed-by: Ola Liljedahl Reviewed-by: Ferruh Yigit --- .../eal/include/exec-env/rte_kni_common.h | 5 ++ lib/librte_kni/rte_kni_fifo.h | 47 ++++++++++++++----- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h index 58e8533bec..5afa087131 100644 --- a/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h +++ b/lib/librte_eal/linuxapp/eal/include/exec-env/rte_kni_common.h @@ -55,8 +55,13 @@ struct rte_kni_request { * Writing should never overwrite the read position */ struct rte_kni_fifo { +#ifdef RTE_USE_C11_MEM_MODEL + unsigned write; /**< Next position to be written*/ + unsigned read; /**< Next position to be read */ +#else volatile unsigned write; /**< Next position to be written*/ volatile unsigned read; /**< Next position to be read */ +#endif unsigned len; /**< Circular buffer length */ unsigned elem_size; /**< Pointer size - for 32/64 bit OS */ void *volatile buffer[]; /**< The buffer contains mbuf pointers */ diff --git a/lib/librte_kni/rte_kni_fifo.h b/lib/librte_kni/rte_kni_fifo.h index 70ac14eec2..287d7deb29 100644 --- a/lib/librte_kni/rte_kni_fifo.h +++ b/lib/librte_kni/rte_kni_fifo.h @@ -4,6 +4,36 @@ +/** + * @internal when c11 memory model enabled use c11 atomic memory barrier. + * when under non c11 memory model use rte_smp_* memory barrier. + * + * @param src + * Pointer to the source data. + * @param dst + * Pointer to the destination data. + * @param value + * Data value. + */ +#ifdef RTE_USE_C11_MEM_MODEL +#define __KNI_LOAD_ACQUIRE(src) ({ \ + __atomic_load_n((src), __ATOMIC_ACQUIRE); \ + }) +#define __KNI_STORE_RELEASE(dst, value) do { \ + __atomic_store_n((dst), value, __ATOMIC_RELEASE); \ + } while(0) +#else +#define __KNI_LOAD_ACQUIRE(src) ({ \ + typeof (*(src)) val = *(src); \ + rte_smp_rmb(); \ + val; \ + }) +#define __KNI_STORE_RELEASE(dst, value) do { \ + *(dst) = value; \ + rte_smp_wmb(); \ + } while(0) +#endif + /** * Initializes the kni fifo structure */ @@ -29,8 +59,7 @@ kni_fifo_put(struct rte_kni_fifo *fifo, void **data, unsigned num) unsigned i = 0; unsigned fifo_write = fifo->write; unsigned new_write = fifo_write; - rte_smp_rmb(); - unsigned fifo_read = fifo->read; + unsigned fifo_read = __KNI_LOAD_ACQUIRE(&fifo->read); for (i = 0; i < num; i++) { new_write = (new_write + 1) & (fifo->len - 1); @@ -40,8 +69,7 @@ kni_fifo_put(struct rte_kni_fifo *fifo, void **data, unsigned num) fifo->buffer[fifo_write] = data[i]; fifo_write = new_write; } - rte_smp_wmb(); - fifo->write = fifo_write; + __KNI_STORE_RELEASE(&fifo->write, fifo_write); return i; } @@ -53,8 +81,7 @@ kni_fifo_get(struct rte_kni_fifo *fifo, void **data, unsigned num) { unsigned i = 0; unsigned new_read = fifo->read; - rte_smp_rmb(); - unsigned fifo_write = fifo->write; + unsigned fifo_write = __KNI_LOAD_ACQUIRE(&fifo->write); for (i = 0; i < num; i++) { if (new_read == fifo_write) @@ -63,8 +90,7 @@ kni_fifo_get(struct rte_kni_fifo *fifo, void **data, unsigned num) data[i] = fifo->buffer[new_read]; new_read = (new_read + 1) & (fifo->len - 1); } - rte_smp_rmb(); - fifo->read = new_read; + __KNI_STORE_RELEASE(&fifo->read, new_read); return i; } @@ -74,8 +100,7 @@ kni_fifo_get(struct rte_kni_fifo *fifo, void **data, unsigned num) static inline uint32_t kni_fifo_count(struct rte_kni_fifo *fifo) { - unsigned fifo_write = fifo->write; - rte_smp_rmb(); - unsigned fifo_read = fifo->read; + unsigned fifo_write = __KNI_LOAD_ACQUIRE(&fifo->write); + unsigned fifo_read = __KNI_LOAD_ACQUIRE(&fifo->read); return (fifo->len + fifo_write - fifo_read) & (fifo->len - 1); } -- 2.20.1