1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2021 Xilinx, Inc.
5 #include <rte_mempool.h>
6 #include <rte_memzone.h>
12 #include "sfc_nic_dma.h"
15 sfc_nic_dma_add_region(struct sfc_nic_dma_info *nic_dma_info,
16 rte_iova_t nic_base, rte_iova_t trgt_base,
19 struct sfc_nic_dma_region *region;
21 if (nic_dma_info->nb_regions >= RTE_DIM(nic_dma_info->regions))
24 region = &nic_dma_info->regions[nic_dma_info->nb_regions];
25 region->nic_base = nic_base;
26 region->trgt_base = trgt_base;
27 region->trgt_end = trgt_base + map_len;
29 nic_dma_info->nb_regions++;
34 * Register mapping for all IOVA mempools at the time of creation to
35 * have mapping for all mbufs.
38 struct sfc_nic_dma_register_mempool_data {
39 struct sfc_adapter *sa;
44 sfc_nic_dma_register_mempool_chunk(struct rte_mempool *mp __rte_unused,
46 struct rte_mempool_memhdr *memhdr,
47 unsigned mem_idx __rte_unused)
49 struct sfc_nic_dma_register_mempool_data *register_data = opaque;
50 struct sfc_adapter *sa = register_data->sa;
51 struct sfc_adapter_shared *sas = sfc_sa2shared(sa);
52 efsys_dma_addr_t nic_base;
53 efsys_dma_addr_t trgt_base;
57 if (memhdr->iova == RTE_BAD_IOVA)
61 * Check if the memory chunk is mapped already. In that case, there's
64 nic_base = sfc_nic_dma_map(&sas->nic_dma_info, memhdr->iova,
66 if (nic_base != RTE_BAD_IOVA)
69 rc = efx_nic_dma_config_add(sa->nic, memhdr->iova, memhdr->len,
70 &nic_base, &trgt_base, &map_len);
73 "cannot handle memory buffer VA=%p IOVA=%" PRIx64 " length=0x%" PRIx64 ": %s",
74 memhdr->addr, (uint64_t)memhdr->iova, memhdr->len,
76 register_data->rc = rc;
81 "registered memory buffer VA=%p IOVA=%" PRIx64 " length=0x%" PRIx64 " -> NIC_BASE=%" PRIx64 " TRGT_BASE=%" PRIx64 " MAP_LEN=%" PRIx64,
82 memhdr->addr, (uint64_t)memhdr->iova, memhdr->len,
83 (uint64_t)nic_base, (uint64_t)trgt_base, (uint64_t)map_len);
85 rc = sfc_nic_dma_add_region(&sas->nic_dma_info, nic_base, trgt_base,
88 sfc_err(sa, "failed to add regioned NIC DMA mapping: %s",
90 register_data->rc = rc;
95 sfc_nic_dma_register_mempool(struct sfc_adapter *sa, struct rte_mempool *mp)
97 struct sfc_nic_dma_register_mempool_data register_data = {
104 SFC_ASSERT(sfc_adapter_is_locked(sa));
106 if (mp->flags & RTE_MEMPOOL_F_NON_IO)
109 iters = rte_mempool_mem_iter(mp, sfc_nic_dma_register_mempool_chunk,
111 if (iters != mp->nb_mem_chunks) {
113 "failed to iterate over memory chunks, some mbufs may be unusable");
116 * Return an error, but try to continue if error is
117 * async and cannot be handled properly.
121 if (register_data.rc != 0) {
123 "failed to map some memory chunks (%s), some mbufs may be unusable",
124 rte_strerror(register_data.rc));
125 result = register_data.rc;
126 /* Try to continue */
130 * There is no point to apply mapping changes triggered by mempool
131 * registration. Configuration will be propagated on start and
132 * mbufs mapping is required in started state only.
134 if (sa->state == SFC_ETHDEV_STARTED) {
136 * It's safe to reconfigure the DMA mapping even if no changes
137 * have been made during memory chunks iteration. In that case,
138 * this operation will not change anything either.
140 rc = efx_nic_dma_reconfigure(sa->nic);
142 sfc_err(sa, "cannot reconfigure NIC DMA: %s",
152 sfc_mempool_event_cb(enum rte_mempool_event event, struct rte_mempool *mp,
155 struct sfc_adapter *sa = user_data;
157 if (event != RTE_MEMPOOL_EVENT_READY)
160 sfc_adapter_lock(sa);
162 (void)sfc_nic_dma_register_mempool(sa, mp);
164 sfc_adapter_unlock(sa);
167 struct sfc_mempool_walk_data {
168 struct sfc_adapter *sa;
173 sfc_mempool_walk_cb(struct rte_mempool *mp, void *arg)
175 struct sfc_mempool_walk_data *walk_data = arg;
178 rc = sfc_nic_dma_register_mempool(walk_data->sa, mp);
184 sfc_nic_dma_attach_regioned(struct sfc_adapter *sa)
186 struct sfc_adapter_shared *sas = sfc_sa2shared(sa);
187 struct sfc_mempool_walk_data walk_data = {
192 rc = rte_mempool_event_callback_register(sfc_mempool_event_cb, sa);
194 sfc_err(sa, "failed to register mempool event callback");
196 goto fail_mempool_event_callback_register;
199 rte_mempool_walk(sfc_mempool_walk_cb, &walk_data);
200 if (walk_data.rc != 0) {
202 goto fail_mempool_walk;
208 rte_mempool_event_callback_unregister(sfc_mempool_event_cb, sa);
209 sas->nic_dma_info.nb_regions = 0;
211 fail_mempool_event_callback_register:
216 sfc_nic_dma_detach_regioned(struct sfc_adapter *sa)
218 struct sfc_adapter_shared *sas = sfc_sa2shared(sa);
220 rte_mempool_event_callback_unregister(sfc_mempool_event_cb, sa);
221 sas->nic_dma_info.nb_regions = 0;
225 sfc_nic_dma_attach(struct sfc_adapter *sa)
227 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
230 sfc_log_init(sa, "dma_mapping_type=%u", encp->enc_dma_mapping);
232 switch (encp->enc_dma_mapping) {
233 case EFX_NIC_DMA_MAPPING_FLAT:
234 /* No mapping required */
237 case EFX_NIC_DMA_MAPPING_REGIONED:
238 rc = sfc_nic_dma_attach_regioned(sa);
245 sfc_log_init(sa, "done: %s", rte_strerror(rc));
250 sfc_nic_dma_detach(struct sfc_adapter *sa)
252 const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
254 sfc_log_init(sa, "dma_mapping_type=%u", encp->enc_dma_mapping);
256 switch (encp->enc_dma_mapping) {
257 case EFX_NIC_DMA_MAPPING_FLAT:
258 /* Nothing to do here */
260 case EFX_NIC_DMA_MAPPING_REGIONED:
261 sfc_nic_dma_detach_regioned(sa);
267 sfc_log_init(sa, "done");
271 sfc_nic_dma_mz_map(struct sfc_adapter *sa, const struct rte_memzone *mz,
272 efx_nic_dma_addr_type_t addr_type,
273 efsys_dma_addr_t *dma_addr)
275 efsys_dma_addr_t nic_base;
276 efsys_dma_addr_t trgt_base;
281 * Check if the memzone can be mapped already without changing the DMA
283 * libefx is used instead of the driver cache since it can take the type
284 * of the buffer into account and make a better decision when it comes
285 * to buffers that are mapped by the FW itself.
287 rc = efx_nic_dma_map(sa->nic, addr_type, mz->iova, mz->len, dma_addr);
293 "failed to map memory buffer VA=%p IOVA=%" PRIx64 " length=0x%" PRIx64 ": %s",
294 mz->addr, (uint64_t)mz->iova, mz->len,
299 rc = efx_nic_dma_config_add(sa->nic, mz->iova, mz->len,
300 &nic_base, &trgt_base, &map_len);
303 "cannot handle memory buffer VA=%p IOVA=%" PRIx64 " length=0x%" PRIx64 ": %s",
304 mz->addr, (uint64_t)mz->iova, mz->len,
309 rc = sfc_nic_dma_add_region(&sfc_sa2shared(sa)->nic_dma_info,
310 nic_base, trgt_base, map_len);
313 "failed to add DMA region VA=%p IOVA=%" PRIx64 " length=0x%" PRIx64 ": %s",
314 mz->addr, (uint64_t)mz->iova, mz->len,
319 rc = efx_nic_dma_reconfigure(sa->nic);
321 sfc_err(sa, "failed to reconfigure DMA");
325 rc = efx_nic_dma_map(sa->nic, addr_type, mz->iova, mz->len, dma_addr);
328 "failed to map memory buffer VA=%p IOVA=%" PRIx64 " length=0x%" PRIx64 ": %s",
329 mz->addr, (uint64_t)mz->iova, mz->len,