2 * Copyright (c) 2007-2016 Solarflare Communications Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
34 #define EFX_EV_QSTAT_INCR(_eep, _stat)
36 #define EFX_EV_PRESENT(_qword) \
37 (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff && \
38 EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff)
42 __checkReturn efx_rc_t
46 const efx_ev_ops_t *eevop;
49 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
50 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
52 if (enp->en_mod_flags & EFX_MOD_EV) {
57 switch (enp->en_family) {
65 EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
67 if ((rc = eevop->eevo_init(enp)) != 0)
70 enp->en_eevop = eevop;
71 enp->en_mod_flags |= EFX_MOD_EV;
78 EFSYS_PROBE1(fail1, efx_rc_t, rc);
81 enp->en_mod_flags &= ~EFX_MOD_EV;
89 const efx_ev_ops_t *eevop = enp->en_eevop;
91 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
92 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR);
93 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
94 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX));
95 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX));
96 EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0);
98 eevop->eevo_fini(enp);
100 enp->en_eevop = NULL;
101 enp->en_mod_flags &= ~EFX_MOD_EV;
105 __checkReturn efx_rc_t
108 __in unsigned int index,
109 __in efsys_mem_t *esmp,
114 __deref_out efx_evq_t **eepp)
116 const efx_ev_ops_t *eevop = enp->en_eevop;
117 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
121 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
122 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV);
124 EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit);
126 switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) {
127 case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT:
129 case EFX_EVQ_FLAGS_NOTIFY_DISABLED:
140 /* Allocate an EVQ object */
141 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep);
147 eep->ee_magic = EFX_EVQ_MAGIC;
149 eep->ee_index = index;
150 eep->ee_mask = n - 1;
151 eep->ee_flags = flags;
155 * Set outputs before the queue is created because interrupts may be
156 * raised for events immediately after the queue is created, before the
157 * function call below returns. See bug58606.
159 * The eepp pointer passed in by the client must therefore point to data
160 * shared with the client's event processing context.
165 if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, us, flags,
176 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
182 EFSYS_PROBE1(fail1, efx_rc_t, rc);
190 efx_nic_t *enp = eep->ee_enp;
191 const efx_ev_ops_t *eevop = enp->en_eevop;
193 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
195 EFSYS_ASSERT(enp->en_ev_qcount != 0);
198 eevop->eevo_qdestroy(eep);
200 /* Free the EVQ object */
201 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep);
204 __checkReturn efx_rc_t
207 __in unsigned int count)
209 efx_nic_t *enp = eep->ee_enp;
210 const efx_ev_ops_t *eevop = enp->en_eevop;
213 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
215 if (!(enp->en_mod_flags & EFX_MOD_INTR)) {
220 if ((rc = eevop->eevo_qprime(eep, count)) != 0)
228 EFSYS_PROBE1(fail1, efx_rc_t, rc);
232 __checkReturn boolean_t
235 __in unsigned int count)
240 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
242 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
243 EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword);
245 return (EFX_EV_PRESENT(qword));
248 #define EFX_EV_BATCH 8
253 __inout unsigned int *countp,
254 __in const efx_ev_callbacks_t *eecp,
257 efx_qword_t ev[EFX_EV_BATCH];
264 /* Ensure events codes match for EF10 (Huntington/Medford) and Siena */
265 EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN);
266 EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH);
268 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV);
269 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV);
270 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV);
271 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV ==
272 FSE_AZ_EV_CODE_DRV_GEN_EV);
274 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
275 EFSYS_ASSERT(countp != NULL);
276 EFSYS_ASSERT(eecp != NULL);
280 /* Read up until the end of the batch period */
281 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1));
282 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
283 for (total = 0; total < batch; ++total) {
284 EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total]));
286 if (!EFX_EV_PRESENT(ev[total]))
289 EFSYS_PROBE3(event, unsigned int, eep->ee_index,
290 uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1),
291 uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0));
293 offset += sizeof (efx_qword_t);
296 /* Process the batch of events */
297 for (index = 0; index < total; ++index) {
298 boolean_t should_abort;
301 EFX_EV_QSTAT_INCR(eep, EV_ALL);
303 code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE);
305 case FSE_AZ_EV_CODE_RX_EV:
306 should_abort = eep->ee_rx(eep,
307 &(ev[index]), eecp, arg);
309 case FSE_AZ_EV_CODE_TX_EV:
310 should_abort = eep->ee_tx(eep,
311 &(ev[index]), eecp, arg);
313 case FSE_AZ_EV_CODE_DRIVER_EV:
314 should_abort = eep->ee_driver(eep,
315 &(ev[index]), eecp, arg);
317 case FSE_AZ_EV_CODE_DRV_GEN_EV:
318 should_abort = eep->ee_drv_gen(eep,
319 &(ev[index]), eecp, arg);
321 case FSE_AZ_EV_CODE_GLOBAL_EV:
322 if (eep->ee_global) {
323 should_abort = eep->ee_global(eep,
324 &(ev[index]), eecp, arg);
327 /* else fallthrough */
329 EFSYS_PROBE3(bad_event,
330 unsigned int, eep->ee_index,
332 EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
334 EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
336 EFSYS_ASSERT(eecp->eec_exception != NULL);
337 (void) eecp->eec_exception(arg,
338 EFX_EXCEPTION_EV_ERROR, code);
339 should_abort = B_TRUE;
342 /* Ignore subsequent events */
349 * Now that the hardware has most likely moved onto dma'ing
350 * into the next cache line, clear the processed events. Take
351 * care to only clear out events that we've processed
353 EFX_SET_QWORD(ev[0]);
354 offset = (count & eep->ee_mask) * sizeof (efx_qword_t);
355 for (index = 0; index < total; ++index) {
356 EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0]));
357 offset += sizeof (efx_qword_t);
362 } while (total == batch);
372 efx_nic_t *enp = eep->ee_enp;
373 const efx_ev_ops_t *eevop = enp->en_eevop;
375 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
377 EFSYS_ASSERT(eevop != NULL &&
378 eevop->eevo_qpost != NULL);
380 eevop->eevo_qpost(eep, data);
383 __checkReturn efx_rc_t
384 efx_ev_usecs_to_ticks(
386 __in unsigned int us,
387 __out unsigned int *ticksp)
389 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
392 /* Convert microseconds to a timer tick count */
395 else if (us * 1000 < encp->enc_evq_timer_quantum_ns)
396 ticks = 1; /* Never round down to zero */
398 ticks = us * 1000 / encp->enc_evq_timer_quantum_ns;
404 __checkReturn efx_rc_t
407 __in unsigned int us)
409 efx_nic_t *enp = eep->ee_enp;
410 const efx_ev_ops_t *eevop = enp->en_eevop;
413 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
415 if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) ==
416 EFX_EVQ_FLAGS_NOTIFY_DISABLED) {
421 if ((rc = eevop->eevo_qmoderate(eep, us)) != 0)
429 EFSYS_PROBE1(fail1, efx_rc_t, rc);