2 * Copyright (c) 2007-2016 Solarflare Communications Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
34 #define EFX_TX_QSTAT_INCR(_etp, _stat)
38 static __checkReturn efx_rc_t
46 static __checkReturn efx_rc_t
49 __in unsigned int index,
50 __in unsigned int label,
51 __in efsys_mem_t *esmp,
57 __out unsigned int *addedp);
63 static __checkReturn efx_rc_t
66 __in_ecount(n) efx_buffer_t *eb,
68 __in unsigned int completed,
69 __inout unsigned int *addedp);
74 __in unsigned int added,
75 __in unsigned int pushed);
77 static __checkReturn efx_rc_t
80 __in unsigned int ns);
82 static __checkReturn efx_rc_t
90 __checkReturn efx_rc_t
93 __in_ecount(n) efx_desc_t *ed,
95 __in unsigned int completed,
96 __inout unsigned int *addedp);
99 siena_tx_qdesc_dma_create(
101 __in efsys_dma_addr_t addr,
104 __out efx_desc_t *edp);
106 #endif /* EFSYS_OPT_SIENA */
110 static const efx_tx_ops_t __efx_tx_siena_ops = {
111 siena_tx_init, /* etxo_init */
112 siena_tx_fini, /* etxo_fini */
113 siena_tx_qcreate, /* etxo_qcreate */
114 siena_tx_qdestroy, /* etxo_qdestroy */
115 siena_tx_qpost, /* etxo_qpost */
116 siena_tx_qpush, /* etxo_qpush */
117 siena_tx_qpace, /* etxo_qpace */
118 siena_tx_qflush, /* etxo_qflush */
119 siena_tx_qenable, /* etxo_qenable */
120 NULL, /* etxo_qpio_enable */
121 NULL, /* etxo_qpio_disable */
122 NULL, /* etxo_qpio_write */
123 NULL, /* etxo_qpio_post */
124 siena_tx_qdesc_post, /* etxo_qdesc_post */
125 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
126 NULL, /* etxo_qdesc_tso_create */
127 NULL, /* etxo_qdesc_tso2_create */
128 NULL, /* etxo_qdesc_vlantci_create */
130 #endif /* EFSYS_OPT_SIENA */
132 #if EFSYS_OPT_HUNTINGTON
133 static const efx_tx_ops_t __efx_tx_hunt_ops = {
134 ef10_tx_init, /* etxo_init */
135 ef10_tx_fini, /* etxo_fini */
136 ef10_tx_qcreate, /* etxo_qcreate */
137 ef10_tx_qdestroy, /* etxo_qdestroy */
138 ef10_tx_qpost, /* etxo_qpost */
139 ef10_tx_qpush, /* etxo_qpush */
140 ef10_tx_qpace, /* etxo_qpace */
141 ef10_tx_qflush, /* etxo_qflush */
142 ef10_tx_qenable, /* etxo_qenable */
143 ef10_tx_qpio_enable, /* etxo_qpio_enable */
144 ef10_tx_qpio_disable, /* etxo_qpio_disable */
145 ef10_tx_qpio_write, /* etxo_qpio_write */
146 ef10_tx_qpio_post, /* etxo_qpio_post */
147 ef10_tx_qdesc_post, /* etxo_qdesc_post */
148 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
149 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
150 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
151 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
153 #endif /* EFSYS_OPT_HUNTINGTON */
155 #if EFSYS_OPT_MEDFORD
156 static const efx_tx_ops_t __efx_tx_medford_ops = {
157 ef10_tx_init, /* etxo_init */
158 ef10_tx_fini, /* etxo_fini */
159 ef10_tx_qcreate, /* etxo_qcreate */
160 ef10_tx_qdestroy, /* etxo_qdestroy */
161 ef10_tx_qpost, /* etxo_qpost */
162 ef10_tx_qpush, /* etxo_qpush */
163 ef10_tx_qpace, /* etxo_qpace */
164 ef10_tx_qflush, /* etxo_qflush */
165 ef10_tx_qenable, /* etxo_qenable */
166 ef10_tx_qpio_enable, /* etxo_qpio_enable */
167 ef10_tx_qpio_disable, /* etxo_qpio_disable */
168 ef10_tx_qpio_write, /* etxo_qpio_write */
169 ef10_tx_qpio_post, /* etxo_qpio_post */
170 ef10_tx_qdesc_post, /* etxo_qdesc_post */
171 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
172 NULL, /* etxo_qdesc_tso_create */
173 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
174 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
176 #endif /* EFSYS_OPT_MEDFORD */
178 __checkReturn efx_rc_t
182 const efx_tx_ops_t *etxop;
185 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
186 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
188 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
193 if (enp->en_mod_flags & EFX_MOD_TX) {
198 switch (enp->en_family) {
200 case EFX_FAMILY_SIENA:
201 etxop = &__efx_tx_siena_ops;
203 #endif /* EFSYS_OPT_SIENA */
205 #if EFSYS_OPT_HUNTINGTON
206 case EFX_FAMILY_HUNTINGTON:
207 etxop = &__efx_tx_hunt_ops;
209 #endif /* EFSYS_OPT_HUNTINGTON */
211 #if EFSYS_OPT_MEDFORD
212 case EFX_FAMILY_MEDFORD:
213 etxop = &__efx_tx_medford_ops;
215 #endif /* EFSYS_OPT_MEDFORD */
223 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
225 if ((rc = etxop->etxo_init(enp)) != 0)
228 enp->en_etxop = etxop;
229 enp->en_mod_flags |= EFX_MOD_TX;
239 EFSYS_PROBE1(fail1, efx_rc_t, rc);
241 enp->en_etxop = NULL;
242 enp->en_mod_flags &= ~EFX_MOD_TX;
250 const efx_tx_ops_t *etxop = enp->en_etxop;
252 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
253 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
254 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
255 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
257 etxop->etxo_fini(enp);
259 enp->en_etxop = NULL;
260 enp->en_mod_flags &= ~EFX_MOD_TX;
263 __checkReturn efx_rc_t
266 __in unsigned int index,
267 __in unsigned int label,
268 __in efsys_mem_t *esmp,
273 __deref_out efx_txq_t **etpp,
274 __out unsigned int *addedp)
276 const efx_tx_ops_t *etxop = enp->en_etxop;
277 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
281 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
282 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
284 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
286 /* Allocate an TXQ object */
287 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
294 etp->et_magic = EFX_TXQ_MAGIC;
296 etp->et_index = index;
297 etp->et_mask = n - 1;
300 /* Initial descriptor index may be modified by etxo_qcreate */
303 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
304 n, id, flags, eep, etp, addedp)) != 0)
314 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
316 EFSYS_PROBE1(fail1, efx_rc_t, rc);
324 efx_nic_t *enp = etp->et_enp;
325 const efx_tx_ops_t *etxop = enp->en_etxop;
327 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
329 EFSYS_ASSERT(enp->en_tx_qcount != 0);
332 etxop->etxo_qdestroy(etp);
334 /* Free the TXQ object */
335 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
338 __checkReturn efx_rc_t
341 __in_ecount(n) efx_buffer_t *eb,
343 __in unsigned int completed,
344 __inout unsigned int *addedp)
346 efx_nic_t *enp = etp->et_enp;
347 const efx_tx_ops_t *etxop = enp->en_etxop;
350 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
352 if ((rc = etxop->etxo_qpost(etp, eb,
353 n, completed, addedp)) != 0)
359 EFSYS_PROBE1(fail1, efx_rc_t, rc);
366 __in unsigned int added,
367 __in unsigned int pushed)
369 efx_nic_t *enp = etp->et_enp;
370 const efx_tx_ops_t *etxop = enp->en_etxop;
372 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
374 etxop->etxo_qpush(etp, added, pushed);
377 __checkReturn efx_rc_t
380 __in unsigned int ns)
382 efx_nic_t *enp = etp->et_enp;
383 const efx_tx_ops_t *etxop = enp->en_etxop;
386 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
388 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
394 EFSYS_PROBE1(fail1, efx_rc_t, rc);
398 __checkReturn efx_rc_t
402 efx_nic_t *enp = etp->et_enp;
403 const efx_tx_ops_t *etxop = enp->en_etxop;
406 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
408 if ((rc = etxop->etxo_qflush(etp)) != 0)
414 EFSYS_PROBE1(fail1, efx_rc_t, rc);
422 efx_nic_t *enp = etp->et_enp;
423 const efx_tx_ops_t *etxop = enp->en_etxop;
425 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
427 etxop->etxo_qenable(etp);
430 __checkReturn efx_rc_t
434 efx_nic_t *enp = etp->et_enp;
435 const efx_tx_ops_t *etxop = enp->en_etxop;
438 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
440 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
444 if (etxop->etxo_qpio_enable == NULL) {
448 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
458 EFSYS_PROBE1(fail1, efx_rc_t, rc);
466 efx_nic_t *enp = etp->et_enp;
467 const efx_tx_ops_t *etxop = enp->en_etxop;
469 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
471 if (etxop->etxo_qpio_disable != NULL)
472 etxop->etxo_qpio_disable(etp);
475 __checkReturn efx_rc_t
478 __in_ecount(buf_length) uint8_t *buffer,
479 __in size_t buf_length,
480 __in size_t pio_buf_offset)
482 efx_nic_t *enp = etp->et_enp;
483 const efx_tx_ops_t *etxop = enp->en_etxop;
486 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
488 if (etxop->etxo_qpio_write != NULL) {
489 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
490 pio_buf_offset)) != 0)
498 EFSYS_PROBE1(fail1, efx_rc_t, rc);
502 __checkReturn efx_rc_t
505 __in size_t pkt_length,
506 __in unsigned int completed,
507 __inout unsigned int *addedp)
509 efx_nic_t *enp = etp->et_enp;
510 const efx_tx_ops_t *etxop = enp->en_etxop;
513 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
515 if (etxop->etxo_qpio_post != NULL) {
516 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
525 EFSYS_PROBE1(fail1, efx_rc_t, rc);
529 __checkReturn efx_rc_t
532 __in_ecount(n) efx_desc_t *ed,
534 __in unsigned int completed,
535 __inout unsigned int *addedp)
537 efx_nic_t *enp = etp->et_enp;
538 const efx_tx_ops_t *etxop = enp->en_etxop;
541 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
543 if ((rc = etxop->etxo_qdesc_post(etp, ed,
544 n, completed, addedp)) != 0)
550 EFSYS_PROBE1(fail1, efx_rc_t, rc);
555 efx_tx_qdesc_dma_create(
557 __in efsys_dma_addr_t addr,
560 __out efx_desc_t *edp)
562 efx_nic_t *enp = etp->et_enp;
563 const efx_tx_ops_t *etxop = enp->en_etxop;
565 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
566 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
568 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
572 efx_tx_qdesc_tso_create(
574 __in uint16_t ipv4_id,
575 __in uint32_t tcp_seq,
576 __in uint8_t tcp_flags,
577 __out efx_desc_t *edp)
579 efx_nic_t *enp = etp->et_enp;
580 const efx_tx_ops_t *etxop = enp->en_etxop;
582 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
583 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
585 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
589 efx_tx_qdesc_tso2_create(
591 __in uint16_t ipv4_id,
592 __in uint32_t tcp_seq,
594 __out_ecount(count) efx_desc_t *edp,
597 efx_nic_t *enp = etp->et_enp;
598 const efx_tx_ops_t *etxop = enp->en_etxop;
600 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
601 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
603 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
607 efx_tx_qdesc_vlantci_create(
610 __out efx_desc_t *edp)
612 efx_nic_t *enp = etp->et_enp;
613 const efx_tx_ops_t *etxop = enp->en_etxop;
615 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
616 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
618 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
624 static __checkReturn efx_rc_t
631 * Disable the timer-based TX DMA backoff and allow TX DMA to be
632 * controlled by the RX FIFO fill level (although always allow a
635 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
636 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
637 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
638 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
639 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
640 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
641 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
642 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
645 * Filter all packets less than 14 bytes to avoid parsing
648 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
649 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
652 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
653 * descriptors (which is bad).
655 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
656 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
657 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
662 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
668 id = (_added)++ & (_etp)->et_mask; \
669 offset = id * sizeof (efx_qword_t); \
671 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
672 unsigned int, id, efsys_dma_addr_t, (_addr), \
673 size_t, (_size), boolean_t, (_eop)); \
675 EFX_POPULATE_QWORD_4(qword, \
676 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
677 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
678 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
679 (uint32_t)((_addr) & 0xffffffff), \
680 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
681 (uint32_t)((_addr) >> 32)); \
682 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
684 _NOTE(CONSTANTCONDITION) \
687 static __checkReturn efx_rc_t
690 __in_ecount(n) efx_buffer_t *eb,
692 __in unsigned int completed,
693 __inout unsigned int *addedp)
695 unsigned int added = *addedp;
699 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
702 for (i = 0; i < n; i++) {
703 efx_buffer_t *ebp = &eb[i];
704 efsys_dma_addr_t start = ebp->eb_addr;
705 size_t size = ebp->eb_size;
706 efsys_dma_addr_t end = start + size;
708 /* Fragments must not span 4k boundaries. */
709 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
711 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
714 EFX_TX_QSTAT_INCR(etp, TX_POST);
720 EFSYS_PROBE1(fail1, efx_rc_t, rc);
728 __in unsigned int added,
729 __in unsigned int pushed)
731 efx_nic_t *enp = etp->et_enp;
736 /* Push the populated descriptors out */
737 wptr = added & etp->et_mask;
739 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
741 /* Only write the third DWORD */
742 EFX_POPULATE_DWORD_1(dword,
743 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
745 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
746 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
747 wptr, pushed & etp->et_mask);
748 EFSYS_PIO_WRITE_BARRIER();
749 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
750 etp->et_index, &dword, B_FALSE);
753 #define EFX_MAX_PACE_VALUE 20
755 static __checkReturn efx_rc_t
758 __in unsigned int ns)
760 efx_nic_t *enp = etp->et_enp;
761 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
763 unsigned int pace_val;
764 unsigned int timer_period;
771 * The pace_val to write into the table is s.t
772 * ns <= timer_period * (2 ^ pace_val)
774 timer_period = 104 / encp->enc_clk_mult;
775 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
776 if ((timer_period << pace_val) >= ns)
780 if (pace_val > EFX_MAX_PACE_VALUE) {
785 /* Update the pacing table */
786 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
787 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
793 EFSYS_PROBE1(fail1, efx_rc_t, rc);
798 static __checkReturn efx_rc_t
802 efx_nic_t *enp = etp->et_enp;
806 efx_tx_qpace(etp, 0);
808 label = etp->et_index;
810 /* Flush the queue */
811 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
812 FRF_AZ_TX_FLUSH_DESCQ, label);
813 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
822 efx_nic_t *enp = etp->et_enp;
825 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
826 etp->et_index, &oword, B_TRUE);
828 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
829 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
830 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
831 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
832 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
834 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
835 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
836 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
838 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
839 etp->et_index, &oword, B_TRUE);
842 static __checkReturn efx_rc_t
845 __in unsigned int index,
846 __in unsigned int label,
847 __in efsys_mem_t *esmp,
853 __out unsigned int *addedp)
855 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
860 _NOTE(ARGUNUSED(esmp))
862 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
863 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
864 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
866 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
867 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
869 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
873 if (index >= encp->enc_txq_limit) {
878 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
880 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
882 if (id + (1 << size) >= encp->enc_buftbl_limit) {
887 /* Set up the new descriptor queue */
890 EFX_POPULATE_OWORD_6(oword,
891 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
892 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
893 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
894 FRF_AZ_TX_DESCQ_LABEL, label,
895 FRF_AZ_TX_DESCQ_SIZE, size,
896 FRF_AZ_TX_DESCQ_TYPE, 0);
898 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
899 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
900 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
901 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
902 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
904 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
905 etp->et_index, &oword, B_TRUE);
914 EFSYS_PROBE1(fail1, efx_rc_t, rc);
919 __checkReturn efx_rc_t
922 __in_ecount(n) efx_desc_t *ed,
924 __in unsigned int completed,
925 __inout unsigned int *addedp)
927 unsigned int added = *addedp;
931 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
936 for (i = 0; i < n; i++) {
937 efx_desc_t *edp = &ed[i];
941 id = added++ & etp->et_mask;
942 offset = id * sizeof (efx_desc_t);
944 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
947 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
948 unsigned int, added, unsigned int, n);
950 EFX_TX_QSTAT_INCR(etp, TX_POST);
956 EFSYS_PROBE1(fail1, efx_rc_t, rc);
961 siena_tx_qdesc_dma_create(
963 __in efsys_dma_addr_t addr,
966 __out efx_desc_t *edp)
968 /* Fragments must not span 4k boundaries. */
969 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
971 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
972 efsys_dma_addr_t, addr,
973 size_t, size, boolean_t, eop);
975 EFX_POPULATE_QWORD_4(edp->ed_eq,
976 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
977 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
978 FSF_AZ_TX_KER_BUF_ADDR_DW0,
979 (uint32_t)(addr & 0xffffffff),
980 FSF_AZ_TX_KER_BUF_ADDR_DW1,
981 (uint32_t)(addr >> 32));
984 #endif /* EFSYS_OPT_SIENA */
992 efx_nic_t *enp = etp->et_enp;
995 /* Purge descriptor queue */
996 EFX_ZERO_OWORD(oword);
998 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
999 etp->et_index, &oword, B_TRUE);
1004 __in efx_nic_t *enp)
1006 _NOTE(ARGUNUSED(enp))
1009 #endif /* EFSYS_OPT_SIENA */