2 * Copyright (c) 2007-2016 Solarflare Communications Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
35 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
37 (_etp)->et_stat[_stat]++; \
38 _NOTE(CONSTANTCONDITION) \
41 #define EFX_TX_QSTAT_INCR(_etp, _stat)
46 static __checkReturn efx_rc_t
54 static __checkReturn efx_rc_t
57 __in unsigned int index,
58 __in unsigned int label,
59 __in efsys_mem_t *esmp,
65 __out unsigned int *addedp);
71 static __checkReturn efx_rc_t
74 __in_ecount(ndescs) efx_buffer_t *eb,
75 __in unsigned int ndescs,
76 __in unsigned int completed,
77 __inout unsigned int *addedp);
82 __in unsigned int added,
83 __in unsigned int pushed);
85 static __checkReturn efx_rc_t
88 __in unsigned int ns);
90 static __checkReturn efx_rc_t
98 __checkReturn efx_rc_t
101 __in_ecount(ndescs) efx_desc_t *ed,
102 __in unsigned int ndescs,
103 __in unsigned int completed,
104 __inout unsigned int *addedp);
107 siena_tx_qdesc_dma_create(
109 __in efsys_dma_addr_t addr,
112 __out efx_desc_t *edp);
116 siena_tx_qstats_update(
118 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
121 #endif /* EFSYS_OPT_SIENA */
125 static const efx_tx_ops_t __efx_tx_siena_ops = {
126 siena_tx_init, /* etxo_init */
127 siena_tx_fini, /* etxo_fini */
128 siena_tx_qcreate, /* etxo_qcreate */
129 siena_tx_qdestroy, /* etxo_qdestroy */
130 siena_tx_qpost, /* etxo_qpost */
131 siena_tx_qpush, /* etxo_qpush */
132 siena_tx_qpace, /* etxo_qpace */
133 siena_tx_qflush, /* etxo_qflush */
134 siena_tx_qenable, /* etxo_qenable */
135 NULL, /* etxo_qpio_enable */
136 NULL, /* etxo_qpio_disable */
137 NULL, /* etxo_qpio_write */
138 NULL, /* etxo_qpio_post */
139 siena_tx_qdesc_post, /* etxo_qdesc_post */
140 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
141 NULL, /* etxo_qdesc_tso_create */
142 NULL, /* etxo_qdesc_tso2_create */
143 NULL, /* etxo_qdesc_vlantci_create */
145 siena_tx_qstats_update, /* etxo_qstats_update */
148 #endif /* EFSYS_OPT_SIENA */
150 #if EFSYS_OPT_HUNTINGTON
151 static const efx_tx_ops_t __efx_tx_hunt_ops = {
152 ef10_tx_init, /* etxo_init */
153 ef10_tx_fini, /* etxo_fini */
154 ef10_tx_qcreate, /* etxo_qcreate */
155 ef10_tx_qdestroy, /* etxo_qdestroy */
156 ef10_tx_qpost, /* etxo_qpost */
157 ef10_tx_qpush, /* etxo_qpush */
158 ef10_tx_qpace, /* etxo_qpace */
159 ef10_tx_qflush, /* etxo_qflush */
160 ef10_tx_qenable, /* etxo_qenable */
161 ef10_tx_qpio_enable, /* etxo_qpio_enable */
162 ef10_tx_qpio_disable, /* etxo_qpio_disable */
163 ef10_tx_qpio_write, /* etxo_qpio_write */
164 ef10_tx_qpio_post, /* etxo_qpio_post */
165 ef10_tx_qdesc_post, /* etxo_qdesc_post */
166 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
167 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
168 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
169 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
171 ef10_tx_qstats_update, /* etxo_qstats_update */
174 #endif /* EFSYS_OPT_HUNTINGTON */
176 #if EFSYS_OPT_MEDFORD
177 static const efx_tx_ops_t __efx_tx_medford_ops = {
178 ef10_tx_init, /* etxo_init */
179 ef10_tx_fini, /* etxo_fini */
180 ef10_tx_qcreate, /* etxo_qcreate */
181 ef10_tx_qdestroy, /* etxo_qdestroy */
182 ef10_tx_qpost, /* etxo_qpost */
183 ef10_tx_qpush, /* etxo_qpush */
184 ef10_tx_qpace, /* etxo_qpace */
185 ef10_tx_qflush, /* etxo_qflush */
186 ef10_tx_qenable, /* etxo_qenable */
187 ef10_tx_qpio_enable, /* etxo_qpio_enable */
188 ef10_tx_qpio_disable, /* etxo_qpio_disable */
189 ef10_tx_qpio_write, /* etxo_qpio_write */
190 ef10_tx_qpio_post, /* etxo_qpio_post */
191 ef10_tx_qdesc_post, /* etxo_qdesc_post */
192 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
193 NULL, /* etxo_qdesc_tso_create */
194 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
195 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
197 ef10_tx_qstats_update, /* etxo_qstats_update */
200 #endif /* EFSYS_OPT_MEDFORD */
202 __checkReturn efx_rc_t
206 const efx_tx_ops_t *etxop;
209 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
210 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
212 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
217 if (enp->en_mod_flags & EFX_MOD_TX) {
222 switch (enp->en_family) {
224 case EFX_FAMILY_SIENA:
225 etxop = &__efx_tx_siena_ops;
227 #endif /* EFSYS_OPT_SIENA */
229 #if EFSYS_OPT_HUNTINGTON
230 case EFX_FAMILY_HUNTINGTON:
231 etxop = &__efx_tx_hunt_ops;
233 #endif /* EFSYS_OPT_HUNTINGTON */
235 #if EFSYS_OPT_MEDFORD
236 case EFX_FAMILY_MEDFORD:
237 etxop = &__efx_tx_medford_ops;
239 #endif /* EFSYS_OPT_MEDFORD */
247 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
249 if ((rc = etxop->etxo_init(enp)) != 0)
252 enp->en_etxop = etxop;
253 enp->en_mod_flags |= EFX_MOD_TX;
263 EFSYS_PROBE1(fail1, efx_rc_t, rc);
265 enp->en_etxop = NULL;
266 enp->en_mod_flags &= ~EFX_MOD_TX;
274 const efx_tx_ops_t *etxop = enp->en_etxop;
276 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
277 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
278 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
279 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
281 etxop->etxo_fini(enp);
283 enp->en_etxop = NULL;
284 enp->en_mod_flags &= ~EFX_MOD_TX;
287 __checkReturn efx_rc_t
290 __in unsigned int index,
291 __in unsigned int label,
292 __in efsys_mem_t *esmp,
297 __deref_out efx_txq_t **etpp,
298 __out unsigned int *addedp)
300 const efx_tx_ops_t *etxop = enp->en_etxop;
304 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
305 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
307 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
308 enp->en_nic_cfg.enc_txq_limit);
310 /* Allocate an TXQ object */
311 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
318 etp->et_magic = EFX_TXQ_MAGIC;
320 etp->et_index = index;
321 etp->et_mask = ndescs - 1;
324 /* Initial descriptor index may be modified by etxo_qcreate */
327 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
328 ndescs, id, flags, eep, etp, addedp)) != 0)
338 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
340 EFSYS_PROBE1(fail1, efx_rc_t, rc);
348 efx_nic_t *enp = etp->et_enp;
349 const efx_tx_ops_t *etxop = enp->en_etxop;
351 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
353 EFSYS_ASSERT(enp->en_tx_qcount != 0);
356 etxop->etxo_qdestroy(etp);
358 /* Free the TXQ object */
359 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
362 __checkReturn efx_rc_t
365 __in_ecount(ndescs) efx_buffer_t *eb,
366 __in unsigned int ndescs,
367 __in unsigned int completed,
368 __inout unsigned int *addedp)
370 efx_nic_t *enp = etp->et_enp;
371 const efx_tx_ops_t *etxop = enp->en_etxop;
374 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
376 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
382 EFSYS_PROBE1(fail1, efx_rc_t, rc);
389 __in unsigned int added,
390 __in unsigned int pushed)
392 efx_nic_t *enp = etp->et_enp;
393 const efx_tx_ops_t *etxop = enp->en_etxop;
395 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
397 etxop->etxo_qpush(etp, added, pushed);
400 __checkReturn efx_rc_t
403 __in unsigned int ns)
405 efx_nic_t *enp = etp->et_enp;
406 const efx_tx_ops_t *etxop = enp->en_etxop;
409 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
411 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
417 EFSYS_PROBE1(fail1, efx_rc_t, rc);
421 __checkReturn efx_rc_t
425 efx_nic_t *enp = etp->et_enp;
426 const efx_tx_ops_t *etxop = enp->en_etxop;
429 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
431 if ((rc = etxop->etxo_qflush(etp)) != 0)
437 EFSYS_PROBE1(fail1, efx_rc_t, rc);
445 efx_nic_t *enp = etp->et_enp;
446 const efx_tx_ops_t *etxop = enp->en_etxop;
448 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
450 etxop->etxo_qenable(etp);
453 __checkReturn efx_rc_t
457 efx_nic_t *enp = etp->et_enp;
458 const efx_tx_ops_t *etxop = enp->en_etxop;
461 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
463 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
467 if (etxop->etxo_qpio_enable == NULL) {
471 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
481 EFSYS_PROBE1(fail1, efx_rc_t, rc);
489 efx_nic_t *enp = etp->et_enp;
490 const efx_tx_ops_t *etxop = enp->en_etxop;
492 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
494 if (etxop->etxo_qpio_disable != NULL)
495 etxop->etxo_qpio_disable(etp);
498 __checkReturn efx_rc_t
501 __in_ecount(buf_length) uint8_t *buffer,
502 __in size_t buf_length,
503 __in size_t pio_buf_offset)
505 efx_nic_t *enp = etp->et_enp;
506 const efx_tx_ops_t *etxop = enp->en_etxop;
509 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
511 if (etxop->etxo_qpio_write != NULL) {
512 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
513 pio_buf_offset)) != 0)
521 EFSYS_PROBE1(fail1, efx_rc_t, rc);
525 __checkReturn efx_rc_t
528 __in size_t pkt_length,
529 __in unsigned int completed,
530 __inout unsigned int *addedp)
532 efx_nic_t *enp = etp->et_enp;
533 const efx_tx_ops_t *etxop = enp->en_etxop;
536 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
538 if (etxop->etxo_qpio_post != NULL) {
539 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
548 EFSYS_PROBE1(fail1, efx_rc_t, rc);
552 __checkReturn efx_rc_t
555 __in_ecount(ndescs) efx_desc_t *ed,
556 __in unsigned int ndescs,
557 __in unsigned int completed,
558 __inout unsigned int *addedp)
560 efx_nic_t *enp = etp->et_enp;
561 const efx_tx_ops_t *etxop = enp->en_etxop;
564 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
566 if ((rc = etxop->etxo_qdesc_post(etp, ed,
567 ndescs, completed, addedp)) != 0)
573 EFSYS_PROBE1(fail1, efx_rc_t, rc);
578 efx_tx_qdesc_dma_create(
580 __in efsys_dma_addr_t addr,
583 __out efx_desc_t *edp)
585 efx_nic_t *enp = etp->et_enp;
586 const efx_tx_ops_t *etxop = enp->en_etxop;
588 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
589 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
591 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
595 efx_tx_qdesc_tso_create(
597 __in uint16_t ipv4_id,
598 __in uint32_t tcp_seq,
599 __in uint8_t tcp_flags,
600 __out efx_desc_t *edp)
602 efx_nic_t *enp = etp->et_enp;
603 const efx_tx_ops_t *etxop = enp->en_etxop;
605 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
606 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
608 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
612 efx_tx_qdesc_tso2_create(
614 __in uint16_t ipv4_id,
615 __in uint32_t tcp_seq,
617 __out_ecount(count) efx_desc_t *edp,
620 efx_nic_t *enp = etp->et_enp;
621 const efx_tx_ops_t *etxop = enp->en_etxop;
623 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
624 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
626 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
630 efx_tx_qdesc_vlantci_create(
633 __out efx_desc_t *edp)
635 efx_nic_t *enp = etp->et_enp;
636 const efx_tx_ops_t *etxop = enp->en_etxop;
638 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
639 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
641 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
647 efx_tx_qstats_update(
649 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
651 efx_nic_t *enp = etp->et_enp;
652 const efx_tx_ops_t *etxop = enp->en_etxop;
654 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
656 etxop->etxo_qstats_update(etp, stat);
663 static __checkReturn efx_rc_t
670 * Disable the timer-based TX DMA backoff and allow TX DMA to be
671 * controlled by the RX FIFO fill level (although always allow a
674 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
675 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
676 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
677 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
678 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
679 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
680 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
684 * Filter all packets less than 14 bytes to avoid parsing
687 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
688 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
691 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
692 * descriptors (which is bad).
694 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
695 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
696 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
701 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
707 id = (_added)++ & (_etp)->et_mask; \
708 offset = id * sizeof (efx_qword_t); \
710 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
711 unsigned int, id, efsys_dma_addr_t, (_addr), \
712 size_t, (_size), boolean_t, (_eop)); \
714 EFX_POPULATE_QWORD_4(qword, \
715 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
716 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
717 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
718 (uint32_t)((_addr) & 0xffffffff), \
719 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
720 (uint32_t)((_addr) >> 32)); \
721 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
723 _NOTE(CONSTANTCONDITION) \
726 static __checkReturn efx_rc_t
729 __in_ecount(ndescs) efx_buffer_t *eb,
730 __in unsigned int ndescs,
731 __in unsigned int completed,
732 __inout unsigned int *addedp)
734 unsigned int added = *addedp;
738 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
741 for (i = 0; i < ndescs; i++) {
742 efx_buffer_t *ebp = &eb[i];
743 efsys_dma_addr_t start = ebp->eb_addr;
744 size_t size = ebp->eb_size;
745 efsys_dma_addr_t end = start + size;
748 * Fragments must not span 4k boundaries.
749 * Here it is a stricter requirement than the maximum length.
751 EFSYS_ASSERT(P2ROUNDUP(start + 1,
752 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
754 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
757 EFX_TX_QSTAT_INCR(etp, TX_POST);
763 EFSYS_PROBE1(fail1, efx_rc_t, rc);
771 __in unsigned int added,
772 __in unsigned int pushed)
774 efx_nic_t *enp = etp->et_enp;
779 /* Push the populated descriptors out */
780 wptr = added & etp->et_mask;
782 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
784 /* Only write the third DWORD */
785 EFX_POPULATE_DWORD_1(dword,
786 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
788 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
789 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
790 wptr, pushed & etp->et_mask);
791 EFSYS_PIO_WRITE_BARRIER();
792 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
793 etp->et_index, &dword, B_FALSE);
796 #define EFX_MAX_PACE_VALUE 20
798 static __checkReturn efx_rc_t
801 __in unsigned int ns)
803 efx_nic_t *enp = etp->et_enp;
804 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
806 unsigned int pace_val;
807 unsigned int timer_period;
814 * The pace_val to write into the table is s.t
815 * ns <= timer_period * (2 ^ pace_val)
817 timer_period = 104 / encp->enc_clk_mult;
818 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
819 if ((timer_period << pace_val) >= ns)
823 if (pace_val > EFX_MAX_PACE_VALUE) {
828 /* Update the pacing table */
829 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
830 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
836 EFSYS_PROBE1(fail1, efx_rc_t, rc);
841 static __checkReturn efx_rc_t
845 efx_nic_t *enp = etp->et_enp;
849 efx_tx_qpace(etp, 0);
851 label = etp->et_index;
853 /* Flush the queue */
854 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
855 FRF_AZ_TX_FLUSH_DESCQ, label);
856 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
865 efx_nic_t *enp = etp->et_enp;
868 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
869 etp->et_index, &oword, B_TRUE);
871 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
872 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
873 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
874 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
875 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
877 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
878 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
879 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
881 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
882 etp->et_index, &oword, B_TRUE);
885 static __checkReturn efx_rc_t
888 __in unsigned int index,
889 __in unsigned int label,
890 __in efsys_mem_t *esmp,
896 __out unsigned int *addedp)
898 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
904 _NOTE(ARGUNUSED(esmp))
906 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
907 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
908 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
910 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
911 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
914 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) {
918 if (index >= encp->enc_txq_limit) {
923 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
925 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS))
927 if (id + (1 << size) >= encp->enc_buftbl_limit) {
932 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
933 if ((flags & inner_csum) != 0) {
938 /* Set up the new descriptor queue */
941 EFX_POPULATE_OWORD_6(oword,
942 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
943 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
944 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
945 FRF_AZ_TX_DESCQ_LABEL, label,
946 FRF_AZ_TX_DESCQ_SIZE, size,
947 FRF_AZ_TX_DESCQ_TYPE, 0);
949 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
950 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
951 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
952 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
953 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
955 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
956 etp->et_index, &oword, B_TRUE);
967 EFSYS_PROBE1(fail1, efx_rc_t, rc);
972 __checkReturn efx_rc_t
975 __in_ecount(ndescs) efx_desc_t *ed,
976 __in unsigned int ndescs,
977 __in unsigned int completed,
978 __inout unsigned int *addedp)
980 unsigned int added = *addedp;
984 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
989 for (i = 0; i < ndescs; i++) {
990 efx_desc_t *edp = &ed[i];
994 id = added++ & etp->et_mask;
995 offset = id * sizeof (efx_desc_t);
997 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1000 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1001 unsigned int, added, unsigned int, ndescs);
1003 EFX_TX_QSTAT_INCR(etp, TX_POST);
1009 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1014 siena_tx_qdesc_dma_create(
1015 __in efx_txq_t *etp,
1016 __in efsys_dma_addr_t addr,
1019 __out efx_desc_t *edp)
1022 * Fragments must not span 4k boundaries.
1023 * Here it is a stricter requirement than the maximum length.
1025 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1026 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1028 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1029 efsys_dma_addr_t, addr,
1030 size_t, size, boolean_t, eop);
1032 EFX_POPULATE_QWORD_4(edp->ed_eq,
1033 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1034 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1035 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1036 (uint32_t)(addr & 0xffffffff),
1037 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1038 (uint32_t)(addr >> 32));
1041 #endif /* EFSYS_OPT_SIENA */
1043 #if EFSYS_OPT_QSTATS
1045 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1046 static const char * const __efx_tx_qstat_name[] = {
1050 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1054 __in efx_nic_t *enp,
1055 __in unsigned int id)
1057 _NOTE(ARGUNUSED(enp))
1058 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1059 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1061 return (__efx_tx_qstat_name[id]);
1063 #endif /* EFSYS_OPT_NAMES */
1064 #endif /* EFSYS_OPT_QSTATS */
1068 #if EFSYS_OPT_QSTATS
1070 siena_tx_qstats_update(
1071 __in efx_txq_t *etp,
1072 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1076 for (id = 0; id < TX_NQSTATS; id++) {
1077 efsys_stat_t *essp = &stat[id];
1079 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1080 etp->et_stat[id] = 0;
1083 #endif /* EFSYS_OPT_QSTATS */
1087 __in efx_txq_t *etp)
1089 efx_nic_t *enp = etp->et_enp;
1092 /* Purge descriptor queue */
1093 EFX_ZERO_OWORD(oword);
1095 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1096 etp->et_index, &oword, B_TRUE);
1101 __in efx_nic_t *enp)
1103 _NOTE(ARGUNUSED(enp))
1106 #endif /* EFSYS_OPT_SIENA */