1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
41 __out unsigned int *addedp);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
58 __in unsigned int added,
59 __in unsigned int pushed);
61 static __checkReturn efx_rc_t
64 __in unsigned int ns);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr,
88 __out efx_desc_t *edp);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
122 siena_tx_qstats_update, /* etxo_qstats_update */
125 #endif /* EFSYS_OPT_SIENA */
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
149 ef10_tx_qstats_update, /* etxo_qstats_update */
152 #endif /* EFSYS_OPT_HUNTINGTON */
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
176 ef10_tx_qstats_update, /* etxo_qstats_update */
179 #endif /* EFSYS_OPT_MEDFORD */
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
203 ef10_tx_qstats_update, /* etxo_qstats_update */
206 #endif /* EFSYS_OPT_MEDFORD2 */
209 __checkReturn efx_rc_t
213 const efx_tx_ops_t *etxop;
216 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
217 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
219 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
224 if (enp->en_mod_flags & EFX_MOD_TX) {
229 switch (enp->en_family) {
231 case EFX_FAMILY_SIENA:
232 etxop = &__efx_tx_siena_ops;
234 #endif /* EFSYS_OPT_SIENA */
236 #if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON:
238 etxop = &__efx_tx_hunt_ops;
240 #endif /* EFSYS_OPT_HUNTINGTON */
242 #if EFSYS_OPT_MEDFORD
243 case EFX_FAMILY_MEDFORD:
244 etxop = &__efx_tx_medford_ops;
246 #endif /* EFSYS_OPT_MEDFORD */
248 #if EFSYS_OPT_MEDFORD2
249 case EFX_FAMILY_MEDFORD2:
250 etxop = &__efx_tx_medford2_ops;
252 #endif /* EFSYS_OPT_MEDFORD2 */
260 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
262 if ((rc = etxop->etxo_init(enp)) != 0)
265 enp->en_etxop = etxop;
266 enp->en_mod_flags |= EFX_MOD_TX;
276 EFSYS_PROBE1(fail1, efx_rc_t, rc);
278 enp->en_etxop = NULL;
279 enp->en_mod_flags &= ~EFX_MOD_TX;
287 const efx_tx_ops_t *etxop = enp->en_etxop;
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
291 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
292 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
294 etxop->etxo_fini(enp);
296 enp->en_etxop = NULL;
297 enp->en_mod_flags &= ~EFX_MOD_TX;
302 __in const efx_nic_t *enp,
303 __in unsigned int ndescs)
305 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
307 return (ndescs * encp->enc_tx_desc_size);
310 __checkReturn unsigned int
312 __in const efx_nic_t *enp,
313 __in unsigned int ndescs)
315 return (EFX_DIV_ROUND_UP(efx_txq_size(enp, ndescs), EFX_BUF_SIZE));
318 __checkReturn efx_rc_t
321 __in unsigned int index,
322 __in unsigned int label,
323 __in efsys_mem_t *esmp,
328 __deref_out efx_txq_t **etpp,
329 __out unsigned int *addedp)
331 const efx_tx_ops_t *etxop = enp->en_etxop;
333 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
336 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
337 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
339 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
340 enp->en_nic_cfg.enc_txq_limit);
342 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
343 EFSYS_ASSERT(ISP2(encp->enc_txq_min_ndescs));
346 ndescs < encp->enc_txq_min_ndescs ||
347 ndescs > encp->enc_txq_max_ndescs) {
352 /* Allocate an TXQ object */
353 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
360 etp->et_magic = EFX_TXQ_MAGIC;
362 etp->et_index = index;
363 etp->et_mask = ndescs - 1;
366 /* Initial descriptor index may be modified by etxo_qcreate */
369 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
370 ndescs, id, flags, eep, etp, addedp)) != 0)
380 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
384 EFSYS_PROBE1(fail1, efx_rc_t, rc);
392 efx_nic_t *enp = etp->et_enp;
393 const efx_tx_ops_t *etxop = enp->en_etxop;
395 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
397 EFSYS_ASSERT(enp->en_tx_qcount != 0);
400 etxop->etxo_qdestroy(etp);
402 /* Free the TXQ object */
403 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
406 __checkReturn efx_rc_t
409 __in_ecount(ndescs) efx_buffer_t *eb,
410 __in unsigned int ndescs,
411 __in unsigned int completed,
412 __inout unsigned int *addedp)
414 efx_nic_t *enp = etp->et_enp;
415 const efx_tx_ops_t *etxop = enp->en_etxop;
418 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
420 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
426 EFSYS_PROBE1(fail1, efx_rc_t, rc);
433 __in unsigned int added,
434 __in unsigned int pushed)
436 efx_nic_t *enp = etp->et_enp;
437 const efx_tx_ops_t *etxop = enp->en_etxop;
439 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
441 etxop->etxo_qpush(etp, added, pushed);
444 __checkReturn efx_rc_t
447 __in unsigned int ns)
449 efx_nic_t *enp = etp->et_enp;
450 const efx_tx_ops_t *etxop = enp->en_etxop;
453 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
455 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
461 EFSYS_PROBE1(fail1, efx_rc_t, rc);
465 __checkReturn efx_rc_t
469 efx_nic_t *enp = etp->et_enp;
470 const efx_tx_ops_t *etxop = enp->en_etxop;
473 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
475 if ((rc = etxop->etxo_qflush(etp)) != 0)
481 EFSYS_PROBE1(fail1, efx_rc_t, rc);
489 efx_nic_t *enp = etp->et_enp;
490 const efx_tx_ops_t *etxop = enp->en_etxop;
492 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
494 etxop->etxo_qenable(etp);
497 __checkReturn efx_rc_t
501 efx_nic_t *enp = etp->et_enp;
502 const efx_tx_ops_t *etxop = enp->en_etxop;
505 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
507 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
511 if (etxop->etxo_qpio_enable == NULL) {
515 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
525 EFSYS_PROBE1(fail1, efx_rc_t, rc);
533 efx_nic_t *enp = etp->et_enp;
534 const efx_tx_ops_t *etxop = enp->en_etxop;
536 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
538 if (etxop->etxo_qpio_disable != NULL)
539 etxop->etxo_qpio_disable(etp);
542 __checkReturn efx_rc_t
545 __in_ecount(buf_length) uint8_t *buffer,
546 __in size_t buf_length,
547 __in size_t pio_buf_offset)
549 efx_nic_t *enp = etp->et_enp;
550 const efx_tx_ops_t *etxop = enp->en_etxop;
553 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
555 if (etxop->etxo_qpio_write != NULL) {
556 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
557 pio_buf_offset)) != 0)
565 EFSYS_PROBE1(fail1, efx_rc_t, rc);
569 __checkReturn efx_rc_t
572 __in size_t pkt_length,
573 __in unsigned int completed,
574 __inout unsigned int *addedp)
576 efx_nic_t *enp = etp->et_enp;
577 const efx_tx_ops_t *etxop = enp->en_etxop;
580 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
582 if (etxop->etxo_qpio_post != NULL) {
583 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
592 EFSYS_PROBE1(fail1, efx_rc_t, rc);
596 __checkReturn efx_rc_t
599 __in_ecount(ndescs) efx_desc_t *ed,
600 __in unsigned int ndescs,
601 __in unsigned int completed,
602 __inout unsigned int *addedp)
604 efx_nic_t *enp = etp->et_enp;
605 const efx_tx_ops_t *etxop = enp->en_etxop;
607 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
609 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp));
613 efx_tx_qdesc_dma_create(
615 __in efsys_dma_addr_t addr,
618 __out efx_desc_t *edp)
620 efx_nic_t *enp = etp->et_enp;
621 const efx_tx_ops_t *etxop = enp->en_etxop;
623 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
624 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
626 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
630 efx_tx_qdesc_tso_create(
632 __in uint16_t ipv4_id,
633 __in uint32_t tcp_seq,
634 __in uint8_t tcp_flags,
635 __out efx_desc_t *edp)
637 efx_nic_t *enp = etp->et_enp;
638 const efx_tx_ops_t *etxop = enp->en_etxop;
640 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
641 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
643 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
647 efx_tx_qdesc_tso2_create(
649 __in uint16_t ipv4_id,
650 __in uint16_t outer_ipv4_id,
651 __in uint32_t tcp_seq,
653 __out_ecount(count) efx_desc_t *edp,
656 efx_nic_t *enp = etp->et_enp;
657 const efx_tx_ops_t *etxop = enp->en_etxop;
659 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
660 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
662 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
663 tcp_seq, mss, edp, count);
667 efx_tx_qdesc_vlantci_create(
670 __out efx_desc_t *edp)
672 efx_nic_t *enp = etp->et_enp;
673 const efx_tx_ops_t *etxop = enp->en_etxop;
675 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
676 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
678 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
682 efx_tx_qdesc_checksum_create(
685 __out efx_desc_t *edp)
687 efx_nic_t *enp = etp->et_enp;
688 const efx_tx_ops_t *etxop = enp->en_etxop;
690 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
691 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
693 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
699 efx_tx_qstats_update(
701 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
703 efx_nic_t *enp = etp->et_enp;
704 const efx_tx_ops_t *etxop = enp->en_etxop;
706 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
708 etxop->etxo_qstats_update(etp, stat);
715 static __checkReturn efx_rc_t
722 * Disable the timer-based TX DMA backoff and allow TX DMA to be
723 * controlled by the RX FIFO fill level (although always allow a
726 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
727 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
728 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
729 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
730 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
731 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
732 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
733 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
736 * Filter all packets less than 14 bytes to avoid parsing
739 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
740 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
743 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
744 * descriptors (which is bad).
746 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
747 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
748 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
753 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
759 id = (_added)++ & (_etp)->et_mask; \
760 offset = id * sizeof (efx_qword_t); \
762 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
763 unsigned int, id, efsys_dma_addr_t, (_addr), \
764 size_t, (_size), boolean_t, (_eop)); \
766 EFX_POPULATE_QWORD_4(qword, \
767 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
768 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
769 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
770 (uint32_t)((_addr) & 0xffffffff), \
771 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
772 (uint32_t)((_addr) >> 32)); \
773 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
775 _NOTE(CONSTANTCONDITION) \
778 static __checkReturn efx_rc_t
781 __in_ecount(ndescs) efx_buffer_t *eb,
782 __in unsigned int ndescs,
783 __in unsigned int completed,
784 __inout unsigned int *addedp)
786 unsigned int added = *addedp;
789 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
792 for (i = 0; i < ndescs; i++) {
793 efx_buffer_t *ebp = &eb[i];
794 efsys_dma_addr_t start = ebp->eb_addr;
795 size_t size = ebp->eb_size;
796 efsys_dma_addr_t end = start + size;
799 * Fragments must not span 4k boundaries.
800 * Here it is a stricter requirement than the maximum length.
802 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1,
803 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
805 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
808 EFX_TX_QSTAT_INCR(etp, TX_POST);
817 __in unsigned int added,
818 __in unsigned int pushed)
820 efx_nic_t *enp = etp->et_enp;
825 /* Push the populated descriptors out */
826 wptr = added & etp->et_mask;
828 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
830 /* Only write the third DWORD */
831 EFX_POPULATE_DWORD_1(dword,
832 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
834 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
835 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
836 wptr, pushed & etp->et_mask);
837 EFSYS_PIO_WRITE_BARRIER();
838 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
839 etp->et_index, &dword, B_FALSE);
842 #define EFX_MAX_PACE_VALUE 20
844 static __checkReturn efx_rc_t
847 __in unsigned int ns)
849 efx_nic_t *enp = etp->et_enp;
850 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
852 unsigned int pace_val;
853 unsigned int timer_period;
860 * The pace_val to write into the table is s.t
861 * ns <= timer_period * (2 ^ pace_val)
863 timer_period = 104 / encp->enc_clk_mult;
864 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
865 if ((timer_period << pace_val) >= ns)
869 if (pace_val > EFX_MAX_PACE_VALUE) {
874 /* Update the pacing table */
875 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
876 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
882 EFSYS_PROBE1(fail1, efx_rc_t, rc);
887 static __checkReturn efx_rc_t
891 efx_nic_t *enp = etp->et_enp;
895 efx_tx_qpace(etp, 0);
897 label = etp->et_index;
899 /* Flush the queue */
900 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
901 FRF_AZ_TX_FLUSH_DESCQ, label);
902 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
911 efx_nic_t *enp = etp->et_enp;
914 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
915 etp->et_index, &oword, B_TRUE);
917 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
918 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
919 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
920 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
921 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
923 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
924 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
925 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
927 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
928 etp->et_index, &oword, B_TRUE);
931 static __checkReturn efx_rc_t
934 __in unsigned int index,
935 __in unsigned int label,
936 __in efsys_mem_t *esmp,
942 __out unsigned int *addedp)
944 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
950 _NOTE(ARGUNUSED(esmp))
952 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
953 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
954 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
956 if (index >= encp->enc_txq_limit) {
961 (1U << size) <= encp->enc_txq_max_ndescs / encp->enc_txq_min_ndescs;
963 if ((1U << size) == (uint32_t)ndescs / encp->enc_txq_min_ndescs)
965 if (id + (1 << size) >= encp->enc_buftbl_limit) {
970 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
971 if ((flags & inner_csum) != 0) {
976 /* Set up the new descriptor queue */
979 EFX_POPULATE_OWORD_6(oword,
980 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
981 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
982 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
983 FRF_AZ_TX_DESCQ_LABEL, label,
984 FRF_AZ_TX_DESCQ_SIZE, size,
985 FRF_AZ_TX_DESCQ_TYPE, 0);
987 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
988 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
989 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
990 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
991 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
993 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
994 etp->et_index, &oword, B_TRUE);
1003 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1008 __checkReturn efx_rc_t
1009 siena_tx_qdesc_post(
1010 __in efx_txq_t *etp,
1011 __in_ecount(ndescs) efx_desc_t *ed,
1012 __in unsigned int ndescs,
1013 __in unsigned int completed,
1014 __inout unsigned int *addedp)
1016 unsigned int added = *addedp;
1020 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1025 for (i = 0; i < ndescs; i++) {
1026 efx_desc_t *edp = &ed[i];
1030 id = added++ & etp->et_mask;
1031 offset = id * sizeof (efx_desc_t);
1033 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1036 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1037 unsigned int, added, unsigned int, ndescs);
1039 EFX_TX_QSTAT_INCR(etp, TX_POST);
1045 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1050 siena_tx_qdesc_dma_create(
1051 __in efx_txq_t *etp,
1052 __in efsys_dma_addr_t addr,
1055 __out efx_desc_t *edp)
1058 * Fragments must not span 4k boundaries.
1059 * Here it is a stricter requirement than the maximum length.
1061 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1,
1062 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1064 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1065 efsys_dma_addr_t, addr,
1066 size_t, size, boolean_t, eop);
1068 EFX_POPULATE_QWORD_4(edp->ed_eq,
1069 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1070 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1071 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1072 (uint32_t)(addr & 0xffffffff),
1073 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1074 (uint32_t)(addr >> 32));
1077 #endif /* EFSYS_OPT_SIENA */
1079 #if EFSYS_OPT_QSTATS
1081 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1082 static const char * const __efx_tx_qstat_name[] = {
1086 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1090 __in efx_nic_t *enp,
1091 __in unsigned int id)
1093 _NOTE(ARGUNUSED(enp))
1094 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1095 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1097 return (__efx_tx_qstat_name[id]);
1099 #endif /* EFSYS_OPT_NAMES */
1100 #endif /* EFSYS_OPT_QSTATS */
1104 #if EFSYS_OPT_QSTATS
1106 siena_tx_qstats_update(
1107 __in efx_txq_t *etp,
1108 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1112 for (id = 0; id < TX_NQSTATS; id++) {
1113 efsys_stat_t *essp = &stat[id];
1115 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1116 etp->et_stat[id] = 0;
1119 #endif /* EFSYS_OPT_QSTATS */
1123 __in efx_txq_t *etp)
1125 efx_nic_t *enp = etp->et_enp;
1128 /* Purge descriptor queue */
1129 EFX_ZERO_OWORD(oword);
1131 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1132 etp->et_index, &oword, B_TRUE);
1137 __in efx_nic_t *enp)
1139 _NOTE(ARGUNUSED(enp))
1142 #endif /* EFSYS_OPT_SIENA */