1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
41 __out unsigned int *addedp);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
58 __in unsigned int added,
59 __in unsigned int pushed);
61 static __checkReturn efx_rc_t
64 __in unsigned int ns);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr,
88 __out efx_desc_t *edp);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
122 siena_tx_qstats_update, /* etxo_qstats_update */
125 #endif /* EFSYS_OPT_SIENA */
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
149 ef10_tx_qstats_update, /* etxo_qstats_update */
152 #endif /* EFSYS_OPT_HUNTINGTON */
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
176 ef10_tx_qstats_update, /* etxo_qstats_update */
179 #endif /* EFSYS_OPT_MEDFORD */
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
203 ef10_tx_qstats_update, /* etxo_qstats_update */
206 #endif /* EFSYS_OPT_MEDFORD2 */
209 __checkReturn efx_rc_t
213 const efx_tx_ops_t *etxop;
216 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
217 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
219 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
224 if (enp->en_mod_flags & EFX_MOD_TX) {
229 switch (enp->en_family) {
231 case EFX_FAMILY_SIENA:
232 etxop = &__efx_tx_siena_ops;
234 #endif /* EFSYS_OPT_SIENA */
236 #if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON:
238 etxop = &__efx_tx_hunt_ops;
240 #endif /* EFSYS_OPT_HUNTINGTON */
242 #if EFSYS_OPT_MEDFORD
243 case EFX_FAMILY_MEDFORD:
244 etxop = &__efx_tx_medford_ops;
246 #endif /* EFSYS_OPT_MEDFORD */
248 #if EFSYS_OPT_MEDFORD2
249 case EFX_FAMILY_MEDFORD2:
250 etxop = &__efx_tx_medford2_ops;
252 #endif /* EFSYS_OPT_MEDFORD2 */
260 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
262 if ((rc = etxop->etxo_init(enp)) != 0)
265 enp->en_etxop = etxop;
266 enp->en_mod_flags |= EFX_MOD_TX;
276 EFSYS_PROBE1(fail1, efx_rc_t, rc);
278 enp->en_etxop = NULL;
279 enp->en_mod_flags &= ~EFX_MOD_TX;
287 const efx_tx_ops_t *etxop = enp->en_etxop;
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
291 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
292 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
294 etxop->etxo_fini(enp);
296 enp->en_etxop = NULL;
297 enp->en_mod_flags &= ~EFX_MOD_TX;
300 __checkReturn efx_rc_t
303 __in unsigned int index,
304 __in unsigned int label,
305 __in efsys_mem_t *esmp,
310 __deref_out efx_txq_t **etpp,
311 __out unsigned int *addedp)
313 const efx_tx_ops_t *etxop = enp->en_etxop;
315 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
318 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
319 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
321 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
322 enp->en_nic_cfg.enc_txq_limit);
324 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
325 EFSYS_ASSERT(ISP2(encp->enc_txq_min_ndescs));
328 ndescs < encp->enc_txq_min_ndescs ||
329 ndescs > encp->enc_txq_max_ndescs) {
334 /* Allocate an TXQ object */
335 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
342 etp->et_magic = EFX_TXQ_MAGIC;
344 etp->et_index = index;
345 etp->et_mask = ndescs - 1;
348 /* Initial descriptor index may be modified by etxo_qcreate */
351 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
352 ndescs, id, flags, eep, etp, addedp)) != 0)
362 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
366 EFSYS_PROBE1(fail1, efx_rc_t, rc);
374 efx_nic_t *enp = etp->et_enp;
375 const efx_tx_ops_t *etxop = enp->en_etxop;
377 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
379 EFSYS_ASSERT(enp->en_tx_qcount != 0);
382 etxop->etxo_qdestroy(etp);
384 /* Free the TXQ object */
385 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
388 __checkReturn efx_rc_t
391 __in_ecount(ndescs) efx_buffer_t *eb,
392 __in unsigned int ndescs,
393 __in unsigned int completed,
394 __inout unsigned int *addedp)
396 efx_nic_t *enp = etp->et_enp;
397 const efx_tx_ops_t *etxop = enp->en_etxop;
400 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
402 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
408 EFSYS_PROBE1(fail1, efx_rc_t, rc);
415 __in unsigned int added,
416 __in unsigned int pushed)
418 efx_nic_t *enp = etp->et_enp;
419 const efx_tx_ops_t *etxop = enp->en_etxop;
421 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
423 etxop->etxo_qpush(etp, added, pushed);
426 __checkReturn efx_rc_t
429 __in unsigned int ns)
431 efx_nic_t *enp = etp->et_enp;
432 const efx_tx_ops_t *etxop = enp->en_etxop;
435 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
437 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
443 EFSYS_PROBE1(fail1, efx_rc_t, rc);
447 __checkReturn efx_rc_t
451 efx_nic_t *enp = etp->et_enp;
452 const efx_tx_ops_t *etxop = enp->en_etxop;
455 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
457 if ((rc = etxop->etxo_qflush(etp)) != 0)
463 EFSYS_PROBE1(fail1, efx_rc_t, rc);
471 efx_nic_t *enp = etp->et_enp;
472 const efx_tx_ops_t *etxop = enp->en_etxop;
474 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
476 etxop->etxo_qenable(etp);
479 __checkReturn efx_rc_t
483 efx_nic_t *enp = etp->et_enp;
484 const efx_tx_ops_t *etxop = enp->en_etxop;
487 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
489 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
493 if (etxop->etxo_qpio_enable == NULL) {
497 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
507 EFSYS_PROBE1(fail1, efx_rc_t, rc);
515 efx_nic_t *enp = etp->et_enp;
516 const efx_tx_ops_t *etxop = enp->en_etxop;
518 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
520 if (etxop->etxo_qpio_disable != NULL)
521 etxop->etxo_qpio_disable(etp);
524 __checkReturn efx_rc_t
527 __in_ecount(buf_length) uint8_t *buffer,
528 __in size_t buf_length,
529 __in size_t pio_buf_offset)
531 efx_nic_t *enp = etp->et_enp;
532 const efx_tx_ops_t *etxop = enp->en_etxop;
535 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
537 if (etxop->etxo_qpio_write != NULL) {
538 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
539 pio_buf_offset)) != 0)
547 EFSYS_PROBE1(fail1, efx_rc_t, rc);
551 __checkReturn efx_rc_t
554 __in size_t pkt_length,
555 __in unsigned int completed,
556 __inout unsigned int *addedp)
558 efx_nic_t *enp = etp->et_enp;
559 const efx_tx_ops_t *etxop = enp->en_etxop;
562 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
564 if (etxop->etxo_qpio_post != NULL) {
565 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
574 EFSYS_PROBE1(fail1, efx_rc_t, rc);
578 __checkReturn efx_rc_t
581 __in_ecount(ndescs) efx_desc_t *ed,
582 __in unsigned int ndescs,
583 __in unsigned int completed,
584 __inout unsigned int *addedp)
586 efx_nic_t *enp = etp->et_enp;
587 const efx_tx_ops_t *etxop = enp->en_etxop;
589 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
591 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp));
595 efx_tx_qdesc_dma_create(
597 __in efsys_dma_addr_t addr,
600 __out efx_desc_t *edp)
602 efx_nic_t *enp = etp->et_enp;
603 const efx_tx_ops_t *etxop = enp->en_etxop;
605 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
606 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
608 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
612 efx_tx_qdesc_tso_create(
614 __in uint16_t ipv4_id,
615 __in uint32_t tcp_seq,
616 __in uint8_t tcp_flags,
617 __out efx_desc_t *edp)
619 efx_nic_t *enp = etp->et_enp;
620 const efx_tx_ops_t *etxop = enp->en_etxop;
622 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
623 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
625 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
629 efx_tx_qdesc_tso2_create(
631 __in uint16_t ipv4_id,
632 __in uint16_t outer_ipv4_id,
633 __in uint32_t tcp_seq,
635 __out_ecount(count) efx_desc_t *edp,
638 efx_nic_t *enp = etp->et_enp;
639 const efx_tx_ops_t *etxop = enp->en_etxop;
641 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
642 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
644 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
645 tcp_seq, mss, edp, count);
649 efx_tx_qdesc_vlantci_create(
652 __out efx_desc_t *edp)
654 efx_nic_t *enp = etp->et_enp;
655 const efx_tx_ops_t *etxop = enp->en_etxop;
657 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
658 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
660 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
664 efx_tx_qdesc_checksum_create(
667 __out efx_desc_t *edp)
669 efx_nic_t *enp = etp->et_enp;
670 const efx_tx_ops_t *etxop = enp->en_etxop;
672 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
673 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
675 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
681 efx_tx_qstats_update(
683 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
685 efx_nic_t *enp = etp->et_enp;
686 const efx_tx_ops_t *etxop = enp->en_etxop;
688 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
690 etxop->etxo_qstats_update(etp, stat);
697 static __checkReturn efx_rc_t
704 * Disable the timer-based TX DMA backoff and allow TX DMA to be
705 * controlled by the RX FIFO fill level (although always allow a
708 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
709 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
710 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
711 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
712 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
713 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
714 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
715 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
718 * Filter all packets less than 14 bytes to avoid parsing
721 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
722 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
725 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
726 * descriptors (which is bad).
728 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
729 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
730 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
735 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
741 id = (_added)++ & (_etp)->et_mask; \
742 offset = id * sizeof (efx_qword_t); \
744 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
745 unsigned int, id, efsys_dma_addr_t, (_addr), \
746 size_t, (_size), boolean_t, (_eop)); \
748 EFX_POPULATE_QWORD_4(qword, \
749 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
750 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
751 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
752 (uint32_t)((_addr) & 0xffffffff), \
753 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
754 (uint32_t)((_addr) >> 32)); \
755 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
757 _NOTE(CONSTANTCONDITION) \
760 static __checkReturn efx_rc_t
763 __in_ecount(ndescs) efx_buffer_t *eb,
764 __in unsigned int ndescs,
765 __in unsigned int completed,
766 __inout unsigned int *addedp)
768 unsigned int added = *addedp;
771 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
774 for (i = 0; i < ndescs; i++) {
775 efx_buffer_t *ebp = &eb[i];
776 efsys_dma_addr_t start = ebp->eb_addr;
777 size_t size = ebp->eb_size;
778 efsys_dma_addr_t end = start + size;
781 * Fragments must not span 4k boundaries.
782 * Here it is a stricter requirement than the maximum length.
784 EFSYS_ASSERT(P2ROUNDUP(start + 1,
785 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
787 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
790 EFX_TX_QSTAT_INCR(etp, TX_POST);
799 __in unsigned int added,
800 __in unsigned int pushed)
802 efx_nic_t *enp = etp->et_enp;
807 /* Push the populated descriptors out */
808 wptr = added & etp->et_mask;
810 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
812 /* Only write the third DWORD */
813 EFX_POPULATE_DWORD_1(dword,
814 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
816 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
817 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
818 wptr, pushed & etp->et_mask);
819 EFSYS_PIO_WRITE_BARRIER();
820 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
821 etp->et_index, &dword, B_FALSE);
824 #define EFX_MAX_PACE_VALUE 20
826 static __checkReturn efx_rc_t
829 __in unsigned int ns)
831 efx_nic_t *enp = etp->et_enp;
832 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
834 unsigned int pace_val;
835 unsigned int timer_period;
842 * The pace_val to write into the table is s.t
843 * ns <= timer_period * (2 ^ pace_val)
845 timer_period = 104 / encp->enc_clk_mult;
846 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
847 if ((timer_period << pace_val) >= ns)
851 if (pace_val > EFX_MAX_PACE_VALUE) {
856 /* Update the pacing table */
857 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
858 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
864 EFSYS_PROBE1(fail1, efx_rc_t, rc);
869 static __checkReturn efx_rc_t
873 efx_nic_t *enp = etp->et_enp;
877 efx_tx_qpace(etp, 0);
879 label = etp->et_index;
881 /* Flush the queue */
882 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
883 FRF_AZ_TX_FLUSH_DESCQ, label);
884 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
893 efx_nic_t *enp = etp->et_enp;
896 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
897 etp->et_index, &oword, B_TRUE);
899 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
900 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
901 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
902 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
903 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
905 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
906 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
907 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
909 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
910 etp->et_index, &oword, B_TRUE);
913 static __checkReturn efx_rc_t
916 __in unsigned int index,
917 __in unsigned int label,
918 __in efsys_mem_t *esmp,
924 __out unsigned int *addedp)
926 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
932 _NOTE(ARGUNUSED(esmp))
934 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
935 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
936 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
938 if (index >= encp->enc_txq_limit) {
943 (1U << size) <= encp->enc_txq_max_ndescs / encp->enc_txq_min_ndescs;
945 if ((1U << size) == (uint32_t)ndescs / encp->enc_txq_min_ndescs)
947 if (id + (1 << size) >= encp->enc_buftbl_limit) {
952 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
953 if ((flags & inner_csum) != 0) {
958 /* Set up the new descriptor queue */
961 EFX_POPULATE_OWORD_6(oword,
962 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
963 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
964 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
965 FRF_AZ_TX_DESCQ_LABEL, label,
966 FRF_AZ_TX_DESCQ_SIZE, size,
967 FRF_AZ_TX_DESCQ_TYPE, 0);
969 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
970 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
971 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
972 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
973 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
975 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
976 etp->et_index, &oword, B_TRUE);
985 EFSYS_PROBE1(fail1, efx_rc_t, rc);
990 __checkReturn efx_rc_t
993 __in_ecount(ndescs) efx_desc_t *ed,
994 __in unsigned int ndescs,
995 __in unsigned int completed,
996 __inout unsigned int *addedp)
998 unsigned int added = *addedp;
1002 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1007 for (i = 0; i < ndescs; i++) {
1008 efx_desc_t *edp = &ed[i];
1012 id = added++ & etp->et_mask;
1013 offset = id * sizeof (efx_desc_t);
1015 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1018 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1019 unsigned int, added, unsigned int, ndescs);
1021 EFX_TX_QSTAT_INCR(etp, TX_POST);
1027 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1032 siena_tx_qdesc_dma_create(
1033 __in efx_txq_t *etp,
1034 __in efsys_dma_addr_t addr,
1037 __out efx_desc_t *edp)
1040 * Fragments must not span 4k boundaries.
1041 * Here it is a stricter requirement than the maximum length.
1043 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1044 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1046 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1047 efsys_dma_addr_t, addr,
1048 size_t, size, boolean_t, eop);
1050 EFX_POPULATE_QWORD_4(edp->ed_eq,
1051 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1052 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1053 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1054 (uint32_t)(addr & 0xffffffff),
1055 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1056 (uint32_t)(addr >> 32));
1059 #endif /* EFSYS_OPT_SIENA */
1061 #if EFSYS_OPT_QSTATS
1063 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1064 static const char * const __efx_tx_qstat_name[] = {
1068 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1072 __in efx_nic_t *enp,
1073 __in unsigned int id)
1075 _NOTE(ARGUNUSED(enp))
1076 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1077 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1079 return (__efx_tx_qstat_name[id]);
1081 #endif /* EFSYS_OPT_NAMES */
1082 #endif /* EFSYS_OPT_QSTATS */
1086 #if EFSYS_OPT_QSTATS
1088 siena_tx_qstats_update(
1089 __in efx_txq_t *etp,
1090 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1094 for (id = 0; id < TX_NQSTATS; id++) {
1095 efsys_stat_t *essp = &stat[id];
1097 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1098 etp->et_stat[id] = 0;
1101 #endif /* EFSYS_OPT_QSTATS */
1105 __in efx_txq_t *etp)
1107 efx_nic_t *enp = etp->et_enp;
1110 /* Purge descriptor queue */
1111 EFX_ZERO_OWORD(oword);
1113 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1114 etp->et_index, &oword, B_TRUE);
1119 __in efx_nic_t *enp)
1121 _NOTE(ARGUNUSED(enp))
1124 #endif /* EFSYS_OPT_SIENA */