1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright(c) 2019-2020 Xilinx, Inc.
4 * Copyright(c) 2007-2019 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
41 __out unsigned int *addedp);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
58 __in unsigned int added,
59 __in unsigned int pushed);
61 static __checkReturn efx_rc_t
64 __in unsigned int ns);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr,
88 __out efx_desc_t *edp);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
122 siena_tx_qstats_update, /* etxo_qstats_update */
125 #endif /* EFSYS_OPT_SIENA */
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
149 ef10_tx_qstats_update, /* etxo_qstats_update */
152 #endif /* EFSYS_OPT_HUNTINGTON */
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
176 ef10_tx_qstats_update, /* etxo_qstats_update */
179 #endif /* EFSYS_OPT_MEDFORD */
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
203 ef10_tx_qstats_update, /* etxo_qstats_update */
206 #endif /* EFSYS_OPT_MEDFORD2 */
208 #if EFSYS_OPT_RIVERHEAD
209 static const efx_tx_ops_t __efx_tx_rhead_ops = {
210 rhead_tx_init, /* etxo_init */
211 rhead_tx_fini, /* etxo_fini */
212 rhead_tx_qcreate, /* etxo_qcreate */
213 rhead_tx_qdestroy, /* etxo_qdestroy */
214 rhead_tx_qpost, /* etxo_qpost */
215 rhead_tx_qpush, /* etxo_qpush */
216 rhead_tx_qpace, /* etxo_qpace */
217 rhead_tx_qflush, /* etxo_qflush */
218 rhead_tx_qenable, /* etxo_qenable */
219 NULL, /* etxo_qpio_enable */
220 NULL, /* etxo_qpio_disable */
221 NULL, /* etxo_qpio_write */
222 NULL, /* etxo_qpio_post */
223 rhead_tx_qdesc_post, /* etxo_qdesc_post */
224 NULL, /* etxo_qdesc_dma_create */
225 NULL, /* etxo_qdesc_tso_create */
226 NULL, /* etxo_qdesc_tso2_create */
227 NULL, /* etxo_qdesc_vlantci_create */
228 NULL, /* etxo_qdesc_checksum_create */
230 rhead_tx_qstats_update, /* etxo_qstats_update */
233 #endif /* EFSYS_OPT_RIVERHEAD */
236 __checkReturn efx_rc_t
240 const efx_tx_ops_t *etxop;
243 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
244 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
246 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
251 if (enp->en_mod_flags & EFX_MOD_TX) {
256 switch (enp->en_family) {
258 case EFX_FAMILY_SIENA:
259 etxop = &__efx_tx_siena_ops;
261 #endif /* EFSYS_OPT_SIENA */
263 #if EFSYS_OPT_HUNTINGTON
264 case EFX_FAMILY_HUNTINGTON:
265 etxop = &__efx_tx_hunt_ops;
267 #endif /* EFSYS_OPT_HUNTINGTON */
269 #if EFSYS_OPT_MEDFORD
270 case EFX_FAMILY_MEDFORD:
271 etxop = &__efx_tx_medford_ops;
273 #endif /* EFSYS_OPT_MEDFORD */
275 #if EFSYS_OPT_MEDFORD2
276 case EFX_FAMILY_MEDFORD2:
277 etxop = &__efx_tx_medford2_ops;
279 #endif /* EFSYS_OPT_MEDFORD2 */
281 #if EFSYS_OPT_RIVERHEAD
282 case EFX_FAMILY_RIVERHEAD:
283 etxop = &__efx_tx_rhead_ops;
285 #endif /* EFSYS_OPT_RIVERHEAD */
293 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
295 if ((rc = etxop->etxo_init(enp)) != 0)
298 enp->en_etxop = etxop;
299 enp->en_mod_flags |= EFX_MOD_TX;
309 EFSYS_PROBE1(fail1, efx_rc_t, rc);
311 enp->en_etxop = NULL;
312 enp->en_mod_flags &= ~EFX_MOD_TX;
320 const efx_tx_ops_t *etxop = enp->en_etxop;
322 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
323 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
324 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
325 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
327 etxop->etxo_fini(enp);
329 enp->en_etxop = NULL;
330 enp->en_mod_flags &= ~EFX_MOD_TX;
335 __in const efx_nic_t *enp,
336 __in unsigned int ndescs)
338 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
340 return (ndescs * encp->enc_tx_desc_size);
343 __checkReturn unsigned int
345 __in const efx_nic_t *enp,
346 __in unsigned int ndescs)
348 return (EFX_DIV_ROUND_UP(efx_txq_size(enp, ndescs), EFX_BUF_SIZE));
351 __checkReturn efx_rc_t
354 __in unsigned int index,
355 __in unsigned int label,
356 __in efsys_mem_t *esmp,
361 __deref_out efx_txq_t **etpp,
362 __out unsigned int *addedp)
364 const efx_tx_ops_t *etxop = enp->en_etxop;
366 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
369 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
370 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
372 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
373 enp->en_nic_cfg.enc_txq_limit);
375 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
376 EFSYS_ASSERT(ISP2(encp->enc_txq_min_ndescs));
379 ndescs < encp->enc_txq_min_ndescs ||
380 ndescs > encp->enc_txq_max_ndescs) {
385 /* Allocate an TXQ object */
386 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
393 etp->et_magic = EFX_TXQ_MAGIC;
395 etp->et_index = index;
396 etp->et_mask = ndescs - 1;
399 /* Initial descriptor index may be modified by etxo_qcreate */
402 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
403 ndescs, id, flags, eep, etp, addedp)) != 0)
413 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
417 EFSYS_PROBE1(fail1, efx_rc_t, rc);
425 efx_nic_t *enp = etp->et_enp;
426 const efx_tx_ops_t *etxop = enp->en_etxop;
428 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
430 EFSYS_ASSERT(enp->en_tx_qcount != 0);
433 etxop->etxo_qdestroy(etp);
435 /* Free the TXQ object */
436 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
439 __checkReturn efx_rc_t
442 __in_ecount(ndescs) efx_buffer_t *eb,
443 __in unsigned int ndescs,
444 __in unsigned int completed,
445 __inout unsigned int *addedp)
447 efx_nic_t *enp = etp->et_enp;
448 const efx_tx_ops_t *etxop = enp->en_etxop;
451 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
453 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
459 EFSYS_PROBE1(fail1, efx_rc_t, rc);
466 __in unsigned int added,
467 __in unsigned int pushed)
469 efx_nic_t *enp = etp->et_enp;
470 const efx_tx_ops_t *etxop = enp->en_etxop;
472 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
474 etxop->etxo_qpush(etp, added, pushed);
477 __checkReturn efx_rc_t
480 __in unsigned int ns)
482 efx_nic_t *enp = etp->et_enp;
483 const efx_tx_ops_t *etxop = enp->en_etxop;
486 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
488 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
494 EFSYS_PROBE1(fail1, efx_rc_t, rc);
498 __checkReturn efx_rc_t
502 efx_nic_t *enp = etp->et_enp;
503 const efx_tx_ops_t *etxop = enp->en_etxop;
506 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
508 if ((rc = etxop->etxo_qflush(etp)) != 0)
514 EFSYS_PROBE1(fail1, efx_rc_t, rc);
522 efx_nic_t *enp = etp->et_enp;
523 const efx_tx_ops_t *etxop = enp->en_etxop;
525 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
527 etxop->etxo_qenable(etp);
530 __checkReturn efx_rc_t
534 efx_nic_t *enp = etp->et_enp;
535 const efx_tx_ops_t *etxop = enp->en_etxop;
538 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
540 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
544 if (etxop->etxo_qpio_enable == NULL) {
548 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
558 EFSYS_PROBE1(fail1, efx_rc_t, rc);
566 efx_nic_t *enp = etp->et_enp;
567 const efx_tx_ops_t *etxop = enp->en_etxop;
569 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
571 if (etxop->etxo_qpio_disable != NULL)
572 etxop->etxo_qpio_disable(etp);
575 __checkReturn efx_rc_t
578 __in_ecount(buf_length) uint8_t *buffer,
579 __in size_t buf_length,
580 __in size_t pio_buf_offset)
582 efx_nic_t *enp = etp->et_enp;
583 const efx_tx_ops_t *etxop = enp->en_etxop;
586 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
588 if (etxop->etxo_qpio_write != NULL) {
589 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
590 pio_buf_offset)) != 0)
598 EFSYS_PROBE1(fail1, efx_rc_t, rc);
602 __checkReturn efx_rc_t
605 __in size_t pkt_length,
606 __in unsigned int completed,
607 __inout unsigned int *addedp)
609 efx_nic_t *enp = etp->et_enp;
610 const efx_tx_ops_t *etxop = enp->en_etxop;
613 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
615 if (etxop->etxo_qpio_post != NULL) {
616 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
625 EFSYS_PROBE1(fail1, efx_rc_t, rc);
629 __checkReturn efx_rc_t
632 __in_ecount(ndescs) efx_desc_t *ed,
633 __in unsigned int ndescs,
634 __in unsigned int completed,
635 __inout unsigned int *addedp)
637 efx_nic_t *enp = etp->et_enp;
638 const efx_tx_ops_t *etxop = enp->en_etxop;
640 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
642 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp));
646 efx_tx_qdesc_dma_create(
648 __in efsys_dma_addr_t addr,
651 __out efx_desc_t *edp)
653 efx_nic_t *enp = etp->et_enp;
654 const efx_tx_ops_t *etxop = enp->en_etxop;
656 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
657 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
659 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
663 efx_tx_qdesc_tso_create(
665 __in uint16_t ipv4_id,
666 __in uint32_t tcp_seq,
667 __in uint8_t tcp_flags,
668 __out efx_desc_t *edp)
670 efx_nic_t *enp = etp->et_enp;
671 const efx_tx_ops_t *etxop = enp->en_etxop;
673 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
674 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
676 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
680 efx_tx_qdesc_tso2_create(
682 __in uint16_t ipv4_id,
683 __in uint16_t outer_ipv4_id,
684 __in uint32_t tcp_seq,
686 __out_ecount(count) efx_desc_t *edp,
689 efx_nic_t *enp = etp->et_enp;
690 const efx_tx_ops_t *etxop = enp->en_etxop;
692 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
693 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
695 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
696 tcp_seq, mss, edp, count);
700 efx_tx_qdesc_vlantci_create(
703 __out efx_desc_t *edp)
705 efx_nic_t *enp = etp->et_enp;
706 const efx_tx_ops_t *etxop = enp->en_etxop;
708 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
709 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
711 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
715 efx_tx_qdesc_checksum_create(
718 __out efx_desc_t *edp)
720 efx_nic_t *enp = etp->et_enp;
721 const efx_tx_ops_t *etxop = enp->en_etxop;
723 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
724 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
726 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
732 efx_tx_qstats_update(
734 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
736 efx_nic_t *enp = etp->et_enp;
737 const efx_tx_ops_t *etxop = enp->en_etxop;
739 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
741 etxop->etxo_qstats_update(etp, stat);
748 static __checkReturn efx_rc_t
755 * Disable the timer-based TX DMA backoff and allow TX DMA to be
756 * controlled by the RX FIFO fill level (although always allow a
759 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
760 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
761 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
762 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
763 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
764 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
765 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
766 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
769 * Filter all packets less than 14 bytes to avoid parsing
772 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
773 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
776 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
777 * descriptors (which is bad).
779 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
780 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
781 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
786 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
792 id = (_added)++ & (_etp)->et_mask; \
793 offset = id * sizeof (efx_qword_t); \
795 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
796 unsigned int, id, efsys_dma_addr_t, (_addr), \
797 size_t, (_size), boolean_t, (_eop)); \
799 EFX_POPULATE_QWORD_4(qword, \
800 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
801 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
802 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
803 (uint32_t)((_addr) & 0xffffffff), \
804 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
805 (uint32_t)((_addr) >> 32)); \
806 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
808 _NOTE(CONSTANTCONDITION) \
811 static __checkReturn efx_rc_t
814 __in_ecount(ndescs) efx_buffer_t *eb,
815 __in unsigned int ndescs,
816 __in unsigned int completed,
817 __inout unsigned int *addedp)
819 unsigned int added = *addedp;
822 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
825 for (i = 0; i < ndescs; i++) {
826 efx_buffer_t *ebp = &eb[i];
827 efsys_dma_addr_t start = ebp->eb_addr;
828 size_t size = ebp->eb_size;
829 efsys_dma_addr_t end = start + size;
832 * Fragments must not span 4k boundaries.
833 * Here it is a stricter requirement than the maximum length.
835 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1,
836 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
838 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
841 EFX_TX_QSTAT_INCR(etp, TX_POST);
850 __in unsigned int added,
851 __in unsigned int pushed)
853 efx_nic_t *enp = etp->et_enp;
858 /* Push the populated descriptors out */
859 wptr = added & etp->et_mask;
861 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
863 /* Only write the third DWORD */
864 EFX_POPULATE_DWORD_1(dword,
865 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
867 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
868 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
869 SIENA_TXQ_DESC_SIZE, wptr, pushed & etp->et_mask);
870 EFSYS_PIO_WRITE_BARRIER();
871 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
872 etp->et_index, &dword, B_FALSE);
875 #define EFX_MAX_PACE_VALUE 20
877 static __checkReturn efx_rc_t
880 __in unsigned int ns)
882 efx_nic_t *enp = etp->et_enp;
883 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
885 unsigned int pace_val;
886 unsigned int timer_period;
893 * The pace_val to write into the table is s.t
894 * ns <= timer_period * (2 ^ pace_val)
896 timer_period = 104 / encp->enc_clk_mult;
897 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
898 if ((timer_period << pace_val) >= ns)
902 if (pace_val > EFX_MAX_PACE_VALUE) {
907 /* Update the pacing table */
908 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
909 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
915 EFSYS_PROBE1(fail1, efx_rc_t, rc);
920 static __checkReturn efx_rc_t
924 efx_nic_t *enp = etp->et_enp;
928 efx_tx_qpace(etp, 0);
930 label = etp->et_index;
932 /* Flush the queue */
933 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
934 FRF_AZ_TX_FLUSH_DESCQ, label);
935 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
944 efx_nic_t *enp = etp->et_enp;
947 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
948 etp->et_index, &oword, B_TRUE);
950 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
951 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
952 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
953 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
954 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
956 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
957 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
958 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
960 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
961 etp->et_index, &oword, B_TRUE);
964 static __checkReturn efx_rc_t
967 __in unsigned int index,
968 __in unsigned int label,
969 __in efsys_mem_t *esmp,
975 __out unsigned int *addedp)
977 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
983 _NOTE(ARGUNUSED(esmp))
985 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
986 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
987 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
989 if (index >= encp->enc_txq_limit) {
994 (1U << size) <= encp->enc_txq_max_ndescs / encp->enc_txq_min_ndescs;
996 if ((1U << size) == (uint32_t)ndescs / encp->enc_txq_min_ndescs)
998 if (id + (1 << size) >= encp->enc_buftbl_limit) {
1003 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
1004 if ((flags & inner_csum) != 0) {
1009 /* Set up the new descriptor queue */
1012 EFX_POPULATE_OWORD_6(oword,
1013 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
1014 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
1015 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
1016 FRF_AZ_TX_DESCQ_LABEL, label,
1017 FRF_AZ_TX_DESCQ_SIZE, size,
1018 FRF_AZ_TX_DESCQ_TYPE, 0);
1020 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
1021 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
1022 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
1023 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
1024 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
1026 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1027 etp->et_index, &oword, B_TRUE);
1036 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1041 __checkReturn efx_rc_t
1042 siena_tx_qdesc_post(
1043 __in efx_txq_t *etp,
1044 __in_ecount(ndescs) efx_desc_t *ed,
1045 __in unsigned int ndescs,
1046 __in unsigned int completed,
1047 __inout unsigned int *addedp)
1049 unsigned int added = *addedp;
1053 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1058 for (i = 0; i < ndescs; i++) {
1059 efx_desc_t *edp = &ed[i];
1063 id = added++ & etp->et_mask;
1064 offset = id * sizeof (efx_desc_t);
1066 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1069 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1070 unsigned int, added, unsigned int, ndescs);
1072 EFX_TX_QSTAT_INCR(etp, TX_POST);
1078 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1083 siena_tx_qdesc_dma_create(
1084 __in efx_txq_t *etp,
1085 __in efsys_dma_addr_t addr,
1088 __out efx_desc_t *edp)
1091 * Fragments must not span 4k boundaries.
1092 * Here it is a stricter requirement than the maximum length.
1094 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1,
1095 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1097 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1098 efsys_dma_addr_t, addr,
1099 size_t, size, boolean_t, eop);
1101 EFX_POPULATE_QWORD_4(edp->ed_eq,
1102 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1103 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1104 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1105 (uint32_t)(addr & 0xffffffff),
1106 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1107 (uint32_t)(addr >> 32));
1110 #endif /* EFSYS_OPT_SIENA */
1112 #if EFSYS_OPT_QSTATS
1114 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1115 static const char * const __efx_tx_qstat_name[] = {
1119 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1123 __in efx_nic_t *enp,
1124 __in unsigned int id)
1126 _NOTE(ARGUNUSED(enp))
1127 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1128 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1130 return (__efx_tx_qstat_name[id]);
1132 #endif /* EFSYS_OPT_NAMES */
1133 #endif /* EFSYS_OPT_QSTATS */
1137 #if EFSYS_OPT_QSTATS
1139 siena_tx_qstats_update(
1140 __in efx_txq_t *etp,
1141 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1145 for (id = 0; id < TX_NQSTATS; id++) {
1146 efsys_stat_t *essp = &stat[id];
1148 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1149 etp->et_stat[id] = 0;
1152 #endif /* EFSYS_OPT_QSTATS */
1156 __in efx_txq_t *etp)
1158 efx_nic_t *enp = etp->et_enp;
1161 /* Purge descriptor queue */
1162 EFX_ZERO_OWORD(oword);
1164 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1165 etp->et_index, &oword, B_TRUE);
1170 __in efx_nic_t *enp)
1172 _NOTE(ARGUNUSED(enp))
1175 #endif /* EFSYS_OPT_SIENA */