1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
41 __out unsigned int *addedp);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
58 __in unsigned int added,
59 __in unsigned int pushed);
61 static __checkReturn efx_rc_t
64 __in unsigned int ns);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr,
88 __out efx_desc_t *edp);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
121 siena_tx_qstats_update, /* etxo_qstats_update */
124 #endif /* EFSYS_OPT_SIENA */
126 #if EFSYS_OPT_HUNTINGTON
127 static const efx_tx_ops_t __efx_tx_hunt_ops = {
128 ef10_tx_init, /* etxo_init */
129 ef10_tx_fini, /* etxo_fini */
130 ef10_tx_qcreate, /* etxo_qcreate */
131 ef10_tx_qdestroy, /* etxo_qdestroy */
132 ef10_tx_qpost, /* etxo_qpost */
133 ef10_tx_qpush, /* etxo_qpush */
134 ef10_tx_qpace, /* etxo_qpace */
135 ef10_tx_qflush, /* etxo_qflush */
136 ef10_tx_qenable, /* etxo_qenable */
137 ef10_tx_qpio_enable, /* etxo_qpio_enable */
138 ef10_tx_qpio_disable, /* etxo_qpio_disable */
139 ef10_tx_qpio_write, /* etxo_qpio_write */
140 ef10_tx_qpio_post, /* etxo_qpio_post */
141 ef10_tx_qdesc_post, /* etxo_qdesc_post */
142 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
143 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
144 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
145 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qstats_update, /* etxo_qstats_update */
150 #endif /* EFSYS_OPT_HUNTINGTON */
152 #if EFSYS_OPT_MEDFORD
153 static const efx_tx_ops_t __efx_tx_medford_ops = {
154 ef10_tx_init, /* etxo_init */
155 ef10_tx_fini, /* etxo_fini */
156 ef10_tx_qcreate, /* etxo_qcreate */
157 ef10_tx_qdestroy, /* etxo_qdestroy */
158 ef10_tx_qpost, /* etxo_qpost */
159 ef10_tx_qpush, /* etxo_qpush */
160 ef10_tx_qpace, /* etxo_qpace */
161 ef10_tx_qflush, /* etxo_qflush */
162 ef10_tx_qenable, /* etxo_qenable */
163 ef10_tx_qpio_enable, /* etxo_qpio_enable */
164 ef10_tx_qpio_disable, /* etxo_qpio_disable */
165 ef10_tx_qpio_write, /* etxo_qpio_write */
166 ef10_tx_qpio_post, /* etxo_qpio_post */
167 ef10_tx_qdesc_post, /* etxo_qdesc_post */
168 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
169 NULL, /* etxo_qdesc_tso_create */
170 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
171 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
173 ef10_tx_qstats_update, /* etxo_qstats_update */
176 #endif /* EFSYS_OPT_MEDFORD */
178 #if EFSYS_OPT_MEDFORD2
179 static const efx_tx_ops_t __efx_tx_medford2_ops = {
180 ef10_tx_init, /* etxo_init */
181 ef10_tx_fini, /* etxo_fini */
182 ef10_tx_qcreate, /* etxo_qcreate */
183 ef10_tx_qdestroy, /* etxo_qdestroy */
184 ef10_tx_qpost, /* etxo_qpost */
185 ef10_tx_qpush, /* etxo_qpush */
186 ef10_tx_qpace, /* etxo_qpace */
187 ef10_tx_qflush, /* etxo_qflush */
188 ef10_tx_qenable, /* etxo_qenable */
189 ef10_tx_qpio_enable, /* etxo_qpio_enable */
190 ef10_tx_qpio_disable, /* etxo_qpio_disable */
191 ef10_tx_qpio_write, /* etxo_qpio_write */
192 ef10_tx_qpio_post, /* etxo_qpio_post */
193 ef10_tx_qdesc_post, /* etxo_qdesc_post */
194 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
195 NULL, /* etxo_qdesc_tso_create */
196 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
197 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
199 ef10_tx_qstats_update, /* etxo_qstats_update */
202 #endif /* EFSYS_OPT_MEDFORD2 */
205 __checkReturn efx_rc_t
209 const efx_tx_ops_t *etxop;
212 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
213 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
215 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
220 if (enp->en_mod_flags & EFX_MOD_TX) {
225 switch (enp->en_family) {
227 case EFX_FAMILY_SIENA:
228 etxop = &__efx_tx_siena_ops;
230 #endif /* EFSYS_OPT_SIENA */
232 #if EFSYS_OPT_HUNTINGTON
233 case EFX_FAMILY_HUNTINGTON:
234 etxop = &__efx_tx_hunt_ops;
236 #endif /* EFSYS_OPT_HUNTINGTON */
238 #if EFSYS_OPT_MEDFORD
239 case EFX_FAMILY_MEDFORD:
240 etxop = &__efx_tx_medford_ops;
242 #endif /* EFSYS_OPT_MEDFORD */
244 #if EFSYS_OPT_MEDFORD2
245 case EFX_FAMILY_MEDFORD2:
246 etxop = &__efx_tx_medford2_ops;
248 #endif /* EFSYS_OPT_MEDFORD2 */
256 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
258 if ((rc = etxop->etxo_init(enp)) != 0)
261 enp->en_etxop = etxop;
262 enp->en_mod_flags |= EFX_MOD_TX;
272 EFSYS_PROBE1(fail1, efx_rc_t, rc);
274 enp->en_etxop = NULL;
275 enp->en_mod_flags &= ~EFX_MOD_TX;
283 const efx_tx_ops_t *etxop = enp->en_etxop;
285 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
286 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
287 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
288 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
290 etxop->etxo_fini(enp);
292 enp->en_etxop = NULL;
293 enp->en_mod_flags &= ~EFX_MOD_TX;
296 __checkReturn efx_rc_t
299 __in unsigned int index,
300 __in unsigned int label,
301 __in efsys_mem_t *esmp,
306 __deref_out efx_txq_t **etpp,
307 __out unsigned int *addedp)
309 const efx_tx_ops_t *etxop = enp->en_etxop;
313 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
314 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
316 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
317 enp->en_nic_cfg.enc_txq_limit);
319 /* Allocate an TXQ object */
320 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
327 etp->et_magic = EFX_TXQ_MAGIC;
329 etp->et_index = index;
330 etp->et_mask = ndescs - 1;
333 /* Initial descriptor index may be modified by etxo_qcreate */
336 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
337 ndescs, id, flags, eep, etp, addedp)) != 0)
347 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
349 EFSYS_PROBE1(fail1, efx_rc_t, rc);
357 efx_nic_t *enp = etp->et_enp;
358 const efx_tx_ops_t *etxop = enp->en_etxop;
360 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
362 EFSYS_ASSERT(enp->en_tx_qcount != 0);
365 etxop->etxo_qdestroy(etp);
367 /* Free the TXQ object */
368 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
371 __checkReturn efx_rc_t
374 __in_ecount(ndescs) efx_buffer_t *eb,
375 __in unsigned int ndescs,
376 __in unsigned int completed,
377 __inout unsigned int *addedp)
379 efx_nic_t *enp = etp->et_enp;
380 const efx_tx_ops_t *etxop = enp->en_etxop;
383 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
385 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
391 EFSYS_PROBE1(fail1, efx_rc_t, rc);
398 __in unsigned int added,
399 __in unsigned int pushed)
401 efx_nic_t *enp = etp->et_enp;
402 const efx_tx_ops_t *etxop = enp->en_etxop;
404 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
406 etxop->etxo_qpush(etp, added, pushed);
409 __checkReturn efx_rc_t
412 __in unsigned int ns)
414 efx_nic_t *enp = etp->et_enp;
415 const efx_tx_ops_t *etxop = enp->en_etxop;
418 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
420 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
426 EFSYS_PROBE1(fail1, efx_rc_t, rc);
430 __checkReturn efx_rc_t
434 efx_nic_t *enp = etp->et_enp;
435 const efx_tx_ops_t *etxop = enp->en_etxop;
438 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
440 if ((rc = etxop->etxo_qflush(etp)) != 0)
446 EFSYS_PROBE1(fail1, efx_rc_t, rc);
454 efx_nic_t *enp = etp->et_enp;
455 const efx_tx_ops_t *etxop = enp->en_etxop;
457 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
459 etxop->etxo_qenable(etp);
462 __checkReturn efx_rc_t
466 efx_nic_t *enp = etp->et_enp;
467 const efx_tx_ops_t *etxop = enp->en_etxop;
470 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
472 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
476 if (etxop->etxo_qpio_enable == NULL) {
480 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
490 EFSYS_PROBE1(fail1, efx_rc_t, rc);
498 efx_nic_t *enp = etp->et_enp;
499 const efx_tx_ops_t *etxop = enp->en_etxop;
501 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
503 if (etxop->etxo_qpio_disable != NULL)
504 etxop->etxo_qpio_disable(etp);
507 __checkReturn efx_rc_t
510 __in_ecount(buf_length) uint8_t *buffer,
511 __in size_t buf_length,
512 __in size_t pio_buf_offset)
514 efx_nic_t *enp = etp->et_enp;
515 const efx_tx_ops_t *etxop = enp->en_etxop;
518 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
520 if (etxop->etxo_qpio_write != NULL) {
521 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
522 pio_buf_offset)) != 0)
530 EFSYS_PROBE1(fail1, efx_rc_t, rc);
534 __checkReturn efx_rc_t
537 __in size_t pkt_length,
538 __in unsigned int completed,
539 __inout unsigned int *addedp)
541 efx_nic_t *enp = etp->et_enp;
542 const efx_tx_ops_t *etxop = enp->en_etxop;
545 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
547 if (etxop->etxo_qpio_post != NULL) {
548 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
557 EFSYS_PROBE1(fail1, efx_rc_t, rc);
561 __checkReturn efx_rc_t
564 __in_ecount(ndescs) efx_desc_t *ed,
565 __in unsigned int ndescs,
566 __in unsigned int completed,
567 __inout unsigned int *addedp)
569 efx_nic_t *enp = etp->et_enp;
570 const efx_tx_ops_t *etxop = enp->en_etxop;
573 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
575 if ((rc = etxop->etxo_qdesc_post(etp, ed,
576 ndescs, completed, addedp)) != 0)
582 EFSYS_PROBE1(fail1, efx_rc_t, rc);
587 efx_tx_qdesc_dma_create(
589 __in efsys_dma_addr_t addr,
592 __out efx_desc_t *edp)
594 efx_nic_t *enp = etp->et_enp;
595 const efx_tx_ops_t *etxop = enp->en_etxop;
597 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
598 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
600 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
604 efx_tx_qdesc_tso_create(
606 __in uint16_t ipv4_id,
607 __in uint32_t tcp_seq,
608 __in uint8_t tcp_flags,
609 __out efx_desc_t *edp)
611 efx_nic_t *enp = etp->et_enp;
612 const efx_tx_ops_t *etxop = enp->en_etxop;
614 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
615 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
617 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
621 efx_tx_qdesc_tso2_create(
623 __in uint16_t ipv4_id,
624 __in uint32_t tcp_seq,
626 __out_ecount(count) efx_desc_t *edp,
629 efx_nic_t *enp = etp->et_enp;
630 const efx_tx_ops_t *etxop = enp->en_etxop;
632 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
633 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
635 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
639 efx_tx_qdesc_vlantci_create(
642 __out efx_desc_t *edp)
644 efx_nic_t *enp = etp->et_enp;
645 const efx_tx_ops_t *etxop = enp->en_etxop;
647 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
648 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
650 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
656 efx_tx_qstats_update(
658 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
660 efx_nic_t *enp = etp->et_enp;
661 const efx_tx_ops_t *etxop = enp->en_etxop;
663 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
665 etxop->etxo_qstats_update(etp, stat);
672 static __checkReturn efx_rc_t
679 * Disable the timer-based TX DMA backoff and allow TX DMA to be
680 * controlled by the RX FIFO fill level (although always allow a
683 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
684 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
685 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
686 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
687 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
688 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
689 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
690 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
693 * Filter all packets less than 14 bytes to avoid parsing
696 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
697 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
700 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
701 * descriptors (which is bad).
703 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
704 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
705 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
710 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
716 id = (_added)++ & (_etp)->et_mask; \
717 offset = id * sizeof (efx_qword_t); \
719 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
720 unsigned int, id, efsys_dma_addr_t, (_addr), \
721 size_t, (_size), boolean_t, (_eop)); \
723 EFX_POPULATE_QWORD_4(qword, \
724 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
725 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
726 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
727 (uint32_t)((_addr) & 0xffffffff), \
728 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
729 (uint32_t)((_addr) >> 32)); \
730 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
732 _NOTE(CONSTANTCONDITION) \
735 static __checkReturn efx_rc_t
738 __in_ecount(ndescs) efx_buffer_t *eb,
739 __in unsigned int ndescs,
740 __in unsigned int completed,
741 __inout unsigned int *addedp)
743 unsigned int added = *addedp;
747 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
750 for (i = 0; i < ndescs; i++) {
751 efx_buffer_t *ebp = &eb[i];
752 efsys_dma_addr_t start = ebp->eb_addr;
753 size_t size = ebp->eb_size;
754 efsys_dma_addr_t end = start + size;
757 * Fragments must not span 4k boundaries.
758 * Here it is a stricter requirement than the maximum length.
760 EFSYS_ASSERT(P2ROUNDUP(start + 1,
761 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
763 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
766 EFX_TX_QSTAT_INCR(etp, TX_POST);
772 EFSYS_PROBE1(fail1, efx_rc_t, rc);
780 __in unsigned int added,
781 __in unsigned int pushed)
783 efx_nic_t *enp = etp->et_enp;
788 /* Push the populated descriptors out */
789 wptr = added & etp->et_mask;
791 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
793 /* Only write the third DWORD */
794 EFX_POPULATE_DWORD_1(dword,
795 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
797 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
798 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
799 wptr, pushed & etp->et_mask);
800 EFSYS_PIO_WRITE_BARRIER();
801 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
802 etp->et_index, &dword, B_FALSE);
805 #define EFX_MAX_PACE_VALUE 20
807 static __checkReturn efx_rc_t
810 __in unsigned int ns)
812 efx_nic_t *enp = etp->et_enp;
813 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
815 unsigned int pace_val;
816 unsigned int timer_period;
823 * The pace_val to write into the table is s.t
824 * ns <= timer_period * (2 ^ pace_val)
826 timer_period = 104 / encp->enc_clk_mult;
827 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
828 if ((timer_period << pace_val) >= ns)
832 if (pace_val > EFX_MAX_PACE_VALUE) {
837 /* Update the pacing table */
838 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
839 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
845 EFSYS_PROBE1(fail1, efx_rc_t, rc);
850 static __checkReturn efx_rc_t
854 efx_nic_t *enp = etp->et_enp;
858 efx_tx_qpace(etp, 0);
860 label = etp->et_index;
862 /* Flush the queue */
863 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
864 FRF_AZ_TX_FLUSH_DESCQ, label);
865 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
874 efx_nic_t *enp = etp->et_enp;
877 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
878 etp->et_index, &oword, B_TRUE);
880 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
881 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
882 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
883 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
884 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
886 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
887 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
888 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
890 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
891 etp->et_index, &oword, B_TRUE);
894 static __checkReturn efx_rc_t
897 __in unsigned int index,
898 __in unsigned int label,
899 __in efsys_mem_t *esmp,
905 __out unsigned int *addedp)
907 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
913 _NOTE(ARGUNUSED(esmp))
915 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
916 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
917 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
919 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
920 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
923 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) {
927 if (index >= encp->enc_txq_limit) {
932 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
934 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS))
936 if (id + (1 << size) >= encp->enc_buftbl_limit) {
941 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
942 if ((flags & inner_csum) != 0) {
947 /* Set up the new descriptor queue */
950 EFX_POPULATE_OWORD_6(oword,
951 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
952 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
953 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
954 FRF_AZ_TX_DESCQ_LABEL, label,
955 FRF_AZ_TX_DESCQ_SIZE, size,
956 FRF_AZ_TX_DESCQ_TYPE, 0);
958 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
959 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
960 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
961 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
962 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
964 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
965 etp->et_index, &oword, B_TRUE);
976 EFSYS_PROBE1(fail1, efx_rc_t, rc);
981 __checkReturn efx_rc_t
984 __in_ecount(ndescs) efx_desc_t *ed,
985 __in unsigned int ndescs,
986 __in unsigned int completed,
987 __inout unsigned int *addedp)
989 unsigned int added = *addedp;
993 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
998 for (i = 0; i < ndescs; i++) {
999 efx_desc_t *edp = &ed[i];
1003 id = added++ & etp->et_mask;
1004 offset = id * sizeof (efx_desc_t);
1006 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1009 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1010 unsigned int, added, unsigned int, ndescs);
1012 EFX_TX_QSTAT_INCR(etp, TX_POST);
1018 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1023 siena_tx_qdesc_dma_create(
1024 __in efx_txq_t *etp,
1025 __in efsys_dma_addr_t addr,
1028 __out efx_desc_t *edp)
1031 * Fragments must not span 4k boundaries.
1032 * Here it is a stricter requirement than the maximum length.
1034 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1035 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1037 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1038 efsys_dma_addr_t, addr,
1039 size_t, size, boolean_t, eop);
1041 EFX_POPULATE_QWORD_4(edp->ed_eq,
1042 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1043 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1044 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1045 (uint32_t)(addr & 0xffffffff),
1046 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1047 (uint32_t)(addr >> 32));
1050 #endif /* EFSYS_OPT_SIENA */
1052 #if EFSYS_OPT_QSTATS
1054 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1055 static const char * const __efx_tx_qstat_name[] = {
1059 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1063 __in efx_nic_t *enp,
1064 __in unsigned int id)
1066 _NOTE(ARGUNUSED(enp))
1067 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1068 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1070 return (__efx_tx_qstat_name[id]);
1072 #endif /* EFSYS_OPT_NAMES */
1073 #endif /* EFSYS_OPT_QSTATS */
1077 #if EFSYS_OPT_QSTATS
1079 siena_tx_qstats_update(
1080 __in efx_txq_t *etp,
1081 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1085 for (id = 0; id < TX_NQSTATS; id++) {
1086 efsys_stat_t *essp = &stat[id];
1088 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1089 etp->et_stat[id] = 0;
1092 #endif /* EFSYS_OPT_QSTATS */
1096 __in efx_txq_t *etp)
1098 efx_nic_t *enp = etp->et_enp;
1101 /* Purge descriptor queue */
1102 EFX_ZERO_OWORD(oword);
1104 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1105 etp->et_index, &oword, B_TRUE);
1110 __in efx_nic_t *enp)
1112 _NOTE(ARGUNUSED(enp))
1115 #endif /* EFSYS_OPT_SIENA */