1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
41 __out unsigned int *addedp);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
58 __in unsigned int added,
59 __in unsigned int pushed);
61 static __checkReturn efx_rc_t
64 __in unsigned int ns);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr,
88 __out efx_desc_t *edp);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
122 siena_tx_qstats_update, /* etxo_qstats_update */
125 #endif /* EFSYS_OPT_SIENA */
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
149 ef10_tx_qstats_update, /* etxo_qstats_update */
152 #endif /* EFSYS_OPT_HUNTINGTON */
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
176 ef10_tx_qstats_update, /* etxo_qstats_update */
179 #endif /* EFSYS_OPT_MEDFORD */
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
203 ef10_tx_qstats_update, /* etxo_qstats_update */
206 #endif /* EFSYS_OPT_MEDFORD2 */
209 __checkReturn efx_rc_t
213 const efx_tx_ops_t *etxop;
216 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
217 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
219 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
224 if (enp->en_mod_flags & EFX_MOD_TX) {
229 switch (enp->en_family) {
231 case EFX_FAMILY_SIENA:
232 etxop = &__efx_tx_siena_ops;
234 #endif /* EFSYS_OPT_SIENA */
236 #if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON:
238 etxop = &__efx_tx_hunt_ops;
240 #endif /* EFSYS_OPT_HUNTINGTON */
242 #if EFSYS_OPT_MEDFORD
243 case EFX_FAMILY_MEDFORD:
244 etxop = &__efx_tx_medford_ops;
246 #endif /* EFSYS_OPT_MEDFORD */
248 #if EFSYS_OPT_MEDFORD2
249 case EFX_FAMILY_MEDFORD2:
250 etxop = &__efx_tx_medford2_ops;
252 #endif /* EFSYS_OPT_MEDFORD2 */
260 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
262 if ((rc = etxop->etxo_init(enp)) != 0)
265 enp->en_etxop = etxop;
266 enp->en_mod_flags |= EFX_MOD_TX;
276 EFSYS_PROBE1(fail1, efx_rc_t, rc);
278 enp->en_etxop = NULL;
279 enp->en_mod_flags &= ~EFX_MOD_TX;
287 const efx_tx_ops_t *etxop = enp->en_etxop;
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
291 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
292 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
294 etxop->etxo_fini(enp);
296 enp->en_etxop = NULL;
297 enp->en_mod_flags &= ~EFX_MOD_TX;
300 __checkReturn efx_rc_t
303 __in unsigned int index,
304 __in unsigned int label,
305 __in efsys_mem_t *esmp,
310 __deref_out efx_txq_t **etpp,
311 __out unsigned int *addedp)
313 const efx_tx_ops_t *etxop = enp->en_etxop;
317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
320 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
321 enp->en_nic_cfg.enc_txq_limit);
323 /* Allocate an TXQ object */
324 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
331 etp->et_magic = EFX_TXQ_MAGIC;
333 etp->et_index = index;
334 etp->et_mask = ndescs - 1;
337 /* Initial descriptor index may be modified by etxo_qcreate */
340 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
341 ndescs, id, flags, eep, etp, addedp)) != 0)
351 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
353 EFSYS_PROBE1(fail1, efx_rc_t, rc);
361 efx_nic_t *enp = etp->et_enp;
362 const efx_tx_ops_t *etxop = enp->en_etxop;
364 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
366 EFSYS_ASSERT(enp->en_tx_qcount != 0);
369 etxop->etxo_qdestroy(etp);
371 /* Free the TXQ object */
372 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
375 __checkReturn efx_rc_t
378 __in_ecount(ndescs) efx_buffer_t *eb,
379 __in unsigned int ndescs,
380 __in unsigned int completed,
381 __inout unsigned int *addedp)
383 efx_nic_t *enp = etp->et_enp;
384 const efx_tx_ops_t *etxop = enp->en_etxop;
387 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
389 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
395 EFSYS_PROBE1(fail1, efx_rc_t, rc);
402 __in unsigned int added,
403 __in unsigned int pushed)
405 efx_nic_t *enp = etp->et_enp;
406 const efx_tx_ops_t *etxop = enp->en_etxop;
408 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
410 etxop->etxo_qpush(etp, added, pushed);
413 __checkReturn efx_rc_t
416 __in unsigned int ns)
418 efx_nic_t *enp = etp->et_enp;
419 const efx_tx_ops_t *etxop = enp->en_etxop;
422 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
424 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
430 EFSYS_PROBE1(fail1, efx_rc_t, rc);
434 __checkReturn efx_rc_t
438 efx_nic_t *enp = etp->et_enp;
439 const efx_tx_ops_t *etxop = enp->en_etxop;
442 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
444 if ((rc = etxop->etxo_qflush(etp)) != 0)
450 EFSYS_PROBE1(fail1, efx_rc_t, rc);
458 efx_nic_t *enp = etp->et_enp;
459 const efx_tx_ops_t *etxop = enp->en_etxop;
461 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
463 etxop->etxo_qenable(etp);
466 __checkReturn efx_rc_t
470 efx_nic_t *enp = etp->et_enp;
471 const efx_tx_ops_t *etxop = enp->en_etxop;
474 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
476 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
480 if (etxop->etxo_qpio_enable == NULL) {
484 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
494 EFSYS_PROBE1(fail1, efx_rc_t, rc);
502 efx_nic_t *enp = etp->et_enp;
503 const efx_tx_ops_t *etxop = enp->en_etxop;
505 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
507 if (etxop->etxo_qpio_disable != NULL)
508 etxop->etxo_qpio_disable(etp);
511 __checkReturn efx_rc_t
514 __in_ecount(buf_length) uint8_t *buffer,
515 __in size_t buf_length,
516 __in size_t pio_buf_offset)
518 efx_nic_t *enp = etp->et_enp;
519 const efx_tx_ops_t *etxop = enp->en_etxop;
522 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
524 if (etxop->etxo_qpio_write != NULL) {
525 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
526 pio_buf_offset)) != 0)
534 EFSYS_PROBE1(fail1, efx_rc_t, rc);
538 __checkReturn efx_rc_t
541 __in size_t pkt_length,
542 __in unsigned int completed,
543 __inout unsigned int *addedp)
545 efx_nic_t *enp = etp->et_enp;
546 const efx_tx_ops_t *etxop = enp->en_etxop;
549 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
551 if (etxop->etxo_qpio_post != NULL) {
552 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
561 EFSYS_PROBE1(fail1, efx_rc_t, rc);
565 __checkReturn efx_rc_t
568 __in_ecount(ndescs) efx_desc_t *ed,
569 __in unsigned int ndescs,
570 __in unsigned int completed,
571 __inout unsigned int *addedp)
573 efx_nic_t *enp = etp->et_enp;
574 const efx_tx_ops_t *etxop = enp->en_etxop;
576 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
578 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp));
582 efx_tx_qdesc_dma_create(
584 __in efsys_dma_addr_t addr,
587 __out efx_desc_t *edp)
589 efx_nic_t *enp = etp->et_enp;
590 const efx_tx_ops_t *etxop = enp->en_etxop;
592 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
593 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
595 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
599 efx_tx_qdesc_tso_create(
601 __in uint16_t ipv4_id,
602 __in uint32_t tcp_seq,
603 __in uint8_t tcp_flags,
604 __out efx_desc_t *edp)
606 efx_nic_t *enp = etp->et_enp;
607 const efx_tx_ops_t *etxop = enp->en_etxop;
609 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
610 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
612 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
616 efx_tx_qdesc_tso2_create(
618 __in uint16_t ipv4_id,
619 __in uint16_t outer_ipv4_id,
620 __in uint32_t tcp_seq,
622 __out_ecount(count) efx_desc_t *edp,
625 efx_nic_t *enp = etp->et_enp;
626 const efx_tx_ops_t *etxop = enp->en_etxop;
628 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
629 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
631 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
632 tcp_seq, mss, edp, count);
636 efx_tx_qdesc_vlantci_create(
639 __out efx_desc_t *edp)
641 efx_nic_t *enp = etp->et_enp;
642 const efx_tx_ops_t *etxop = enp->en_etxop;
644 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
645 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
647 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
651 efx_tx_qdesc_checksum_create(
654 __out efx_desc_t *edp)
656 efx_nic_t *enp = etp->et_enp;
657 const efx_tx_ops_t *etxop = enp->en_etxop;
659 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
660 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
662 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
668 efx_tx_qstats_update(
670 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
672 efx_nic_t *enp = etp->et_enp;
673 const efx_tx_ops_t *etxop = enp->en_etxop;
675 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
677 etxop->etxo_qstats_update(etp, stat);
684 static __checkReturn efx_rc_t
691 * Disable the timer-based TX DMA backoff and allow TX DMA to be
692 * controlled by the RX FIFO fill level (although always allow a
695 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
696 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
697 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
698 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
699 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
700 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
701 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
702 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
705 * Filter all packets less than 14 bytes to avoid parsing
708 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
709 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
712 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
713 * descriptors (which is bad).
715 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
716 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
717 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
722 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
728 id = (_added)++ & (_etp)->et_mask; \
729 offset = id * sizeof (efx_qword_t); \
731 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
732 unsigned int, id, efsys_dma_addr_t, (_addr), \
733 size_t, (_size), boolean_t, (_eop)); \
735 EFX_POPULATE_QWORD_4(qword, \
736 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
737 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
738 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
739 (uint32_t)((_addr) & 0xffffffff), \
740 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
741 (uint32_t)((_addr) >> 32)); \
742 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
744 _NOTE(CONSTANTCONDITION) \
747 static __checkReturn efx_rc_t
750 __in_ecount(ndescs) efx_buffer_t *eb,
751 __in unsigned int ndescs,
752 __in unsigned int completed,
753 __inout unsigned int *addedp)
755 unsigned int added = *addedp;
758 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
761 for (i = 0; i < ndescs; i++) {
762 efx_buffer_t *ebp = &eb[i];
763 efsys_dma_addr_t start = ebp->eb_addr;
764 size_t size = ebp->eb_size;
765 efsys_dma_addr_t end = start + size;
768 * Fragments must not span 4k boundaries.
769 * Here it is a stricter requirement than the maximum length.
771 EFSYS_ASSERT(P2ROUNDUP(start + 1,
772 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
774 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
777 EFX_TX_QSTAT_INCR(etp, TX_POST);
786 __in unsigned int added,
787 __in unsigned int pushed)
789 efx_nic_t *enp = etp->et_enp;
794 /* Push the populated descriptors out */
795 wptr = added & etp->et_mask;
797 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
799 /* Only write the third DWORD */
800 EFX_POPULATE_DWORD_1(dword,
801 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
803 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
804 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
805 wptr, pushed & etp->et_mask);
806 EFSYS_PIO_WRITE_BARRIER();
807 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
808 etp->et_index, &dword, B_FALSE);
811 #define EFX_MAX_PACE_VALUE 20
813 static __checkReturn efx_rc_t
816 __in unsigned int ns)
818 efx_nic_t *enp = etp->et_enp;
819 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
821 unsigned int pace_val;
822 unsigned int timer_period;
829 * The pace_val to write into the table is s.t
830 * ns <= timer_period * (2 ^ pace_val)
832 timer_period = 104 / encp->enc_clk_mult;
833 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
834 if ((timer_period << pace_val) >= ns)
838 if (pace_val > EFX_MAX_PACE_VALUE) {
843 /* Update the pacing table */
844 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
845 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
851 EFSYS_PROBE1(fail1, efx_rc_t, rc);
856 static __checkReturn efx_rc_t
860 efx_nic_t *enp = etp->et_enp;
864 efx_tx_qpace(etp, 0);
866 label = etp->et_index;
868 /* Flush the queue */
869 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
870 FRF_AZ_TX_FLUSH_DESCQ, label);
871 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
880 efx_nic_t *enp = etp->et_enp;
883 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
884 etp->et_index, &oword, B_TRUE);
886 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
887 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
888 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
889 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
890 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
892 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
893 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
894 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
896 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
897 etp->et_index, &oword, B_TRUE);
900 static __checkReturn efx_rc_t
903 __in unsigned int index,
904 __in unsigned int label,
905 __in efsys_mem_t *esmp,
911 __out unsigned int *addedp)
913 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
919 _NOTE(ARGUNUSED(esmp))
921 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
922 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
923 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
925 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
926 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
929 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) {
933 if (index >= encp->enc_txq_limit) {
938 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
940 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS))
942 if (id + (1 << size) >= encp->enc_buftbl_limit) {
947 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
948 if ((flags & inner_csum) != 0) {
953 /* Set up the new descriptor queue */
956 EFX_POPULATE_OWORD_6(oword,
957 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
958 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
959 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
960 FRF_AZ_TX_DESCQ_LABEL, label,
961 FRF_AZ_TX_DESCQ_SIZE, size,
962 FRF_AZ_TX_DESCQ_TYPE, 0);
964 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
965 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
966 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
967 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
968 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
970 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
971 etp->et_index, &oword, B_TRUE);
982 EFSYS_PROBE1(fail1, efx_rc_t, rc);
987 __checkReturn efx_rc_t
990 __in_ecount(ndescs) efx_desc_t *ed,
991 __in unsigned int ndescs,
992 __in unsigned int completed,
993 __inout unsigned int *addedp)
995 unsigned int added = *addedp;
999 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1004 for (i = 0; i < ndescs; i++) {
1005 efx_desc_t *edp = &ed[i];
1009 id = added++ & etp->et_mask;
1010 offset = id * sizeof (efx_desc_t);
1012 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1015 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1016 unsigned int, added, unsigned int, ndescs);
1018 EFX_TX_QSTAT_INCR(etp, TX_POST);
1024 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1029 siena_tx_qdesc_dma_create(
1030 __in efx_txq_t *etp,
1031 __in efsys_dma_addr_t addr,
1034 __out efx_desc_t *edp)
1037 * Fragments must not span 4k boundaries.
1038 * Here it is a stricter requirement than the maximum length.
1040 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1041 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1043 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1044 efsys_dma_addr_t, addr,
1045 size_t, size, boolean_t, eop);
1047 EFX_POPULATE_QWORD_4(edp->ed_eq,
1048 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1049 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1050 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1051 (uint32_t)(addr & 0xffffffff),
1052 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1053 (uint32_t)(addr >> 32));
1056 #endif /* EFSYS_OPT_SIENA */
1058 #if EFSYS_OPT_QSTATS
1060 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1061 static const char * const __efx_tx_qstat_name[] = {
1065 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1069 __in efx_nic_t *enp,
1070 __in unsigned int id)
1072 _NOTE(ARGUNUSED(enp))
1073 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1074 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1076 return (__efx_tx_qstat_name[id]);
1078 #endif /* EFSYS_OPT_NAMES */
1079 #endif /* EFSYS_OPT_QSTATS */
1083 #if EFSYS_OPT_QSTATS
1085 siena_tx_qstats_update(
1086 __in efx_txq_t *etp,
1087 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1091 for (id = 0; id < TX_NQSTATS; id++) {
1092 efsys_stat_t *essp = &stat[id];
1094 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1095 etp->et_stat[id] = 0;
1098 #endif /* EFSYS_OPT_QSTATS */
1102 __in efx_txq_t *etp)
1104 efx_nic_t *enp = etp->et_enp;
1107 /* Purge descriptor queue */
1108 EFX_ZERO_OWORD(oword);
1110 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1111 etp->et_index, &oword, B_TRUE);
1116 __in efx_nic_t *enp)
1118 _NOTE(ARGUNUSED(enp))
1121 #endif /* EFSYS_OPT_SIENA */