1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
41 __out unsigned int *addedp);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
58 __in unsigned int added,
59 __in unsigned int pushed);
61 static __checkReturn efx_rc_t
64 __in unsigned int ns);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr,
88 __out efx_desc_t *edp);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
122 siena_tx_qstats_update, /* etxo_qstats_update */
125 #endif /* EFSYS_OPT_SIENA */
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
149 ef10_tx_qstats_update, /* etxo_qstats_update */
152 #endif /* EFSYS_OPT_HUNTINGTON */
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
176 ef10_tx_qstats_update, /* etxo_qstats_update */
179 #endif /* EFSYS_OPT_MEDFORD */
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
203 ef10_tx_qstats_update, /* etxo_qstats_update */
206 #endif /* EFSYS_OPT_MEDFORD2 */
209 __checkReturn efx_rc_t
213 const efx_tx_ops_t *etxop;
216 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
217 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
219 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
224 if (enp->en_mod_flags & EFX_MOD_TX) {
229 switch (enp->en_family) {
231 case EFX_FAMILY_SIENA:
232 etxop = &__efx_tx_siena_ops;
234 #endif /* EFSYS_OPT_SIENA */
236 #if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON:
238 etxop = &__efx_tx_hunt_ops;
240 #endif /* EFSYS_OPT_HUNTINGTON */
242 #if EFSYS_OPT_MEDFORD
243 case EFX_FAMILY_MEDFORD:
244 etxop = &__efx_tx_medford_ops;
246 #endif /* EFSYS_OPT_MEDFORD */
248 #if EFSYS_OPT_MEDFORD2
249 case EFX_FAMILY_MEDFORD2:
250 etxop = &__efx_tx_medford2_ops;
252 #endif /* EFSYS_OPT_MEDFORD2 */
260 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
262 if ((rc = etxop->etxo_init(enp)) != 0)
265 enp->en_etxop = etxop;
266 enp->en_mod_flags |= EFX_MOD_TX;
276 EFSYS_PROBE1(fail1, efx_rc_t, rc);
278 enp->en_etxop = NULL;
279 enp->en_mod_flags &= ~EFX_MOD_TX;
287 const efx_tx_ops_t *etxop = enp->en_etxop;
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
291 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
292 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
294 etxop->etxo_fini(enp);
296 enp->en_etxop = NULL;
297 enp->en_mod_flags &= ~EFX_MOD_TX;
300 __checkReturn efx_rc_t
303 __in unsigned int index,
304 __in unsigned int label,
305 __in efsys_mem_t *esmp,
310 __deref_out efx_txq_t **etpp,
311 __out unsigned int *addedp)
313 const efx_tx_ops_t *etxop = enp->en_etxop;
317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
320 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
321 enp->en_nic_cfg.enc_txq_limit);
323 /* Allocate an TXQ object */
324 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
331 etp->et_magic = EFX_TXQ_MAGIC;
333 etp->et_index = index;
334 etp->et_mask = ndescs - 1;
337 /* Initial descriptor index may be modified by etxo_qcreate */
340 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
341 ndescs, id, flags, eep, etp, addedp)) != 0)
351 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
353 EFSYS_PROBE1(fail1, efx_rc_t, rc);
361 efx_nic_t *enp = etp->et_enp;
362 const efx_tx_ops_t *etxop = enp->en_etxop;
364 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
366 EFSYS_ASSERT(enp->en_tx_qcount != 0);
369 etxop->etxo_qdestroy(etp);
371 /* Free the TXQ object */
372 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
375 __checkReturn efx_rc_t
378 __in_ecount(ndescs) efx_buffer_t *eb,
379 __in unsigned int ndescs,
380 __in unsigned int completed,
381 __inout unsigned int *addedp)
383 efx_nic_t *enp = etp->et_enp;
384 const efx_tx_ops_t *etxop = enp->en_etxop;
387 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
389 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
395 EFSYS_PROBE1(fail1, efx_rc_t, rc);
402 __in unsigned int added,
403 __in unsigned int pushed)
405 efx_nic_t *enp = etp->et_enp;
406 const efx_tx_ops_t *etxop = enp->en_etxop;
408 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
410 etxop->etxo_qpush(etp, added, pushed);
413 __checkReturn efx_rc_t
416 __in unsigned int ns)
418 efx_nic_t *enp = etp->et_enp;
419 const efx_tx_ops_t *etxop = enp->en_etxop;
422 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
424 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
430 EFSYS_PROBE1(fail1, efx_rc_t, rc);
434 __checkReturn efx_rc_t
438 efx_nic_t *enp = etp->et_enp;
439 const efx_tx_ops_t *etxop = enp->en_etxop;
442 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
444 if ((rc = etxop->etxo_qflush(etp)) != 0)
450 EFSYS_PROBE1(fail1, efx_rc_t, rc);
458 efx_nic_t *enp = etp->et_enp;
459 const efx_tx_ops_t *etxop = enp->en_etxop;
461 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
463 etxop->etxo_qenable(etp);
466 __checkReturn efx_rc_t
470 efx_nic_t *enp = etp->et_enp;
471 const efx_tx_ops_t *etxop = enp->en_etxop;
474 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
476 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
480 if (etxop->etxo_qpio_enable == NULL) {
484 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
494 EFSYS_PROBE1(fail1, efx_rc_t, rc);
502 efx_nic_t *enp = etp->et_enp;
503 const efx_tx_ops_t *etxop = enp->en_etxop;
505 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
507 if (etxop->etxo_qpio_disable != NULL)
508 etxop->etxo_qpio_disable(etp);
511 __checkReturn efx_rc_t
514 __in_ecount(buf_length) uint8_t *buffer,
515 __in size_t buf_length,
516 __in size_t pio_buf_offset)
518 efx_nic_t *enp = etp->et_enp;
519 const efx_tx_ops_t *etxop = enp->en_etxop;
522 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
524 if (etxop->etxo_qpio_write != NULL) {
525 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
526 pio_buf_offset)) != 0)
534 EFSYS_PROBE1(fail1, efx_rc_t, rc);
538 __checkReturn efx_rc_t
541 __in size_t pkt_length,
542 __in unsigned int completed,
543 __inout unsigned int *addedp)
545 efx_nic_t *enp = etp->et_enp;
546 const efx_tx_ops_t *etxop = enp->en_etxop;
549 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
551 if (etxop->etxo_qpio_post != NULL) {
552 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
561 EFSYS_PROBE1(fail1, efx_rc_t, rc);
565 __checkReturn efx_rc_t
568 __in_ecount(ndescs) efx_desc_t *ed,
569 __in unsigned int ndescs,
570 __in unsigned int completed,
571 __inout unsigned int *addedp)
573 efx_nic_t *enp = etp->et_enp;
574 const efx_tx_ops_t *etxop = enp->en_etxop;
577 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
579 if ((rc = etxop->etxo_qdesc_post(etp, ed,
580 ndescs, completed, addedp)) != 0)
586 EFSYS_PROBE1(fail1, efx_rc_t, rc);
591 efx_tx_qdesc_dma_create(
593 __in efsys_dma_addr_t addr,
596 __out efx_desc_t *edp)
598 efx_nic_t *enp = etp->et_enp;
599 const efx_tx_ops_t *etxop = enp->en_etxop;
601 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
602 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
604 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
608 efx_tx_qdesc_tso_create(
610 __in uint16_t ipv4_id,
611 __in uint32_t tcp_seq,
612 __in uint8_t tcp_flags,
613 __out efx_desc_t *edp)
615 efx_nic_t *enp = etp->et_enp;
616 const efx_tx_ops_t *etxop = enp->en_etxop;
618 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
619 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
621 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
625 efx_tx_qdesc_tso2_create(
627 __in uint16_t ipv4_id,
628 __in uint32_t tcp_seq,
630 __out_ecount(count) efx_desc_t *edp,
633 efx_nic_t *enp = etp->et_enp;
634 const efx_tx_ops_t *etxop = enp->en_etxop;
636 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
637 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
639 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
643 efx_tx_qdesc_vlantci_create(
646 __out efx_desc_t *edp)
648 efx_nic_t *enp = etp->et_enp;
649 const efx_tx_ops_t *etxop = enp->en_etxop;
651 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
652 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
654 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
658 efx_tx_qdesc_checksum_create(
661 __out efx_desc_t *edp)
663 efx_nic_t *enp = etp->et_enp;
664 const efx_tx_ops_t *etxop = enp->en_etxop;
666 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
667 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
669 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
675 efx_tx_qstats_update(
677 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
679 efx_nic_t *enp = etp->et_enp;
680 const efx_tx_ops_t *etxop = enp->en_etxop;
682 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
684 etxop->etxo_qstats_update(etp, stat);
691 static __checkReturn efx_rc_t
698 * Disable the timer-based TX DMA backoff and allow TX DMA to be
699 * controlled by the RX FIFO fill level (although always allow a
702 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
703 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
704 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
705 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
706 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
707 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
708 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
709 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
712 * Filter all packets less than 14 bytes to avoid parsing
715 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
716 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
719 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
720 * descriptors (which is bad).
722 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
723 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
724 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
729 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
735 id = (_added)++ & (_etp)->et_mask; \
736 offset = id * sizeof (efx_qword_t); \
738 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
739 unsigned int, id, efsys_dma_addr_t, (_addr), \
740 size_t, (_size), boolean_t, (_eop)); \
742 EFX_POPULATE_QWORD_4(qword, \
743 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
744 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
745 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
746 (uint32_t)((_addr) & 0xffffffff), \
747 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
748 (uint32_t)((_addr) >> 32)); \
749 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
751 _NOTE(CONSTANTCONDITION) \
754 static __checkReturn efx_rc_t
757 __in_ecount(ndescs) efx_buffer_t *eb,
758 __in unsigned int ndescs,
759 __in unsigned int completed,
760 __inout unsigned int *addedp)
762 unsigned int added = *addedp;
766 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
769 for (i = 0; i < ndescs; i++) {
770 efx_buffer_t *ebp = &eb[i];
771 efsys_dma_addr_t start = ebp->eb_addr;
772 size_t size = ebp->eb_size;
773 efsys_dma_addr_t end = start + size;
776 * Fragments must not span 4k boundaries.
777 * Here it is a stricter requirement than the maximum length.
779 EFSYS_ASSERT(P2ROUNDUP(start + 1,
780 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
782 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
785 EFX_TX_QSTAT_INCR(etp, TX_POST);
791 EFSYS_PROBE1(fail1, efx_rc_t, rc);
799 __in unsigned int added,
800 __in unsigned int pushed)
802 efx_nic_t *enp = etp->et_enp;
807 /* Push the populated descriptors out */
808 wptr = added & etp->et_mask;
810 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
812 /* Only write the third DWORD */
813 EFX_POPULATE_DWORD_1(dword,
814 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
816 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
817 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
818 wptr, pushed & etp->et_mask);
819 EFSYS_PIO_WRITE_BARRIER();
820 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
821 etp->et_index, &dword, B_FALSE);
824 #define EFX_MAX_PACE_VALUE 20
826 static __checkReturn efx_rc_t
829 __in unsigned int ns)
831 efx_nic_t *enp = etp->et_enp;
832 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
834 unsigned int pace_val;
835 unsigned int timer_period;
842 * The pace_val to write into the table is s.t
843 * ns <= timer_period * (2 ^ pace_val)
845 timer_period = 104 / encp->enc_clk_mult;
846 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
847 if ((timer_period << pace_val) >= ns)
851 if (pace_val > EFX_MAX_PACE_VALUE) {
856 /* Update the pacing table */
857 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
858 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
864 EFSYS_PROBE1(fail1, efx_rc_t, rc);
869 static __checkReturn efx_rc_t
873 efx_nic_t *enp = etp->et_enp;
877 efx_tx_qpace(etp, 0);
879 label = etp->et_index;
881 /* Flush the queue */
882 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
883 FRF_AZ_TX_FLUSH_DESCQ, label);
884 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
893 efx_nic_t *enp = etp->et_enp;
896 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
897 etp->et_index, &oword, B_TRUE);
899 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
900 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
901 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
902 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
903 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
905 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
906 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
907 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
909 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
910 etp->et_index, &oword, B_TRUE);
913 static __checkReturn efx_rc_t
916 __in unsigned int index,
917 __in unsigned int label,
918 __in efsys_mem_t *esmp,
924 __out unsigned int *addedp)
926 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
932 _NOTE(ARGUNUSED(esmp))
934 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
935 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
936 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
938 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
939 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
942 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) {
946 if (index >= encp->enc_txq_limit) {
951 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
953 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS))
955 if (id + (1 << size) >= encp->enc_buftbl_limit) {
960 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
961 if ((flags & inner_csum) != 0) {
966 /* Set up the new descriptor queue */
969 EFX_POPULATE_OWORD_6(oword,
970 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
971 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
972 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
973 FRF_AZ_TX_DESCQ_LABEL, label,
974 FRF_AZ_TX_DESCQ_SIZE, size,
975 FRF_AZ_TX_DESCQ_TYPE, 0);
977 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
978 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
979 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
980 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
981 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
983 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
984 etp->et_index, &oword, B_TRUE);
995 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1000 __checkReturn efx_rc_t
1001 siena_tx_qdesc_post(
1002 __in efx_txq_t *etp,
1003 __in_ecount(ndescs) efx_desc_t *ed,
1004 __in unsigned int ndescs,
1005 __in unsigned int completed,
1006 __inout unsigned int *addedp)
1008 unsigned int added = *addedp;
1012 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1017 for (i = 0; i < ndescs; i++) {
1018 efx_desc_t *edp = &ed[i];
1022 id = added++ & etp->et_mask;
1023 offset = id * sizeof (efx_desc_t);
1025 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1028 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1029 unsigned int, added, unsigned int, ndescs);
1031 EFX_TX_QSTAT_INCR(etp, TX_POST);
1037 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1042 siena_tx_qdesc_dma_create(
1043 __in efx_txq_t *etp,
1044 __in efsys_dma_addr_t addr,
1047 __out efx_desc_t *edp)
1050 * Fragments must not span 4k boundaries.
1051 * Here it is a stricter requirement than the maximum length.
1053 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1054 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1056 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1057 efsys_dma_addr_t, addr,
1058 size_t, size, boolean_t, eop);
1060 EFX_POPULATE_QWORD_4(edp->ed_eq,
1061 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1062 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1063 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1064 (uint32_t)(addr & 0xffffffff),
1065 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1066 (uint32_t)(addr >> 32));
1069 #endif /* EFSYS_OPT_SIENA */
1071 #if EFSYS_OPT_QSTATS
1073 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1074 static const char * const __efx_tx_qstat_name[] = {
1078 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1082 __in efx_nic_t *enp,
1083 __in unsigned int id)
1085 _NOTE(ARGUNUSED(enp))
1086 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1087 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1089 return (__efx_tx_qstat_name[id]);
1091 #endif /* EFSYS_OPT_NAMES */
1092 #endif /* EFSYS_OPT_QSTATS */
1096 #if EFSYS_OPT_QSTATS
1098 siena_tx_qstats_update(
1099 __in efx_txq_t *etp,
1100 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1104 for (id = 0; id < TX_NQSTATS; id++) {
1105 efsys_stat_t *essp = &stat[id];
1107 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1108 etp->et_stat[id] = 0;
1111 #endif /* EFSYS_OPT_QSTATS */
1115 __in efx_txq_t *etp)
1117 efx_nic_t *enp = etp->et_enp;
1120 /* Purge descriptor queue */
1121 EFX_ZERO_OWORD(oword);
1123 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1124 etp->et_index, &oword, B_TRUE);
1129 __in efx_nic_t *enp)
1131 _NOTE(ARGUNUSED(enp))
1134 #endif /* EFSYS_OPT_SIENA */