1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
41 __out unsigned int *addedp);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
58 __in unsigned int added,
59 __in unsigned int pushed);
61 static __checkReturn efx_rc_t
64 __in unsigned int ns);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr,
88 __out efx_desc_t *edp);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
120 NULL, /* etxo_qdesc_checksum_create */
122 siena_tx_qstats_update, /* etxo_qstats_update */
125 #endif /* EFSYS_OPT_SIENA */
127 #if EFSYS_OPT_HUNTINGTON
128 static const efx_tx_ops_t __efx_tx_hunt_ops = {
129 ef10_tx_init, /* etxo_init */
130 ef10_tx_fini, /* etxo_fini */
131 ef10_tx_qcreate, /* etxo_qcreate */
132 ef10_tx_qdestroy, /* etxo_qdestroy */
133 ef10_tx_qpost, /* etxo_qpost */
134 ef10_tx_qpush, /* etxo_qpush */
135 ef10_tx_qpace, /* etxo_qpace */
136 ef10_tx_qflush, /* etxo_qflush */
137 ef10_tx_qenable, /* etxo_qenable */
138 ef10_tx_qpio_enable, /* etxo_qpio_enable */
139 ef10_tx_qpio_disable, /* etxo_qpio_disable */
140 ef10_tx_qpio_write, /* etxo_qpio_write */
141 ef10_tx_qpio_post, /* etxo_qpio_post */
142 ef10_tx_qdesc_post, /* etxo_qdesc_post */
143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
149 ef10_tx_qstats_update, /* etxo_qstats_update */
152 #endif /* EFSYS_OPT_HUNTINGTON */
154 #if EFSYS_OPT_MEDFORD
155 static const efx_tx_ops_t __efx_tx_medford_ops = {
156 ef10_tx_init, /* etxo_init */
157 ef10_tx_fini, /* etxo_fini */
158 ef10_tx_qcreate, /* etxo_qcreate */
159 ef10_tx_qdestroy, /* etxo_qdestroy */
160 ef10_tx_qpost, /* etxo_qpost */
161 ef10_tx_qpush, /* etxo_qpush */
162 ef10_tx_qpace, /* etxo_qpace */
163 ef10_tx_qflush, /* etxo_qflush */
164 ef10_tx_qenable, /* etxo_qenable */
165 ef10_tx_qpio_enable, /* etxo_qpio_enable */
166 ef10_tx_qpio_disable, /* etxo_qpio_disable */
167 ef10_tx_qpio_write, /* etxo_qpio_write */
168 ef10_tx_qpio_post, /* etxo_qpio_post */
169 ef10_tx_qdesc_post, /* etxo_qdesc_post */
170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
171 NULL, /* etxo_qdesc_tso_create */
172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
176 ef10_tx_qstats_update, /* etxo_qstats_update */
179 #endif /* EFSYS_OPT_MEDFORD */
181 #if EFSYS_OPT_MEDFORD2
182 static const efx_tx_ops_t __efx_tx_medford2_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
203 ef10_tx_qstats_update, /* etxo_qstats_update */
206 #endif /* EFSYS_OPT_MEDFORD2 */
209 __checkReturn efx_rc_t
213 const efx_tx_ops_t *etxop;
216 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
217 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
219 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
224 if (enp->en_mod_flags & EFX_MOD_TX) {
229 switch (enp->en_family) {
231 case EFX_FAMILY_SIENA:
232 etxop = &__efx_tx_siena_ops;
234 #endif /* EFSYS_OPT_SIENA */
236 #if EFSYS_OPT_HUNTINGTON
237 case EFX_FAMILY_HUNTINGTON:
238 etxop = &__efx_tx_hunt_ops;
240 #endif /* EFSYS_OPT_HUNTINGTON */
242 #if EFSYS_OPT_MEDFORD
243 case EFX_FAMILY_MEDFORD:
244 etxop = &__efx_tx_medford_ops;
246 #endif /* EFSYS_OPT_MEDFORD */
248 #if EFSYS_OPT_MEDFORD2
249 case EFX_FAMILY_MEDFORD2:
250 etxop = &__efx_tx_medford2_ops;
252 #endif /* EFSYS_OPT_MEDFORD2 */
260 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
262 if ((rc = etxop->etxo_init(enp)) != 0)
265 enp->en_etxop = etxop;
266 enp->en_mod_flags |= EFX_MOD_TX;
276 EFSYS_PROBE1(fail1, efx_rc_t, rc);
278 enp->en_etxop = NULL;
279 enp->en_mod_flags &= ~EFX_MOD_TX;
287 const efx_tx_ops_t *etxop = enp->en_etxop;
289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
291 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
292 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
294 etxop->etxo_fini(enp);
296 enp->en_etxop = NULL;
297 enp->en_mod_flags &= ~EFX_MOD_TX;
300 __checkReturn efx_rc_t
303 __in unsigned int index,
304 __in unsigned int label,
305 __in efsys_mem_t *esmp,
310 __deref_out efx_txq_t **etpp,
311 __out unsigned int *addedp)
313 const efx_tx_ops_t *etxop = enp->en_etxop;
317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
320 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
321 enp->en_nic_cfg.enc_txq_limit);
323 /* Allocate an TXQ object */
324 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
331 etp->et_magic = EFX_TXQ_MAGIC;
333 etp->et_index = index;
334 etp->et_mask = ndescs - 1;
337 /* Initial descriptor index may be modified by etxo_qcreate */
340 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
341 ndescs, id, flags, eep, etp, addedp)) != 0)
351 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
353 EFSYS_PROBE1(fail1, efx_rc_t, rc);
361 efx_nic_t *enp = etp->et_enp;
362 const efx_tx_ops_t *etxop = enp->en_etxop;
364 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
366 EFSYS_ASSERT(enp->en_tx_qcount != 0);
369 etxop->etxo_qdestroy(etp);
371 /* Free the TXQ object */
372 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
375 __checkReturn efx_rc_t
378 __in_ecount(ndescs) efx_buffer_t *eb,
379 __in unsigned int ndescs,
380 __in unsigned int completed,
381 __inout unsigned int *addedp)
383 efx_nic_t *enp = etp->et_enp;
384 const efx_tx_ops_t *etxop = enp->en_etxop;
387 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
389 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
395 EFSYS_PROBE1(fail1, efx_rc_t, rc);
402 __in unsigned int added,
403 __in unsigned int pushed)
405 efx_nic_t *enp = etp->et_enp;
406 const efx_tx_ops_t *etxop = enp->en_etxop;
408 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
410 etxop->etxo_qpush(etp, added, pushed);
413 __checkReturn efx_rc_t
416 __in unsigned int ns)
418 efx_nic_t *enp = etp->et_enp;
419 const efx_tx_ops_t *etxop = enp->en_etxop;
422 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
424 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
430 EFSYS_PROBE1(fail1, efx_rc_t, rc);
434 __checkReturn efx_rc_t
438 efx_nic_t *enp = etp->et_enp;
439 const efx_tx_ops_t *etxop = enp->en_etxop;
442 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
444 if ((rc = etxop->etxo_qflush(etp)) != 0)
450 EFSYS_PROBE1(fail1, efx_rc_t, rc);
458 efx_nic_t *enp = etp->et_enp;
459 const efx_tx_ops_t *etxop = enp->en_etxop;
461 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
463 etxop->etxo_qenable(etp);
466 __checkReturn efx_rc_t
470 efx_nic_t *enp = etp->et_enp;
471 const efx_tx_ops_t *etxop = enp->en_etxop;
474 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
476 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
480 if (etxop->etxo_qpio_enable == NULL) {
484 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
494 EFSYS_PROBE1(fail1, efx_rc_t, rc);
502 efx_nic_t *enp = etp->et_enp;
503 const efx_tx_ops_t *etxop = enp->en_etxop;
505 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
507 if (etxop->etxo_qpio_disable != NULL)
508 etxop->etxo_qpio_disable(etp);
511 __checkReturn efx_rc_t
514 __in_ecount(buf_length) uint8_t *buffer,
515 __in size_t buf_length,
516 __in size_t pio_buf_offset)
518 efx_nic_t *enp = etp->et_enp;
519 const efx_tx_ops_t *etxop = enp->en_etxop;
522 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
524 if (etxop->etxo_qpio_write != NULL) {
525 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
526 pio_buf_offset)) != 0)
534 EFSYS_PROBE1(fail1, efx_rc_t, rc);
538 __checkReturn efx_rc_t
541 __in size_t pkt_length,
542 __in unsigned int completed,
543 __inout unsigned int *addedp)
545 efx_nic_t *enp = etp->et_enp;
546 const efx_tx_ops_t *etxop = enp->en_etxop;
549 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
551 if (etxop->etxo_qpio_post != NULL) {
552 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
561 EFSYS_PROBE1(fail1, efx_rc_t, rc);
565 __checkReturn efx_rc_t
568 __in_ecount(ndescs) efx_desc_t *ed,
569 __in unsigned int ndescs,
570 __in unsigned int completed,
571 __inout unsigned int *addedp)
573 efx_nic_t *enp = etp->et_enp;
574 const efx_tx_ops_t *etxop = enp->en_etxop;
577 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
579 if ((rc = etxop->etxo_qdesc_post(etp, ed,
580 ndescs, completed, addedp)) != 0)
586 EFSYS_PROBE1(fail1, efx_rc_t, rc);
591 efx_tx_qdesc_dma_create(
593 __in efsys_dma_addr_t addr,
596 __out efx_desc_t *edp)
598 efx_nic_t *enp = etp->et_enp;
599 const efx_tx_ops_t *etxop = enp->en_etxop;
601 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
602 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
604 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
608 efx_tx_qdesc_tso_create(
610 __in uint16_t ipv4_id,
611 __in uint32_t tcp_seq,
612 __in uint8_t tcp_flags,
613 __out efx_desc_t *edp)
615 efx_nic_t *enp = etp->et_enp;
616 const efx_tx_ops_t *etxop = enp->en_etxop;
618 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
619 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
621 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
625 efx_tx_qdesc_tso2_create(
627 __in uint16_t ipv4_id,
628 __in uint16_t outer_ipv4_id,
629 __in uint32_t tcp_seq,
631 __out_ecount(count) efx_desc_t *edp,
634 efx_nic_t *enp = etp->et_enp;
635 const efx_tx_ops_t *etxop = enp->en_etxop;
637 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
638 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
640 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id,
641 tcp_seq, mss, edp, count);
645 efx_tx_qdesc_vlantci_create(
648 __out efx_desc_t *edp)
650 efx_nic_t *enp = etp->et_enp;
651 const efx_tx_ops_t *etxop = enp->en_etxop;
653 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
654 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
656 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
660 efx_tx_qdesc_checksum_create(
663 __out efx_desc_t *edp)
665 efx_nic_t *enp = etp->et_enp;
666 const efx_tx_ops_t *etxop = enp->en_etxop;
668 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
669 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
671 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
677 efx_tx_qstats_update(
679 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
681 efx_nic_t *enp = etp->et_enp;
682 const efx_tx_ops_t *etxop = enp->en_etxop;
684 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
686 etxop->etxo_qstats_update(etp, stat);
693 static __checkReturn efx_rc_t
700 * Disable the timer-based TX DMA backoff and allow TX DMA to be
701 * controlled by the RX FIFO fill level (although always allow a
704 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
705 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
706 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
707 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
708 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
709 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
710 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
711 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
714 * Filter all packets less than 14 bytes to avoid parsing
717 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
718 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
721 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
722 * descriptors (which is bad).
724 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
725 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
726 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
731 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
737 id = (_added)++ & (_etp)->et_mask; \
738 offset = id * sizeof (efx_qword_t); \
740 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
741 unsigned int, id, efsys_dma_addr_t, (_addr), \
742 size_t, (_size), boolean_t, (_eop)); \
744 EFX_POPULATE_QWORD_4(qword, \
745 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
746 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
747 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
748 (uint32_t)((_addr) & 0xffffffff), \
749 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
750 (uint32_t)((_addr) >> 32)); \
751 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
753 _NOTE(CONSTANTCONDITION) \
756 static __checkReturn efx_rc_t
759 __in_ecount(ndescs) efx_buffer_t *eb,
760 __in unsigned int ndescs,
761 __in unsigned int completed,
762 __inout unsigned int *addedp)
764 unsigned int added = *addedp;
768 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
771 for (i = 0; i < ndescs; i++) {
772 efx_buffer_t *ebp = &eb[i];
773 efsys_dma_addr_t start = ebp->eb_addr;
774 size_t size = ebp->eb_size;
775 efsys_dma_addr_t end = start + size;
778 * Fragments must not span 4k boundaries.
779 * Here it is a stricter requirement than the maximum length.
781 EFSYS_ASSERT(P2ROUNDUP(start + 1,
782 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
784 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
787 EFX_TX_QSTAT_INCR(etp, TX_POST);
793 EFSYS_PROBE1(fail1, efx_rc_t, rc);
801 __in unsigned int added,
802 __in unsigned int pushed)
804 efx_nic_t *enp = etp->et_enp;
809 /* Push the populated descriptors out */
810 wptr = added & etp->et_mask;
812 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
814 /* Only write the third DWORD */
815 EFX_POPULATE_DWORD_1(dword,
816 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
818 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
819 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
820 wptr, pushed & etp->et_mask);
821 EFSYS_PIO_WRITE_BARRIER();
822 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
823 etp->et_index, &dword, B_FALSE);
826 #define EFX_MAX_PACE_VALUE 20
828 static __checkReturn efx_rc_t
831 __in unsigned int ns)
833 efx_nic_t *enp = etp->et_enp;
834 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
836 unsigned int pace_val;
837 unsigned int timer_period;
844 * The pace_val to write into the table is s.t
845 * ns <= timer_period * (2 ^ pace_val)
847 timer_period = 104 / encp->enc_clk_mult;
848 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
849 if ((timer_period << pace_val) >= ns)
853 if (pace_val > EFX_MAX_PACE_VALUE) {
858 /* Update the pacing table */
859 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
860 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
866 EFSYS_PROBE1(fail1, efx_rc_t, rc);
871 static __checkReturn efx_rc_t
875 efx_nic_t *enp = etp->et_enp;
879 efx_tx_qpace(etp, 0);
881 label = etp->et_index;
883 /* Flush the queue */
884 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
885 FRF_AZ_TX_FLUSH_DESCQ, label);
886 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
895 efx_nic_t *enp = etp->et_enp;
898 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
899 etp->et_index, &oword, B_TRUE);
901 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
902 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
903 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
904 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
905 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
907 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
908 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
909 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
911 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
912 etp->et_index, &oword, B_TRUE);
915 static __checkReturn efx_rc_t
918 __in unsigned int index,
919 __in unsigned int label,
920 __in efsys_mem_t *esmp,
926 __out unsigned int *addedp)
928 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
934 _NOTE(ARGUNUSED(esmp))
936 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
937 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
938 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
940 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
941 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
944 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) {
948 if (index >= encp->enc_txq_limit) {
953 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
955 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS))
957 if (id + (1 << size) >= encp->enc_buftbl_limit) {
962 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
963 if ((flags & inner_csum) != 0) {
968 /* Set up the new descriptor queue */
971 EFX_POPULATE_OWORD_6(oword,
972 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
973 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
974 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
975 FRF_AZ_TX_DESCQ_LABEL, label,
976 FRF_AZ_TX_DESCQ_SIZE, size,
977 FRF_AZ_TX_DESCQ_TYPE, 0);
979 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
980 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
981 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
982 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
983 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
985 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
986 etp->et_index, &oword, B_TRUE);
997 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1002 __checkReturn efx_rc_t
1003 siena_tx_qdesc_post(
1004 __in efx_txq_t *etp,
1005 __in_ecount(ndescs) efx_desc_t *ed,
1006 __in unsigned int ndescs,
1007 __in unsigned int completed,
1008 __inout unsigned int *addedp)
1010 unsigned int added = *addedp;
1014 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1019 for (i = 0; i < ndescs; i++) {
1020 efx_desc_t *edp = &ed[i];
1024 id = added++ & etp->et_mask;
1025 offset = id * sizeof (efx_desc_t);
1027 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1030 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1031 unsigned int, added, unsigned int, ndescs);
1033 EFX_TX_QSTAT_INCR(etp, TX_POST);
1039 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1044 siena_tx_qdesc_dma_create(
1045 __in efx_txq_t *etp,
1046 __in efsys_dma_addr_t addr,
1049 __out efx_desc_t *edp)
1052 * Fragments must not span 4k boundaries.
1053 * Here it is a stricter requirement than the maximum length.
1055 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1056 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1058 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1059 efsys_dma_addr_t, addr,
1060 size_t, size, boolean_t, eop);
1062 EFX_POPULATE_QWORD_4(edp->ed_eq,
1063 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1064 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1065 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1066 (uint32_t)(addr & 0xffffffff),
1067 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1068 (uint32_t)(addr >> 32));
1071 #endif /* EFSYS_OPT_SIENA */
1073 #if EFSYS_OPT_QSTATS
1075 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1076 static const char * const __efx_tx_qstat_name[] = {
1080 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1084 __in efx_nic_t *enp,
1085 __in unsigned int id)
1087 _NOTE(ARGUNUSED(enp))
1088 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1089 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1091 return (__efx_tx_qstat_name[id]);
1093 #endif /* EFSYS_OPT_NAMES */
1094 #endif /* EFSYS_OPT_QSTATS */
1098 #if EFSYS_OPT_QSTATS
1100 siena_tx_qstats_update(
1101 __in efx_txq_t *etp,
1102 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1106 for (id = 0; id < TX_NQSTATS; id++) {
1107 efsys_stat_t *essp = &stat[id];
1109 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1110 etp->et_stat[id] = 0;
1113 #endif /* EFSYS_OPT_QSTATS */
1117 __in efx_txq_t *etp)
1119 efx_nic_t *enp = etp->et_enp;
1122 /* Purge descriptor queue */
1123 EFX_ZERO_OWORD(oword);
1125 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1126 etp->et_index, &oword, B_TRUE);
1131 __in efx_nic_t *enp)
1133 _NOTE(ARGUNUSED(enp))
1136 #endif /* EFSYS_OPT_SIENA */