1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2007-2018 Solarflare Communications Inc.
11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
13 (_etp)->et_stat[_stat]++; \
14 _NOTE(CONSTANTCONDITION) \
17 #define EFX_TX_QSTAT_INCR(_etp, _stat)
22 static __checkReturn efx_rc_t
30 static __checkReturn efx_rc_t
33 __in unsigned int index,
34 __in unsigned int label,
35 __in efsys_mem_t *esmp,
41 __out unsigned int *addedp);
47 static __checkReturn efx_rc_t
50 __in_ecount(ndescs) efx_buffer_t *eb,
51 __in unsigned int ndescs,
52 __in unsigned int completed,
53 __inout unsigned int *addedp);
58 __in unsigned int added,
59 __in unsigned int pushed);
61 static __checkReturn efx_rc_t
64 __in unsigned int ns);
66 static __checkReturn efx_rc_t
74 __checkReturn efx_rc_t
77 __in_ecount(ndescs) efx_desc_t *ed,
78 __in unsigned int ndescs,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
83 siena_tx_qdesc_dma_create(
85 __in efsys_dma_addr_t addr,
88 __out efx_desc_t *edp);
92 siena_tx_qstats_update(
94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
97 #endif /* EFSYS_OPT_SIENA */
101 static const efx_tx_ops_t __efx_tx_siena_ops = {
102 siena_tx_init, /* etxo_init */
103 siena_tx_fini, /* etxo_fini */
104 siena_tx_qcreate, /* etxo_qcreate */
105 siena_tx_qdestroy, /* etxo_qdestroy */
106 siena_tx_qpost, /* etxo_qpost */
107 siena_tx_qpush, /* etxo_qpush */
108 siena_tx_qpace, /* etxo_qpace */
109 siena_tx_qflush, /* etxo_qflush */
110 siena_tx_qenable, /* etxo_qenable */
111 NULL, /* etxo_qpio_enable */
112 NULL, /* etxo_qpio_disable */
113 NULL, /* etxo_qpio_write */
114 NULL, /* etxo_qpio_post */
115 siena_tx_qdesc_post, /* etxo_qdesc_post */
116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
117 NULL, /* etxo_qdesc_tso_create */
118 NULL, /* etxo_qdesc_tso2_create */
119 NULL, /* etxo_qdesc_vlantci_create */
121 siena_tx_qstats_update, /* etxo_qstats_update */
124 #endif /* EFSYS_OPT_SIENA */
126 #if EFSYS_OPT_HUNTINGTON
127 static const efx_tx_ops_t __efx_tx_hunt_ops = {
128 ef10_tx_init, /* etxo_init */
129 ef10_tx_fini, /* etxo_fini */
130 ef10_tx_qcreate, /* etxo_qcreate */
131 ef10_tx_qdestroy, /* etxo_qdestroy */
132 ef10_tx_qpost, /* etxo_qpost */
133 ef10_tx_qpush, /* etxo_qpush */
134 ef10_tx_qpace, /* etxo_qpace */
135 ef10_tx_qflush, /* etxo_qflush */
136 ef10_tx_qenable, /* etxo_qenable */
137 ef10_tx_qpio_enable, /* etxo_qpio_enable */
138 ef10_tx_qpio_disable, /* etxo_qpio_disable */
139 ef10_tx_qpio_write, /* etxo_qpio_write */
140 ef10_tx_qpio_post, /* etxo_qpio_post */
141 ef10_tx_qdesc_post, /* etxo_qdesc_post */
142 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
143 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
144 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
145 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
147 ef10_tx_qstats_update, /* etxo_qstats_update */
150 #endif /* EFSYS_OPT_HUNTINGTON */
152 #if EFSYS_OPT_MEDFORD
153 static const efx_tx_ops_t __efx_tx_medford_ops = {
154 ef10_tx_init, /* etxo_init */
155 ef10_tx_fini, /* etxo_fini */
156 ef10_tx_qcreate, /* etxo_qcreate */
157 ef10_tx_qdestroy, /* etxo_qdestroy */
158 ef10_tx_qpost, /* etxo_qpost */
159 ef10_tx_qpush, /* etxo_qpush */
160 ef10_tx_qpace, /* etxo_qpace */
161 ef10_tx_qflush, /* etxo_qflush */
162 ef10_tx_qenable, /* etxo_qenable */
163 ef10_tx_qpio_enable, /* etxo_qpio_enable */
164 ef10_tx_qpio_disable, /* etxo_qpio_disable */
165 ef10_tx_qpio_write, /* etxo_qpio_write */
166 ef10_tx_qpio_post, /* etxo_qpio_post */
167 ef10_tx_qdesc_post, /* etxo_qdesc_post */
168 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
169 NULL, /* etxo_qdesc_tso_create */
170 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
171 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
173 ef10_tx_qstats_update, /* etxo_qstats_update */
176 #endif /* EFSYS_OPT_MEDFORD */
178 __checkReturn efx_rc_t
182 const efx_tx_ops_t *etxop;
185 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
186 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
188 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
193 if (enp->en_mod_flags & EFX_MOD_TX) {
198 switch (enp->en_family) {
200 case EFX_FAMILY_SIENA:
201 etxop = &__efx_tx_siena_ops;
203 #endif /* EFSYS_OPT_SIENA */
205 #if EFSYS_OPT_HUNTINGTON
206 case EFX_FAMILY_HUNTINGTON:
207 etxop = &__efx_tx_hunt_ops;
209 #endif /* EFSYS_OPT_HUNTINGTON */
211 #if EFSYS_OPT_MEDFORD
212 case EFX_FAMILY_MEDFORD:
213 etxop = &__efx_tx_medford_ops;
215 #endif /* EFSYS_OPT_MEDFORD */
223 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
225 if ((rc = etxop->etxo_init(enp)) != 0)
228 enp->en_etxop = etxop;
229 enp->en_mod_flags |= EFX_MOD_TX;
239 EFSYS_PROBE1(fail1, efx_rc_t, rc);
241 enp->en_etxop = NULL;
242 enp->en_mod_flags &= ~EFX_MOD_TX;
250 const efx_tx_ops_t *etxop = enp->en_etxop;
252 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
253 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
254 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
255 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
257 etxop->etxo_fini(enp);
259 enp->en_etxop = NULL;
260 enp->en_mod_flags &= ~EFX_MOD_TX;
263 __checkReturn efx_rc_t
266 __in unsigned int index,
267 __in unsigned int label,
268 __in efsys_mem_t *esmp,
273 __deref_out efx_txq_t **etpp,
274 __out unsigned int *addedp)
276 const efx_tx_ops_t *etxop = enp->en_etxop;
280 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
281 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
283 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
284 enp->en_nic_cfg.enc_txq_limit);
286 /* Allocate an TXQ object */
287 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
294 etp->et_magic = EFX_TXQ_MAGIC;
296 etp->et_index = index;
297 etp->et_mask = ndescs - 1;
300 /* Initial descriptor index may be modified by etxo_qcreate */
303 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
304 ndescs, id, flags, eep, etp, addedp)) != 0)
314 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
316 EFSYS_PROBE1(fail1, efx_rc_t, rc);
324 efx_nic_t *enp = etp->et_enp;
325 const efx_tx_ops_t *etxop = enp->en_etxop;
327 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
329 EFSYS_ASSERT(enp->en_tx_qcount != 0);
332 etxop->etxo_qdestroy(etp);
334 /* Free the TXQ object */
335 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
338 __checkReturn efx_rc_t
341 __in_ecount(ndescs) efx_buffer_t *eb,
342 __in unsigned int ndescs,
343 __in unsigned int completed,
344 __inout unsigned int *addedp)
346 efx_nic_t *enp = etp->et_enp;
347 const efx_tx_ops_t *etxop = enp->en_etxop;
350 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
352 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0)
358 EFSYS_PROBE1(fail1, efx_rc_t, rc);
365 __in unsigned int added,
366 __in unsigned int pushed)
368 efx_nic_t *enp = etp->et_enp;
369 const efx_tx_ops_t *etxop = enp->en_etxop;
371 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
373 etxop->etxo_qpush(etp, added, pushed);
376 __checkReturn efx_rc_t
379 __in unsigned int ns)
381 efx_nic_t *enp = etp->et_enp;
382 const efx_tx_ops_t *etxop = enp->en_etxop;
385 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
387 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
393 EFSYS_PROBE1(fail1, efx_rc_t, rc);
397 __checkReturn efx_rc_t
401 efx_nic_t *enp = etp->et_enp;
402 const efx_tx_ops_t *etxop = enp->en_etxop;
405 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
407 if ((rc = etxop->etxo_qflush(etp)) != 0)
413 EFSYS_PROBE1(fail1, efx_rc_t, rc);
421 efx_nic_t *enp = etp->et_enp;
422 const efx_tx_ops_t *etxop = enp->en_etxop;
424 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
426 etxop->etxo_qenable(etp);
429 __checkReturn efx_rc_t
433 efx_nic_t *enp = etp->et_enp;
434 const efx_tx_ops_t *etxop = enp->en_etxop;
437 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
439 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
443 if (etxop->etxo_qpio_enable == NULL) {
447 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
457 EFSYS_PROBE1(fail1, efx_rc_t, rc);
465 efx_nic_t *enp = etp->et_enp;
466 const efx_tx_ops_t *etxop = enp->en_etxop;
468 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
470 if (etxop->etxo_qpio_disable != NULL)
471 etxop->etxo_qpio_disable(etp);
474 __checkReturn efx_rc_t
477 __in_ecount(buf_length) uint8_t *buffer,
478 __in size_t buf_length,
479 __in size_t pio_buf_offset)
481 efx_nic_t *enp = etp->et_enp;
482 const efx_tx_ops_t *etxop = enp->en_etxop;
485 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
487 if (etxop->etxo_qpio_write != NULL) {
488 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
489 pio_buf_offset)) != 0)
497 EFSYS_PROBE1(fail1, efx_rc_t, rc);
501 __checkReturn efx_rc_t
504 __in size_t pkt_length,
505 __in unsigned int completed,
506 __inout unsigned int *addedp)
508 efx_nic_t *enp = etp->et_enp;
509 const efx_tx_ops_t *etxop = enp->en_etxop;
512 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
514 if (etxop->etxo_qpio_post != NULL) {
515 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
524 EFSYS_PROBE1(fail1, efx_rc_t, rc);
528 __checkReturn efx_rc_t
531 __in_ecount(ndescs) efx_desc_t *ed,
532 __in unsigned int ndescs,
533 __in unsigned int completed,
534 __inout unsigned int *addedp)
536 efx_nic_t *enp = etp->et_enp;
537 const efx_tx_ops_t *etxop = enp->en_etxop;
540 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
542 if ((rc = etxop->etxo_qdesc_post(etp, ed,
543 ndescs, completed, addedp)) != 0)
549 EFSYS_PROBE1(fail1, efx_rc_t, rc);
554 efx_tx_qdesc_dma_create(
556 __in efsys_dma_addr_t addr,
559 __out efx_desc_t *edp)
561 efx_nic_t *enp = etp->et_enp;
562 const efx_tx_ops_t *etxop = enp->en_etxop;
564 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
565 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
567 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
571 efx_tx_qdesc_tso_create(
573 __in uint16_t ipv4_id,
574 __in uint32_t tcp_seq,
575 __in uint8_t tcp_flags,
576 __out efx_desc_t *edp)
578 efx_nic_t *enp = etp->et_enp;
579 const efx_tx_ops_t *etxop = enp->en_etxop;
581 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
582 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
584 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
588 efx_tx_qdesc_tso2_create(
590 __in uint16_t ipv4_id,
591 __in uint32_t tcp_seq,
593 __out_ecount(count) efx_desc_t *edp,
596 efx_nic_t *enp = etp->et_enp;
597 const efx_tx_ops_t *etxop = enp->en_etxop;
599 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
600 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
602 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
606 efx_tx_qdesc_vlantci_create(
609 __out efx_desc_t *edp)
611 efx_nic_t *enp = etp->et_enp;
612 const efx_tx_ops_t *etxop = enp->en_etxop;
614 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
615 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
617 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
623 efx_tx_qstats_update(
625 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
627 efx_nic_t *enp = etp->et_enp;
628 const efx_tx_ops_t *etxop = enp->en_etxop;
630 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
632 etxop->etxo_qstats_update(etp, stat);
639 static __checkReturn efx_rc_t
646 * Disable the timer-based TX DMA backoff and allow TX DMA to be
647 * controlled by the RX FIFO fill level (although always allow a
650 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
651 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
652 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
653 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
654 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
655 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
656 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
657 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
660 * Filter all packets less than 14 bytes to avoid parsing
663 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
664 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
667 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
668 * descriptors (which is bad).
670 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
671 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
672 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
677 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
683 id = (_added)++ & (_etp)->et_mask; \
684 offset = id * sizeof (efx_qword_t); \
686 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
687 unsigned int, id, efsys_dma_addr_t, (_addr), \
688 size_t, (_size), boolean_t, (_eop)); \
690 EFX_POPULATE_QWORD_4(qword, \
691 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
692 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
693 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
694 (uint32_t)((_addr) & 0xffffffff), \
695 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
696 (uint32_t)((_addr) >> 32)); \
697 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
699 _NOTE(CONSTANTCONDITION) \
702 static __checkReturn efx_rc_t
705 __in_ecount(ndescs) efx_buffer_t *eb,
706 __in unsigned int ndescs,
707 __in unsigned int completed,
708 __inout unsigned int *addedp)
710 unsigned int added = *addedp;
714 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1))
717 for (i = 0; i < ndescs; i++) {
718 efx_buffer_t *ebp = &eb[i];
719 efsys_dma_addr_t start = ebp->eb_addr;
720 size_t size = ebp->eb_size;
721 efsys_dma_addr_t end = start + size;
724 * Fragments must not span 4k boundaries.
725 * Here it is a stricter requirement than the maximum length.
727 EFSYS_ASSERT(P2ROUNDUP(start + 1,
728 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
730 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
733 EFX_TX_QSTAT_INCR(etp, TX_POST);
739 EFSYS_PROBE1(fail1, efx_rc_t, rc);
747 __in unsigned int added,
748 __in unsigned int pushed)
750 efx_nic_t *enp = etp->et_enp;
755 /* Push the populated descriptors out */
756 wptr = added & etp->et_mask;
758 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
760 /* Only write the third DWORD */
761 EFX_POPULATE_DWORD_1(dword,
762 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
764 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
765 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
766 wptr, pushed & etp->et_mask);
767 EFSYS_PIO_WRITE_BARRIER();
768 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
769 etp->et_index, &dword, B_FALSE);
772 #define EFX_MAX_PACE_VALUE 20
774 static __checkReturn efx_rc_t
777 __in unsigned int ns)
779 efx_nic_t *enp = etp->et_enp;
780 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
782 unsigned int pace_val;
783 unsigned int timer_period;
790 * The pace_val to write into the table is s.t
791 * ns <= timer_period * (2 ^ pace_val)
793 timer_period = 104 / encp->enc_clk_mult;
794 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
795 if ((timer_period << pace_val) >= ns)
799 if (pace_val > EFX_MAX_PACE_VALUE) {
804 /* Update the pacing table */
805 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
806 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
812 EFSYS_PROBE1(fail1, efx_rc_t, rc);
817 static __checkReturn efx_rc_t
821 efx_nic_t *enp = etp->et_enp;
825 efx_tx_qpace(etp, 0);
827 label = etp->et_index;
829 /* Flush the queue */
830 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
831 FRF_AZ_TX_FLUSH_DESCQ, label);
832 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
841 efx_nic_t *enp = etp->et_enp;
844 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
845 etp->et_index, &oword, B_TRUE);
847 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
848 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
849 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
850 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
851 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
853 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
854 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
855 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
857 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
858 etp->et_index, &oword, B_TRUE);
861 static __checkReturn efx_rc_t
864 __in unsigned int index,
865 __in unsigned int label,
866 __in efsys_mem_t *esmp,
872 __out unsigned int *addedp)
874 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
880 _NOTE(ARGUNUSED(esmp))
882 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
883 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
884 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
886 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
887 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
890 (ndescs < EFX_TXQ_MINNDESCS) || (ndescs > EFX_EVQ_MAXNEVS)) {
894 if (index >= encp->enc_txq_limit) {
899 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
901 if ((1 << size) == (int)(ndescs / EFX_TXQ_MINNDESCS))
903 if (id + (1 << size) >= encp->enc_buftbl_limit) {
908 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
909 if ((flags & inner_csum) != 0) {
914 /* Set up the new descriptor queue */
917 EFX_POPULATE_OWORD_6(oword,
918 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
919 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
920 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
921 FRF_AZ_TX_DESCQ_LABEL, label,
922 FRF_AZ_TX_DESCQ_SIZE, size,
923 FRF_AZ_TX_DESCQ_TYPE, 0);
925 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
926 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
927 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
928 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
929 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
931 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
932 etp->et_index, &oword, B_TRUE);
943 EFSYS_PROBE1(fail1, efx_rc_t, rc);
948 __checkReturn efx_rc_t
951 __in_ecount(ndescs) efx_desc_t *ed,
952 __in unsigned int ndescs,
953 __in unsigned int completed,
954 __inout unsigned int *addedp)
956 unsigned int added = *addedp;
960 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
965 for (i = 0; i < ndescs; i++) {
966 efx_desc_t *edp = &ed[i];
970 id = added++ & etp->et_mask;
971 offset = id * sizeof (efx_desc_t);
973 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
976 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
977 unsigned int, added, unsigned int, ndescs);
979 EFX_TX_QSTAT_INCR(etp, TX_POST);
985 EFSYS_PROBE1(fail1, efx_rc_t, rc);
990 siena_tx_qdesc_dma_create(
992 __in efsys_dma_addr_t addr,
995 __out efx_desc_t *edp)
998 * Fragments must not span 4k boundaries.
999 * Here it is a stricter requirement than the maximum length.
1001 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1002 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1004 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1005 efsys_dma_addr_t, addr,
1006 size_t, size, boolean_t, eop);
1008 EFX_POPULATE_QWORD_4(edp->ed_eq,
1009 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1010 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1011 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1012 (uint32_t)(addr & 0xffffffff),
1013 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1014 (uint32_t)(addr >> 32));
1017 #endif /* EFSYS_OPT_SIENA */
1019 #if EFSYS_OPT_QSTATS
1021 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1022 static const char * const __efx_tx_qstat_name[] = {
1026 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1030 __in efx_nic_t *enp,
1031 __in unsigned int id)
1033 _NOTE(ARGUNUSED(enp))
1034 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1035 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1037 return (__efx_tx_qstat_name[id]);
1039 #endif /* EFSYS_OPT_NAMES */
1040 #endif /* EFSYS_OPT_QSTATS */
1044 #if EFSYS_OPT_QSTATS
1046 siena_tx_qstats_update(
1047 __in efx_txq_t *etp,
1048 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1052 for (id = 0; id < TX_NQSTATS; id++) {
1053 efsys_stat_t *essp = &stat[id];
1055 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1056 etp->et_stat[id] = 0;
1059 #endif /* EFSYS_OPT_QSTATS */
1063 __in efx_txq_t *etp)
1065 efx_nic_t *enp = etp->et_enp;
1068 /* Purge descriptor queue */
1069 EFX_ZERO_OWORD(oword);
1071 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1072 etp->et_index, &oword, B_TRUE);
1077 __in efx_nic_t *enp)
1079 _NOTE(ARGUNUSED(enp))
1082 #endif /* EFSYS_OPT_SIENA */