2 * Copyright (c) 2007-2016 Solarflare Communications Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
34 #define EFX_TX_QSTAT_INCR(_etp, _stat)
38 static __checkReturn efx_rc_t
46 static __checkReturn efx_rc_t
49 __in unsigned int index,
50 __in unsigned int label,
51 __in efsys_mem_t *esmp,
57 __out unsigned int *addedp);
63 static __checkReturn efx_rc_t
66 __in_ecount(n) efx_buffer_t *eb,
68 __in unsigned int completed,
69 __inout unsigned int *addedp);
74 __in unsigned int added,
75 __in unsigned int pushed);
77 static __checkReturn efx_rc_t
80 __in unsigned int ns);
82 static __checkReturn efx_rc_t
90 __checkReturn efx_rc_t
93 __in_ecount(n) efx_desc_t *ed,
95 __in unsigned int completed,
96 __inout unsigned int *addedp);
99 siena_tx_qdesc_dma_create(
101 __in efsys_dma_addr_t addr,
104 __out efx_desc_t *edp);
106 #endif /* EFSYS_OPT_SIENA */
110 static const efx_tx_ops_t __efx_tx_siena_ops = {
111 siena_tx_init, /* etxo_init */
112 siena_tx_fini, /* etxo_fini */
113 siena_tx_qcreate, /* etxo_qcreate */
114 siena_tx_qdestroy, /* etxo_qdestroy */
115 siena_tx_qpost, /* etxo_qpost */
116 siena_tx_qpush, /* etxo_qpush */
117 siena_tx_qpace, /* etxo_qpace */
118 siena_tx_qflush, /* etxo_qflush */
119 siena_tx_qenable, /* etxo_qenable */
120 NULL, /* etxo_qpio_enable */
121 NULL, /* etxo_qpio_disable */
122 NULL, /* etxo_qpio_write */
123 NULL, /* etxo_qpio_post */
124 siena_tx_qdesc_post, /* etxo_qdesc_post */
125 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
126 NULL, /* etxo_qdesc_tso_create */
127 NULL, /* etxo_qdesc_tso2_create */
128 NULL, /* etxo_qdesc_vlantci_create */
130 #endif /* EFSYS_OPT_SIENA */
132 __checkReturn efx_rc_t
136 const efx_tx_ops_t *etxop;
139 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
140 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
142 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
147 if (enp->en_mod_flags & EFX_MOD_TX) {
152 switch (enp->en_family) {
154 case EFX_FAMILY_SIENA:
155 etxop = &__efx_tx_siena_ops;
157 #endif /* EFSYS_OPT_SIENA */
165 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
167 if ((rc = etxop->etxo_init(enp)) != 0)
170 enp->en_etxop = etxop;
171 enp->en_mod_flags |= EFX_MOD_TX;
181 EFSYS_PROBE1(fail1, efx_rc_t, rc);
183 enp->en_etxop = NULL;
184 enp->en_mod_flags &= ~EFX_MOD_TX;
192 const efx_tx_ops_t *etxop = enp->en_etxop;
194 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
195 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
196 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
197 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
199 etxop->etxo_fini(enp);
201 enp->en_etxop = NULL;
202 enp->en_mod_flags &= ~EFX_MOD_TX;
205 __checkReturn efx_rc_t
208 __in unsigned int index,
209 __in unsigned int label,
210 __in efsys_mem_t *esmp,
215 __deref_out efx_txq_t **etpp,
216 __out unsigned int *addedp)
218 const efx_tx_ops_t *etxop = enp->en_etxop;
219 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
223 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
224 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
226 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
228 /* Allocate an TXQ object */
229 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
236 etp->et_magic = EFX_TXQ_MAGIC;
238 etp->et_index = index;
239 etp->et_mask = n - 1;
242 /* Initial descriptor index may be modified by etxo_qcreate */
245 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
246 n, id, flags, eep, etp, addedp)) != 0)
256 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
258 EFSYS_PROBE1(fail1, efx_rc_t, rc);
266 efx_nic_t *enp = etp->et_enp;
267 const efx_tx_ops_t *etxop = enp->en_etxop;
269 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
271 EFSYS_ASSERT(enp->en_tx_qcount != 0);
274 etxop->etxo_qdestroy(etp);
276 /* Free the TXQ object */
277 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
280 __checkReturn efx_rc_t
283 __in_ecount(n) efx_buffer_t *eb,
285 __in unsigned int completed,
286 __inout unsigned int *addedp)
288 efx_nic_t *enp = etp->et_enp;
289 const efx_tx_ops_t *etxop = enp->en_etxop;
292 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
294 if ((rc = etxop->etxo_qpost(etp, eb,
295 n, completed, addedp)) != 0)
301 EFSYS_PROBE1(fail1, efx_rc_t, rc);
308 __in unsigned int added,
309 __in unsigned int pushed)
311 efx_nic_t *enp = etp->et_enp;
312 const efx_tx_ops_t *etxop = enp->en_etxop;
314 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
316 etxop->etxo_qpush(etp, added, pushed);
319 __checkReturn efx_rc_t
322 __in unsigned int ns)
324 efx_nic_t *enp = etp->et_enp;
325 const efx_tx_ops_t *etxop = enp->en_etxop;
328 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
330 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
336 EFSYS_PROBE1(fail1, efx_rc_t, rc);
340 __checkReturn efx_rc_t
344 efx_nic_t *enp = etp->et_enp;
345 const efx_tx_ops_t *etxop = enp->en_etxop;
348 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
350 if ((rc = etxop->etxo_qflush(etp)) != 0)
356 EFSYS_PROBE1(fail1, efx_rc_t, rc);
364 efx_nic_t *enp = etp->et_enp;
365 const efx_tx_ops_t *etxop = enp->en_etxop;
367 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
369 etxop->etxo_qenable(etp);
372 __checkReturn efx_rc_t
376 efx_nic_t *enp = etp->et_enp;
377 const efx_tx_ops_t *etxop = enp->en_etxop;
380 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
382 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
386 if (etxop->etxo_qpio_enable == NULL) {
390 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
400 EFSYS_PROBE1(fail1, efx_rc_t, rc);
408 efx_nic_t *enp = etp->et_enp;
409 const efx_tx_ops_t *etxop = enp->en_etxop;
411 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
413 if (etxop->etxo_qpio_disable != NULL)
414 etxop->etxo_qpio_disable(etp);
417 __checkReturn efx_rc_t
420 __in_ecount(buf_length) uint8_t *buffer,
421 __in size_t buf_length,
422 __in size_t pio_buf_offset)
424 efx_nic_t *enp = etp->et_enp;
425 const efx_tx_ops_t *etxop = enp->en_etxop;
428 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
430 if (etxop->etxo_qpio_write != NULL) {
431 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
432 pio_buf_offset)) != 0)
440 EFSYS_PROBE1(fail1, efx_rc_t, rc);
444 __checkReturn efx_rc_t
447 __in size_t pkt_length,
448 __in unsigned int completed,
449 __inout unsigned int *addedp)
451 efx_nic_t *enp = etp->et_enp;
452 const efx_tx_ops_t *etxop = enp->en_etxop;
455 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
457 if (etxop->etxo_qpio_post != NULL) {
458 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
467 EFSYS_PROBE1(fail1, efx_rc_t, rc);
471 __checkReturn efx_rc_t
474 __in_ecount(n) efx_desc_t *ed,
476 __in unsigned int completed,
477 __inout unsigned int *addedp)
479 efx_nic_t *enp = etp->et_enp;
480 const efx_tx_ops_t *etxop = enp->en_etxop;
483 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
485 if ((rc = etxop->etxo_qdesc_post(etp, ed,
486 n, completed, addedp)) != 0)
492 EFSYS_PROBE1(fail1, efx_rc_t, rc);
497 efx_tx_qdesc_dma_create(
499 __in efsys_dma_addr_t addr,
502 __out efx_desc_t *edp)
504 efx_nic_t *enp = etp->et_enp;
505 const efx_tx_ops_t *etxop = enp->en_etxop;
507 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
508 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
510 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
514 efx_tx_qdesc_tso_create(
516 __in uint16_t ipv4_id,
517 __in uint32_t tcp_seq,
518 __in uint8_t tcp_flags,
519 __out efx_desc_t *edp)
521 efx_nic_t *enp = etp->et_enp;
522 const efx_tx_ops_t *etxop = enp->en_etxop;
524 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
525 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
527 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
531 efx_tx_qdesc_tso2_create(
533 __in uint16_t ipv4_id,
534 __in uint32_t tcp_seq,
536 __out_ecount(count) efx_desc_t *edp,
539 efx_nic_t *enp = etp->et_enp;
540 const efx_tx_ops_t *etxop = enp->en_etxop;
542 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
543 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
545 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
549 efx_tx_qdesc_vlantci_create(
552 __out efx_desc_t *edp)
554 efx_nic_t *enp = etp->et_enp;
555 const efx_tx_ops_t *etxop = enp->en_etxop;
557 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
558 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
560 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
566 static __checkReturn efx_rc_t
573 * Disable the timer-based TX DMA backoff and allow TX DMA to be
574 * controlled by the RX FIFO fill level (although always allow a
577 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
578 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
579 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
580 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
581 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
582 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
583 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
584 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
587 * Filter all packets less than 14 bytes to avoid parsing
590 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
591 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
594 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
595 * descriptors (which is bad).
597 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
598 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
599 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
604 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
610 id = (_added)++ & (_etp)->et_mask; \
611 offset = id * sizeof (efx_qword_t); \
613 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
614 unsigned int, id, efsys_dma_addr_t, (_addr), \
615 size_t, (_size), boolean_t, (_eop)); \
617 EFX_POPULATE_QWORD_4(qword, \
618 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
619 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
620 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
621 (uint32_t)((_addr) & 0xffffffff), \
622 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
623 (uint32_t)((_addr) >> 32)); \
624 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
626 _NOTE(CONSTANTCONDITION) \
629 static __checkReturn efx_rc_t
632 __in_ecount(n) efx_buffer_t *eb,
634 __in unsigned int completed,
635 __inout unsigned int *addedp)
637 unsigned int added = *addedp;
641 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
644 for (i = 0; i < n; i++) {
645 efx_buffer_t *ebp = &eb[i];
646 efsys_dma_addr_t start = ebp->eb_addr;
647 size_t size = ebp->eb_size;
648 efsys_dma_addr_t end = start + size;
650 /* Fragments must not span 4k boundaries. */
651 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);
653 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
656 EFX_TX_QSTAT_INCR(etp, TX_POST);
662 EFSYS_PROBE1(fail1, efx_rc_t, rc);
670 __in unsigned int added,
671 __in unsigned int pushed)
673 efx_nic_t *enp = etp->et_enp;
678 /* Push the populated descriptors out */
679 wptr = added & etp->et_mask;
681 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
683 /* Only write the third DWORD */
684 EFX_POPULATE_DWORD_1(dword,
685 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
687 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
688 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
689 wptr, pushed & etp->et_mask);
690 EFSYS_PIO_WRITE_BARRIER();
691 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
692 etp->et_index, &dword, B_FALSE);
695 #define EFX_MAX_PACE_VALUE 20
697 static __checkReturn efx_rc_t
700 __in unsigned int ns)
702 efx_nic_t *enp = etp->et_enp;
703 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
705 unsigned int pace_val;
706 unsigned int timer_period;
713 * The pace_val to write into the table is s.t
714 * ns <= timer_period * (2 ^ pace_val)
716 timer_period = 104 / encp->enc_clk_mult;
717 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
718 if ((timer_period << pace_val) >= ns)
722 if (pace_val > EFX_MAX_PACE_VALUE) {
727 /* Update the pacing table */
728 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
729 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
735 EFSYS_PROBE1(fail1, efx_rc_t, rc);
740 static __checkReturn efx_rc_t
744 efx_nic_t *enp = etp->et_enp;
748 efx_tx_qpace(etp, 0);
750 label = etp->et_index;
752 /* Flush the queue */
753 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
754 FRF_AZ_TX_FLUSH_DESCQ, label);
755 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
764 efx_nic_t *enp = etp->et_enp;
767 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
768 etp->et_index, &oword, B_TRUE);
770 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
771 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
772 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
773 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
774 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
776 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
777 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
778 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
780 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
781 etp->et_index, &oword, B_TRUE);
784 static __checkReturn efx_rc_t
787 __in unsigned int index,
788 __in unsigned int label,
789 __in efsys_mem_t *esmp,
795 __out unsigned int *addedp)
797 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
802 _NOTE(ARGUNUSED(esmp))
804 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
805 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
806 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
808 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
809 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
811 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
815 if (index >= encp->enc_txq_limit) {
820 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
822 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
824 if (id + (1 << size) >= encp->enc_buftbl_limit) {
829 /* Set up the new descriptor queue */
832 EFX_POPULATE_OWORD_6(oword,
833 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
834 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
835 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
836 FRF_AZ_TX_DESCQ_LABEL, label,
837 FRF_AZ_TX_DESCQ_SIZE, size,
838 FRF_AZ_TX_DESCQ_TYPE, 0);
840 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
841 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
842 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
843 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
844 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
846 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
847 etp->et_index, &oword, B_TRUE);
856 EFSYS_PROBE1(fail1, efx_rc_t, rc);
861 __checkReturn efx_rc_t
864 __in_ecount(n) efx_desc_t *ed,
866 __in unsigned int completed,
867 __inout unsigned int *addedp)
869 unsigned int added = *addedp;
873 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
878 for (i = 0; i < n; i++) {
879 efx_desc_t *edp = &ed[i];
883 id = added++ & etp->et_mask;
884 offset = id * sizeof (efx_desc_t);
886 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
889 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
890 unsigned int, added, unsigned int, n);
892 EFX_TX_QSTAT_INCR(etp, TX_POST);
898 EFSYS_PROBE1(fail1, efx_rc_t, rc);
903 siena_tx_qdesc_dma_create(
905 __in efsys_dma_addr_t addr,
908 __out efx_desc_t *edp)
910 /* Fragments must not span 4k boundaries. */
911 EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);
913 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
914 efsys_dma_addr_t, addr,
915 size_t, size, boolean_t, eop);
917 EFX_POPULATE_QWORD_4(edp->ed_eq,
918 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
919 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
920 FSF_AZ_TX_KER_BUF_ADDR_DW0,
921 (uint32_t)(addr & 0xffffffff),
922 FSF_AZ_TX_KER_BUF_ADDR_DW1,
923 (uint32_t)(addr >> 32));
926 #endif /* EFSYS_OPT_SIENA */
934 efx_nic_t *enp = etp->et_enp;
937 /* Purge descriptor queue */
938 EFX_ZERO_OWORD(oword);
940 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
941 etp->et_index, &oword, B_TRUE);
948 _NOTE(ARGUNUSED(enp))
951 #endif /* EFSYS_OPT_SIENA */