- * - (RTE_ETH_RX_DESC_AVAIL): Descriptor is available for the hardware to
- * receive a packet.
- * - (RTE_ETH_RX_DESC_DONE): Descriptor is done, it is filled by hw, but
- * not yet processed by the driver (i.e. in the receive queue).
- * - (RTE_ETH_RX_DESC_UNAVAIL): Descriptor is unavailable, either hold by
- * the driver and not yet returned to hw, or reserved by the hw.
- * - (-EINVAL) bad descriptor offset.
- * - (-ENOTSUP) if the device does not support this function.
- * - (-ENODEV) bad port or queue (only if compiled with debug).
- */
-static inline int
-rte_eth_rx_descriptor_status(uint16_t port_id, uint16_t queue_id,
- uint16_t offset)
-{
- struct rte_eth_dev *dev;
- void *rxq;
-
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
- RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
-#endif
- dev = &rte_eth_devices[port_id];
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
- if (queue_id >= dev->data->nb_rx_queues)
- return -ENODEV;
-#endif
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_descriptor_status, -ENOTSUP);
- rxq = dev->data->rx_queues[queue_id];
-
- return (*dev->dev_ops->rx_descriptor_status)(rxq, offset);
-}
-
-#define RTE_ETH_TX_DESC_FULL 0 /**< Desc filled for hw, waiting xmit. */
-#define RTE_ETH_TX_DESC_DONE 1 /**< Desc done, packet is transmitted. */
-#define RTE_ETH_TX_DESC_UNAVAIL 2 /**< Desc used by driver or hw. */
-
-/**
- * Check the status of a Tx descriptor in the queue.
- *
- * It should be called in a similar context than the Tx function:
- * - on a dataplane core
- * - not concurrently on the same queue
- *
- * Since it's a dataplane function, no check is performed on port_id and
- * queue_id. The caller must therefore ensure that the port is enabled
- * and the queue is configured and running.
- *
- * Note: accessing to a random descriptor in the ring may trigger cache
- * misses and have a performance impact.
- *
- * @param port_id
- * A valid port identifier of the Ethernet device which.
- * @param queue_id
- * A valid Tx queue identifier on this port.
- * @param offset
- * The offset of the descriptor starting from tail (0 is the place where
- * the next packet will be send).
- *
- * @return
- * - (RTE_ETH_TX_DESC_FULL) Descriptor is being processed by the hw, i.e.
- * in the transmit queue.
- * - (RTE_ETH_TX_DESC_DONE) Hardware is done with this descriptor, it can
- * be reused by the driver.
- * - (RTE_ETH_TX_DESC_UNAVAIL): Descriptor is unavailable, reserved by the
- * driver or the hardware.
- * - (-EINVAL) bad descriptor offset.
- * - (-ENOTSUP) if the device does not support this function.
- * - (-ENODEV) bad port or queue (only if compiled with debug).
- */
-static inline int rte_eth_tx_descriptor_status(uint16_t port_id,
- uint16_t queue_id, uint16_t offset)
-{
- struct rte_eth_dev *dev;
- void *txq;
-
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
- RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
-#endif
- dev = &rte_eth_devices[port_id];
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
- if (queue_id >= dev->data->nb_tx_queues)
- return -ENODEV;
-#endif
- RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_descriptor_status, -ENOTSUP);
- txq = dev->data->tx_queues[queue_id];
-
- return (*dev->dev_ops->tx_descriptor_status)(txq, offset);
-}
-
-/**
- * Send a burst of output packets on a transmit queue of an Ethernet device.
- *
- * The rte_eth_tx_burst() function is invoked to transmit output packets
- * on the output queue *queue_id* of the Ethernet device designated by its
- * *port_id*.
- * The *nb_pkts* parameter is the number of packets to send which are
- * supplied in the *tx_pkts* array of *rte_mbuf* structures, each of them
- * allocated from a pool created with rte_pktmbuf_pool_create().
- * The rte_eth_tx_burst() function loops, sending *nb_pkts* packets,
- * up to the number of transmit descriptors available in the TX ring of the
- * transmit queue.
- * For each packet to send, the rte_eth_tx_burst() function performs
- * the following operations:
- *
- * - Pick up the next available descriptor in the transmit ring.
- *
- * - Free the network buffer previously sent with that descriptor, if any.
- *
- * - Initialize the transmit descriptor with the information provided
- * in the *rte_mbuf data structure.
- *
- * In the case of a segmented packet composed of a list of *rte_mbuf* buffers,
- * the rte_eth_tx_burst() function uses several transmit descriptors
- * of the ring.
- *
- * The rte_eth_tx_burst() function returns the number of packets it
- * actually sent. A return value equal to *nb_pkts* means that all packets
- * have been sent, and this is likely to signify that other output packets
- * could be immediately transmitted again. Applications that implement a
- * "send as many packets to transmit as possible" policy can check this
- * specific case and keep invoking the rte_eth_tx_burst() function until
- * a value less than *nb_pkts* is returned.
- *
- * It is the responsibility of the rte_eth_tx_burst() function to
- * transparently free the memory buffers of packets previously sent.
- * This feature is driven by the *tx_free_thresh* value supplied to the
- * rte_eth_dev_configure() function at device configuration time.
- * When the number of free TX descriptors drops below this threshold, the
- * rte_eth_tx_burst() function must [attempt to] free the *rte_mbuf* buffers
- * of those packets whose transmission was effectively completed.
- *
- * If the PMD is DEV_TX_OFFLOAD_MT_LOCKFREE capable, multiple threads can
- * invoke this function concurrently on the same tx queue without SW lock.
- * @see rte_eth_dev_info_get, struct rte_eth_txconf::txq_flags
- *
- * @param port_id
- * The port identifier of the Ethernet device.
- * @param queue_id
- * The index of the transmit queue through which output packets must be
- * sent.
- * The value must be in the range [0, nb_tx_queue - 1] previously supplied
- * to rte_eth_dev_configure().
- * @param tx_pkts
- * The address of an array of *nb_pkts* pointers to *rte_mbuf* structures
- * which contain the output packets.
- * @param nb_pkts
- * The maximum number of packets to transmit.
- * @return
- * The number of output packets actually stored in transmit descriptors of
- * the transmit ring. The return value can be less than the value of the
- * *tx_pkts* parameter when the transmit ring is full or has been filled up.
- */
-static inline uint16_t
-rte_eth_tx_burst(uint16_t port_id, uint16_t queue_id,
- struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
-{
- struct rte_eth_dev *dev = &rte_eth_devices[port_id];
-
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
- RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
- RTE_FUNC_PTR_OR_ERR_RET(*dev->tx_pkt_burst, 0);
-
- if (queue_id >= dev->data->nb_tx_queues) {
- RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
- return 0;
- }
-#endif
-
-#ifdef RTE_ETHDEV_RXTX_CALLBACKS
- struct rte_eth_rxtx_callback *cb = dev->pre_tx_burst_cbs[queue_id];
-
- if (unlikely(cb != NULL)) {
- do {
- nb_pkts = cb->fn.tx(port_id, queue_id, tx_pkts, nb_pkts,
- cb->param);
- cb = cb->next;
- } while (cb != NULL);
- }
-#endif
-
- return (*dev->tx_pkt_burst)(dev->data->tx_queues[queue_id], tx_pkts, nb_pkts);
-}
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice
- *
- * Process a burst of output packets on a transmit queue of an Ethernet device.
- *
- * The rte_eth_tx_prepare() function is invoked to prepare output packets to be
- * transmitted on the output queue *queue_id* of the Ethernet device designated
- * by its *port_id*.
- * The *nb_pkts* parameter is the number of packets to be prepared which are
- * supplied in the *tx_pkts* array of *rte_mbuf* structures, each of them
- * allocated from a pool created with rte_pktmbuf_pool_create().
- * For each packet to send, the rte_eth_tx_prepare() function performs
- * the following operations:
- *
- * - Check if packet meets devices requirements for tx offloads.
- *
- * - Check limitations about number of segments.
- *
- * - Check additional requirements when debug is enabled.
- *
- * - Update and/or reset required checksums when tx offload is set for packet.
- *
- * Since this function can modify packet data, provided mbufs must be safely
- * writable (e.g. modified data cannot be in shared segment).
- *
- * The rte_eth_tx_prepare() function returns the number of packets ready to be
- * sent. A return value equal to *nb_pkts* means that all packets are valid and
- * ready to be sent, otherwise stops processing on the first invalid packet and
- * leaves the rest packets untouched.
- *
- * When this functionality is not implemented in the driver, all packets are
- * are returned untouched.
- *
- * @param port_id
- * The port identifier of the Ethernet device.
- * The value must be a valid port id.
- * @param queue_id
- * The index of the transmit queue through which output packets must be
- * sent.
- * The value must be in the range [0, nb_tx_queue - 1] previously supplied
- * to rte_eth_dev_configure().
- * @param tx_pkts
- * The address of an array of *nb_pkts* pointers to *rte_mbuf* structures
- * which contain the output packets.
- * @param nb_pkts
- * The maximum number of packets to process.
- * @return
- * The number of packets correct and ready to be sent. The return value can be
- * less than the value of the *tx_pkts* parameter when some packet doesn't
- * meet devices requirements with rte_errno set appropriately:
- * - -EINVAL: offload flags are not correctly set
- * - -ENOTSUP: the offload feature is not supported by the hardware
- *
- */
-
-#ifndef RTE_ETHDEV_TX_PREPARE_NOOP
-
-static inline uint16_t
-rte_eth_tx_prepare(uint16_t port_id, uint16_t queue_id,
- struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
-{
- struct rte_eth_dev *dev;
-
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
- if (!rte_eth_dev_is_valid_port(port_id)) {
- RTE_PMD_DEBUG_TRACE("Invalid TX port_id=%d\n", port_id);
- rte_errno = -EINVAL;
- return 0;
- }
-#endif
-
- dev = &rte_eth_devices[port_id];
-
-#ifdef RTE_LIBRTE_ETHDEV_DEBUG
- if (queue_id >= dev->data->nb_tx_queues) {
- RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
- rte_errno = -EINVAL;
- return 0;
- }
-#endif
-
- if (!dev->tx_pkt_prepare)
- return nb_pkts;
-
- return (*dev->tx_pkt_prepare)(dev->data->tx_queues[queue_id],
- tx_pkts, nb_pkts);
-}
-
-#else
-
-/*
- * Native NOOP operation for compilation targets which doesn't require any
- * preparations steps, and functional NOOP may introduce unnecessary performance
- * drop.
- *
- * Generally this is not a good idea to turn it on globally and didn't should
- * be used if behavior of tx_preparation can change.
- */
-
-static inline uint16_t
-rte_eth_tx_prepare(__rte_unused uint16_t port_id,
- __rte_unused uint16_t queue_id,
- __rte_unused struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
-{
- return nb_pkts;
-}
-
-#endif
-
-typedef void (*buffer_tx_error_fn)(struct rte_mbuf **unsent, uint16_t count,
- void *userdata);
-
-/**
- * Structure used to buffer packets for future TX
- * Used by APIs rte_eth_tx_buffer and rte_eth_tx_buffer_flush
- */
-struct rte_eth_dev_tx_buffer {
- buffer_tx_error_fn error_callback;
- void *error_userdata;
- uint16_t size; /**< Size of buffer for buffered tx */
- uint16_t length; /**< Number of packets in the array */
- struct rte_mbuf *pkts[];
- /**< Pending packets to be sent on explicit flush or when full */
-};
-
-/**
- * Calculate the size of the tx buffer.
- *
- * @param sz
- * Number of stored packets.
- */
-#define RTE_ETH_TX_BUFFER_SIZE(sz) \
- (sizeof(struct rte_eth_dev_tx_buffer) + (sz) * sizeof(struct rte_mbuf *))
-
-/**
- * Initialize default values for buffered transmitting
- *
- * @param buffer
- * Tx buffer to be initialized.
- * @param size
- * Buffer size
- * @return
- * 0 if no error
- */
-int
-rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size);
-
-/**
- * Send any packets queued up for transmission on a port and HW queue
- *
- * This causes an explicit flush of packets previously buffered via the
- * rte_eth_tx_buffer() function. It returns the number of packets successfully
- * sent to the NIC, and calls the error callback for any unsent packets. Unless
- * explicitly set up otherwise, the default callback simply frees the unsent
- * packets back to the owning mempool.
- *
- * @param port_id
- * The port identifier of the Ethernet device.
- * @param queue_id
- * The index of the transmit queue through which output packets must be
- * sent.
- * The value must be in the range [0, nb_tx_queue - 1] previously supplied
- * to rte_eth_dev_configure().
- * @param buffer
- * Buffer of packets to be transmit.
- * @return
- * The number of packets successfully sent to the Ethernet device. The error
- * callback is called for any packets which could not be sent.
- */
-static inline uint16_t
-rte_eth_tx_buffer_flush(uint16_t port_id, uint16_t queue_id,
- struct rte_eth_dev_tx_buffer *buffer)
-{
- uint16_t sent;
- uint16_t to_send = buffer->length;
-
- if (to_send == 0)
- return 0;
-
- sent = rte_eth_tx_burst(port_id, queue_id, buffer->pkts, to_send);
-
- buffer->length = 0;
-
- /* All packets sent, or to be dealt with by callback below */
- if (unlikely(sent != to_send))
- buffer->error_callback(&buffer->pkts[sent], to_send - sent,
- buffer->error_userdata);
-
- return sent;
-}
-
-/**
- * Buffer a single packet for future transmission on a port and queue
- *
- * This function takes a single mbuf/packet and buffers it for later
- * transmission on the particular port and queue specified. Once the buffer is
- * full of packets, an attempt will be made to transmit all the buffered
- * packets. In case of error, where not all packets can be transmitted, a
- * callback is called with the unsent packets as a parameter. If no callback
- * is explicitly set up, the unsent packets are just freed back to the owning
- * mempool. The function returns the number of packets actually sent i.e.
- * 0 if no buffer flush occurred, otherwise the number of packets successfully
- * flushed
- *
- * @param port_id
- * The port identifier of the Ethernet device.
- * @param queue_id
- * The index of the transmit queue through which output packets must be
- * sent.
- * The value must be in the range [0, nb_tx_queue - 1] previously supplied
- * to rte_eth_dev_configure().
- * @param buffer
- * Buffer used to collect packets to be sent.
- * @param tx_pkt
- * Pointer to the packet mbuf to be sent.
- * @return
- * 0 = packet has been buffered for later transmission
- * N > 0 = packet has been buffered, and the buffer was subsequently flushed,
- * causing N packets to be sent, and the error callback to be called for
- * the rest.
- */
-static __rte_always_inline uint16_t
-rte_eth_tx_buffer(uint16_t port_id, uint16_t queue_id,
- struct rte_eth_dev_tx_buffer *buffer, struct rte_mbuf *tx_pkt)
-{
- buffer->pkts[buffer->length++] = tx_pkt;
- if (buffer->length < buffer->size)
- return 0;
-
- return rte_eth_tx_buffer_flush(port_id, queue_id, buffer);
-}
-
-/**
- * Configure a callback for buffered packets which cannot be sent
- *
- * Register a specific callback to be called when an attempt is made to send
- * all packets buffered on an ethernet port, but not all packets can
- * successfully be sent. The callback registered here will be called only
- * from calls to rte_eth_tx_buffer() and rte_eth_tx_buffer_flush() APIs.
- * The default callback configured for each queue by default just frees the
- * packets back to the calling mempool. If additional behaviour is required,
- * for example, to count dropped packets, or to retry transmission of packets
- * which cannot be sent, this function should be used to register a suitable
- * callback function to implement the desired behaviour.
- * The example callback "rte_eth_count_unsent_packet_callback()" is also
- * provided as reference.
- *
- * @param buffer
- * The port identifier of the Ethernet device.
- * @param callback
- * The function to be used as the callback.
- * @param userdata
- * Arbitrary parameter to be passed to the callback function
- * @return
- * 0 on success, or -1 on error with rte_errno set appropriately
- */
-int
-rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
- buffer_tx_error_fn callback, void *userdata);
-
-/**
- * Callback function for silently dropping unsent buffered packets.
- *
- * This function can be passed to rte_eth_tx_buffer_set_err_callback() to
- * adjust the default behavior when buffered packets cannot be sent. This
- * function drops any unsent packets silently and is used by tx buffered
- * operations as default behavior.
- *
- * NOTE: this function should not be called directly, instead it should be used
- * as a callback for packet buffering.
- *
- * NOTE: when configuring this function as a callback with
- * rte_eth_tx_buffer_set_err_callback(), the final, userdata parameter
- * should point to an uint64_t value.
- *
- * @param pkts
- * The previously buffered packets which could not be sent
- * @param unsent
- * The number of unsent packets in the pkts array
- * @param userdata
- * Not used
- */
-void
-rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
- void *userdata);
-
-/**
- * Callback function for tracking unsent buffered packets.
- *
- * This function can be passed to rte_eth_tx_buffer_set_err_callback() to
- * adjust the default behavior when buffered packets cannot be sent. This
- * function drops any unsent packets, but also updates a user-supplied counter
- * to track the overall number of packets dropped. The counter should be an
- * uint64_t variable.
- *
- * NOTE: this function should not be called directly, instead it should be used
- * as a callback for packet buffering.
- *
- * NOTE: when configuring this function as a callback with
- * rte_eth_tx_buffer_set_err_callback(), the final, userdata parameter
- * should point to an uint64_t value.
- *
- * @param pkts
- * The previously buffered packets which could not be sent
- * @param unsent
- * The number of unsent packets in the pkts array
- * @param userdata
- * Pointer to an uint64_t value, which will be incremented by unsent
- */
-void
-rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
- void *userdata);
-
-/**
- * Request the driver to free mbufs currently cached by the driver. The
- * driver will only free the mbuf if it is no longer in use. It is the
- * application's responsibity to ensure rte_eth_tx_buffer_flush(..) is
- * called if needed.
- *
- * @param port_id
- * The port identifier of the Ethernet device.
- * @param queue_id
- * The index of the transmit queue through which output packets must be
- * sent.
- * The value must be in the range [0, nb_tx_queue - 1] previously supplied
- * to rte_eth_dev_configure().
- * @param free_cnt
- * Maximum number of packets to free. Use 0 to indicate all possible packets
- * should be freed. Note that a packet may be using multiple mbufs.
- * @return
- * Failure: < 0
- * -ENODEV: Invalid interface
- * -ENOTSUP: Driver does not support function
- * Success: >= 0
- * 0-n: Number of packets freed. More packets may still remain in ring that
- * are in use.
- */
-int
-rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt);
-
-/**
- * The eth device event type for interrupt, and maybe others in the future.
- */
-enum rte_eth_event_type {
- RTE_ETH_EVENT_UNKNOWN, /**< unknown event type */
- RTE_ETH_EVENT_INTR_LSC, /**< lsc interrupt event */
- RTE_ETH_EVENT_QUEUE_STATE,
- /**< queue state event (enabled/disabled) */
- RTE_ETH_EVENT_INTR_RESET,
- /**< reset interrupt event, sent to VF on PF reset */
- RTE_ETH_EVENT_VF_MBOX, /**< message from the VF received by PF */
- RTE_ETH_EVENT_MACSEC, /**< MACsec offload related event */
- RTE_ETH_EVENT_INTR_RMV, /**< device removal event */
- RTE_ETH_EVENT_MAX /**< max value of this enum */
-};
-
-typedef int (*rte_eth_dev_cb_fn)(uint16_t port_id,
- enum rte_eth_event_type event, void *cb_arg, void *ret_param);
-/**< user application callback to be registered for interrupts */
-
-
-
-/**
- * Register a callback function for specific port id.
- *
- * @param port_id
- * Port id.
- * @param event
- * Event interested.
- * @param cb_fn
- * User supplied callback function to be called.
- * @param cb_arg
- * Pointer to the parameters for the registered callback.
- *
- * @return
- * - On success, zero.
- * - On failure, a negative value.
- */
-int rte_eth_dev_callback_register(uint16_t port_id,
- enum rte_eth_event_type event,
- rte_eth_dev_cb_fn cb_fn, void *cb_arg);
-
-/**
- * Unregister a callback function for specific port id.
- *
- * @param port_id
- * Port id.
- * @param event
- * Event interested.
- * @param cb_fn
- * User supplied callback function to be called.
- * @param cb_arg
- * Pointer to the parameters for the registered callback. -1 means to
- * remove all for the same callback address and same event.
- *
- * @return
- * - On success, zero.
- * - On failure, a negative value.
- */
-int rte_eth_dev_callback_unregister(uint16_t port_id,
- enum rte_eth_event_type event,
- rte_eth_dev_cb_fn cb_fn, void *cb_arg);
-
-/**
- * @internal Executes all the user application registered callbacks for
- * the specific device. It is for DPDK internal user only. User
- * application should not call it directly.
- *
- * @param dev
- * Pointer to struct rte_eth_dev.
- * @param event
- * Eth device interrupt event type.
- * @param cb_arg
- * callback parameter.
- * @param ret_param
- * To pass data back to user application.
- * This allows the user application to decide if a particular function
- * is permitted or not.
- *
- * @return
- * int
- */
-int _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
- enum rte_eth_event_type event, void *cb_arg, void *ret_param);
-
-/**
- * When there is no rx packet coming in Rx Queue for a long time, we can
- * sleep lcore related to RX Queue for power saving, and enable rx interrupt
- * to be triggered when Rx packet arrives.
- *
- * The rte_eth_dev_rx_intr_enable() function enables rx queue
- * interrupt on specific rx queue of a port.
- *
- * @param port_id
- * The port identifier of the Ethernet device.
- * @param queue_id
- * The index of the receive queue from which to retrieve input packets.
- * The value must be in the range [0, nb_rx_queue - 1] previously supplied
- * to rte_eth_dev_configure().
- * @return
- * - (0) if successful.
- * - (-ENOTSUP) if underlying hardware OR driver doesn't support
- * that operation.
- * - (-ENODEV) if *port_id* invalid.