#include "rte_kvargs.h"
#include "rte_crypto.h"
-#include "rte_dev.h"
#include <rte_common.h>
-#include <rte_config.h>
#include <rte_rcu_qsbr.h>
#include "rte_cryptodev_trace_fp.h"
*/
#define RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES RTE_BIT32(0)
#define RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES RTE_BIT32(1)
+#define RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_1_MEGABYTES RTE_BIT32(2)
/**
* Symmetric Crypto Capability
/**< Transform type: RSA/MODEXP/DH/DSA/MODINV */
uint32_t op_types;
- /**< bitmask for supported rte_crypto_asym_op_type */
+ /**<
+ * Bitmask for supported rte_crypto_asym_op_type or
+ * rte_crypto_asym_ke_type. Which enum is used is determined
+ * by the rte_crypto_asym_xform_type. For key exchange algorithms
+ * like Diffie-Hellman it is rte_crypto_asym_ke_type, for others
+ * it is rte_crypto_asym_op_type.
+ */
__extension__
union {
/**< Support operations on multiple data-units message */
#define RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY (1ULL << 26)
/**< Support wrapped key in cipher xform */
+#define RTE_CRYPTODEV_FF_SECURITY_INNER_CSUM (1ULL << 27)
+/**< Support inner checksum computation/verification */
/**
* Get the name of a crypto device feature flag
enum rte_cryptodev_event_type event,
rte_cryptodev_cb_fn cb_fn, void *cb_arg);
-typedef uint16_t (*dequeue_pkt_burst_t)(void *qp,
- struct rte_crypto_op **ops, uint16_t nb_ops);
-/**< Dequeue processed packets from queue pair of a device. */
-
-typedef uint16_t (*enqueue_pkt_burst_t)(void *qp,
- struct rte_crypto_op **ops, uint16_t nb_ops);
-/**< Enqueue packets for processing on queue pair of a device. */
-
-
-
-
struct rte_cryptodev_callback;
/** Structure to keep track of registered callbacks */
-TAILQ_HEAD(rte_cryptodev_cb_list, rte_cryptodev_callback);
+RTE_TAILQ_HEAD(rte_cryptodev_cb_list, rte_cryptodev_callback);
/**
* Structure used to hold information about the callbacks to be called for a
/**< RCU QSBR variable per queue pair */
};
-/** The data structure associated with each crypto device. */
-struct rte_cryptodev {
- dequeue_pkt_burst_t dequeue_burst;
- /**< Pointer to PMD receive function. */
- enqueue_pkt_burst_t enqueue_burst;
- /**< Pointer to PMD transmit function. */
-
- struct rte_cryptodev_data *data;
- /**< Pointer to device data */
- struct rte_cryptodev_ops *dev_ops;
- /**< Functions exported by PMD */
- uint64_t feature_flags;
- /**< Feature flags exposes HW/SW features for the given device */
- struct rte_device *device;
- /**< Backing device */
-
- uint8_t driver_id;
- /**< Crypto driver identifier*/
-
- struct rte_cryptodev_cb_list link_intr_cbs;
- /**< User application callback for interrupts if present */
-
- void *security_ctx;
- /**< Context for security ops */
-
- __extension__
- uint8_t attached : 1;
- /**< Flag indicating the device is attached */
-
- struct rte_cryptodev_cb_rcu *enq_cbs;
- /**< User application callback for pre enqueue processing */
-
- struct rte_cryptodev_cb_rcu *deq_cbs;
- /**< User application callback for post dequeue processing */
-} __rte_cache_aligned;
-
void *
rte_cryptodev_get_sec_ctx(uint8_t dev_id);
-/**
- *
- * The data part, with no function pointers, associated with each device.
- *
- * This structure is safe to place in shared memory to be common among
- * different processes in a multi-process configuration.
- */
-struct rte_cryptodev_data {
- uint8_t dev_id;
- /**< Device ID for this instance */
- uint8_t socket_id;
- /**< Socket ID where memory is allocated */
- char name[RTE_CRYPTODEV_NAME_MAX_LEN];
- /**< Unique identifier name */
-
- __extension__
- uint8_t dev_started : 1;
- /**< Device state: STARTED(1)/STOPPED(0) */
-
- struct rte_mempool *session_pool;
- /**< Session memory pool */
- void **queue_pairs;
- /**< Array of pointers to queue pairs. */
- uint16_t nb_queue_pairs;
- /**< Number of device queue pairs. */
-
- void *dev_private;
- /**< PMD-specific private data */
-} __rte_cache_aligned;
-
-extern struct rte_cryptodev *rte_cryptodevs;
-/**
- *
- * Dequeue a burst of processed crypto operations from a queue on the crypto
- * device. The dequeued operation are stored in *rte_crypto_op* structures
- * whose pointers are supplied in the *ops* array.
- *
- * The rte_cryptodev_dequeue_burst() function returns the number of ops
- * actually dequeued, which is the number of *rte_crypto_op* data structures
- * effectively supplied into the *ops* array.
- *
- * A return value equal to *nb_ops* indicates that the queue contained
- * at least *nb_ops* operations, and this is likely to signify that other
- * processed operations remain in the devices output queue. Applications
- * implementing a "retrieve as many processed operations as possible" policy
- * can check this specific case and keep invoking the
- * rte_cryptodev_dequeue_burst() function until a value less than
- * *nb_ops* is returned.
- *
- * The rte_cryptodev_dequeue_burst() function does not provide any error
- * notification to avoid the corresponding overhead.
- *
- * @param dev_id The symmetric crypto device identifier
- * @param qp_id The index of the queue pair from which to
- * retrieve processed packets. The value must be
- * in the range [0, nb_queue_pair - 1] previously
- * supplied to rte_cryptodev_configure().
- * @param ops The address of an array of pointers to
- * *rte_crypto_op* structures that must be
- * large enough to store *nb_ops* pointers in it.
- * @param nb_ops The maximum number of operations to dequeue.
- *
- * @return
- * - The number of operations actually dequeued, which is the number
- * of pointers to *rte_crypto_op* structures effectively supplied to the
- * *ops* array.
- */
-static inline uint16_t
-rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id,
- struct rte_crypto_op **ops, uint16_t nb_ops)
-{
- struct rte_cryptodev *dev = &rte_cryptodevs[dev_id];
-
- rte_cryptodev_trace_dequeue_burst(dev_id, qp_id, (void **)ops, nb_ops);
- nb_ops = (*dev->dequeue_burst)
- (dev->data->queue_pairs[qp_id], ops, nb_ops);
-#ifdef RTE_CRYPTO_CALLBACKS
- if (unlikely(dev->deq_cbs != NULL)) {
- struct rte_cryptodev_cb_rcu *list;
- struct rte_cryptodev_cb *cb;
-
- /* __ATOMIC_RELEASE memory order was used when the
- * call back was inserted into the list.
- * Since there is a clear dependency between loading
- * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
- * not required.
- */
- list = &dev->deq_cbs[qp_id];
- rte_rcu_qsbr_thread_online(list->qsbr, 0);
- cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
-
- while (cb != NULL) {
- nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
- cb->arg);
- cb = cb->next;
- };
-
- rte_rcu_qsbr_thread_offline(list->qsbr, 0);
- }
-#endif
- return nb_ops;
-}
-
-/**
- * Enqueue a burst of operations for processing on a crypto device.
- *
- * The rte_cryptodev_enqueue_burst() function is invoked to place
- * crypto operations on the queue *qp_id* of the device designated by
- * its *dev_id*.
- *
- * The *nb_ops* parameter is the number of operations to process which are
- * supplied in the *ops* array of *rte_crypto_op* structures.
- *
- * The rte_cryptodev_enqueue_burst() function returns the number of
- * operations it actually enqueued for processing. A return value equal to
- * *nb_ops* means that all packets have been enqueued.
- *
- * @param dev_id The identifier of the device.
- * @param qp_id The index of the queue pair which packets are
- * to be enqueued for processing. The value
- * must be in the range [0, nb_queue_pairs - 1]
- * previously supplied to
- * *rte_cryptodev_configure*.
- * @param ops The address of an array of *nb_ops* pointers
- * to *rte_crypto_op* structures which contain
- * the crypto operations to be processed.
- * @param nb_ops The number of operations to process.
- *
- * @return
- * The number of operations actually enqueued on the crypto device. The return
- * value can be less than the value of the *nb_ops* parameter when the
- * crypto devices queue is full or if invalid parameters are specified in
- * a *rte_crypto_op*.
- */
-static inline uint16_t
-rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id,
- struct rte_crypto_op **ops, uint16_t nb_ops)
-{
- struct rte_cryptodev *dev = &rte_cryptodevs[dev_id];
-
-#ifdef RTE_CRYPTO_CALLBACKS
- if (unlikely(dev->enq_cbs != NULL)) {
- struct rte_cryptodev_cb_rcu *list;
- struct rte_cryptodev_cb *cb;
-
- /* __ATOMIC_RELEASE memory order was used when the
- * call back was inserted into the list.
- * Since there is a clear dependency between loading
- * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
- * not required.
- */
- list = &dev->enq_cbs[qp_id];
- rte_rcu_qsbr_thread_online(list->qsbr, 0);
- cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
-
- while (cb != NULL) {
- nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
- cb->arg);
- cb = cb->next;
- };
-
- rte_rcu_qsbr_thread_offline(list->qsbr, 0);
- }
-#endif
-
- rte_cryptodev_trace_enqueue_burst(dev_id, qp_id, (void **)ops, nb_ops);
- return (*dev->enqueue_burst)(
- dev->data->queue_pairs[qp_id], ops, nb_ops);
-}
-
-
/** Cryptodev symmetric crypto session
* Each session is derived from a fixed xform chain. Therefore each session
* has a fixed algo, key, op-type, digest_len etc.
/**< Driver specific session material, variable size */
};
-/** Cryptodev asymmetric crypto session */
-struct rte_cryptodev_asym_session {
- __extension__ void *sess_private_data[0];
- /**< Private asymmetric session material */
-};
-
/**
* Create a symmetric session mempool.
*
uint32_t elt_size, uint32_t cache_size, uint16_t priv_size,
int socket_id);
+/**
+ * Create an asymmetric session mempool.
+ *
+ * @param name
+ * The unique mempool name.
+ * @param nb_elts
+ * The number of elements in the mempool.
+ * @param cache_size
+ * The number of per-lcore cache elements
+ * @param user_data_size
+ * The size of user data to be placed after session private data.
+ * @param socket_id
+ * The *socket_id* argument is the socket identifier in the case of
+ * NUMA. The value can be *SOCKET_ID_ANY* if there is no NUMA
+ * constraint for the reserved zone.
+ *
+ * @return
+ * - On success return mempool
+ * - On failure returns NULL
+ */
+__rte_experimental
+struct rte_mempool *
+rte_cryptodev_asym_session_pool_create(const char *name, uint32_t nb_elts,
+ uint32_t cache_size, uint16_t user_data_size, int socket_id);
+
/**
* Create symmetric crypto session header (generic with no private data)
*
rte_cryptodev_sym_session_create(struct rte_mempool *mempool);
/**
- * Create asymmetric crypto session header (generic with no private data)
+ * Create and initialise an asymmetric crypto session structure.
+ * Calls the PMD to configure the private session data.
+ *
+ * @param dev_id ID of device that we want the session to be used on
+ * @param xforms Asymmetric crypto transform operations to apply on flow
+ * processed with this session
+ * @param mp mempool to allocate asymmetric session
+ * objects from
+ * @param session void ** for session to be used
*
- * @param mempool mempool to allocate asymmetric session
- * objects from
* @return
- * - On success return pointer to asym-session
- * - On failure returns NULL
+ * - 0 on success.
+ * - -EINVAL on invalid arguments.
+ * - -ENOMEM on memory error for session allocation.
+ * - -ENOTSUP if device doesn't support session configuration.
*/
__rte_experimental
-struct rte_cryptodev_asym_session *
-rte_cryptodev_asym_session_create(struct rte_mempool *mempool);
+int
+rte_cryptodev_asym_session_create(uint8_t dev_id,
+ struct rte_crypto_asym_xform *xforms, struct rte_mempool *mp,
+ void **session);
/**
* Frees symmetric crypto session header, after checking that all
rte_cryptodev_sym_session_free(struct rte_cryptodev_sym_session *sess);
/**
- * Frees asymmetric crypto session header, after checking that all
- * the device private data has been freed, returning it
- * to its original mempool.
+ * Clears and frees asymmetric crypto session header and private data,
+ * returning it to its original mempool.
*
+ * @param dev_id ID of device that uses the asymmetric session.
* @param sess Session header to be freed.
*
* @return
* - 0 if successful.
- * - -EINVAL if session is NULL.
- * - -EBUSY if not all device private data has been freed.
+ * - -EINVAL if device is invalid or session is NULL.
*/
__rte_experimental
int
-rte_cryptodev_asym_session_free(struct rte_cryptodev_asym_session *sess);
+rte_cryptodev_asym_session_free(uint8_t dev_id, void *sess);
/**
* Fill out private data for the device id, based on its device type.
struct rte_crypto_sym_xform *xforms,
struct rte_mempool *mempool);
-/**
- * Initialize asymmetric session on a device with specific asymmetric xform
- *
- * @param dev_id ID of device that we want the session to be used on
- * @param sess Session to be set up on a device
- * @param xforms Asymmetric crypto transform operations to apply on flow
- * processed with this session
- * @param mempool Mempool to be used for internal allocation.
- *
- * @return
- * - On success, zero.
- * - -EINVAL if input parameters are invalid.
- * - -ENOTSUP if crypto device does not support the crypto transform.
- * - -ENOMEM if the private session could not be allocated.
- */
-__rte_experimental
-int
-rte_cryptodev_asym_session_init(uint8_t dev_id,
- struct rte_cryptodev_asym_session *sess,
- struct rte_crypto_asym_xform *xforms,
- struct rte_mempool *mempool);
-
/**
* Frees private data for the device id, based on its device type,
* returning it to its mempool. It is the application's responsibility
rte_cryptodev_sym_session_clear(uint8_t dev_id,
struct rte_cryptodev_sym_session *sess);
-/**
- * Frees resources held by asymmetric session during rte_cryptodev_session_init
- *
- * @param dev_id ID of device that uses the asymmetric session.
- * @param sess Asymmetric session setup on device using
- * rte_cryptodev_session_init
- * @return
- * - 0 if successful.
- * - -EINVAL if device is invalid or session is NULL.
- */
-__rte_experimental
-int
-rte_cryptodev_asym_session_clear(uint8_t dev_id,
- struct rte_cryptodev_asym_session *sess);
-
/**
* Get the size of the header session, for all registered drivers excluding
* the user data size.
struct rte_cryptodev_sym_session *sess);
/**
- * Get the size of the asymmetric session header, for all registered drivers.
+ * Get the size of the asymmetric session header.
*
* @return
* Size of the asymmetric header session.
unsigned int
rte_cryptodev_asym_get_private_session_size(uint8_t dev_id);
+/**
+ * Validate if the crypto device index is valid attached crypto device.
+ *
+ * @param dev_id Crypto device index.
+ *
+ * @return
+ * - If the device index is valid (1) or not (0).
+ */
+unsigned int
+rte_cryptodev_is_valid_dev(uint8_t dev_id);
+
/**
* Provide driver identifier.
*
rte_cryptodev_sym_session_get_user_data(
struct rte_cryptodev_sym_session *sess);
+/**
+ * Store user data in an asymmetric session.
+ *
+ * @param sess Session pointer allocated by
+ * *rte_cryptodev_asym_session_create*.
+ * @param data Pointer to the user data.
+ * @param size Size of the user data.
+ *
+ * @return
+ * - On success, zero.
+ * - -EINVAL if the session pointer is invalid.
+ * - -ENOMEM if the available user data size is smaller than the size parameter.
+ */
+__rte_experimental
+int
+rte_cryptodev_asym_session_set_user_data(void *sess, void *data, uint16_t size);
+
+/**
+ * Get user data stored in an asymmetric session.
+ *
+ * @param sess Session pointer allocated by
+ * *rte_cryptodev_asym_session_create*.
+ *
+ * @return
+ * - On success return pointer to user data.
+ * - On failure returns NULL.
+ */
+__rte_experimental
+void *
+rte_cryptodev_asym_session_get_user_data(void *sess);
+
/**
* Perform actual crypto processing (encrypt/digest or auth/decrypt)
* on user provided data.
int
rte_cryptodev_get_raw_dp_ctx_size(uint8_t dev_id);
+/**
+ * Set session event meta data
+ *
+ * @param dev_id The device identifier.
+ * @param sess Crypto or security session.
+ * @param op_type Operation type.
+ * @param sess_type Session type.
+ * @param ev_mdata Pointer to the event crypto meta data
+ * (aka *union rte_event_crypto_metadata*)
+ * @param size Size of ev_mdata.
+ *
+ * @return
+ * - On success, zero.
+ * - On failure, a negative value.
+ */
+__rte_experimental
+int
+rte_cryptodev_session_event_mdata_set(uint8_t dev_id, void *sess,
+ enum rte_crypto_op_type op_type,
+ enum rte_crypto_op_sess_type sess_type,
+ void *ev_mdata, uint16_t size);
+
/**
* Union of different crypto session types, including session-less xform
* pointer.
uint16_t qp_id,
struct rte_cryptodev_cb *cb);
+#include <rte_cryptodev_core.h>
+/**
+ *
+ * Dequeue a burst of processed crypto operations from a queue on the crypto
+ * device. The dequeued operation are stored in *rte_crypto_op* structures
+ * whose pointers are supplied in the *ops* array.
+ *
+ * The rte_cryptodev_dequeue_burst() function returns the number of ops
+ * actually dequeued, which is the number of *rte_crypto_op* data structures
+ * effectively supplied into the *ops* array.
+ *
+ * A return value equal to *nb_ops* indicates that the queue contained
+ * at least *nb_ops* operations, and this is likely to signify that other
+ * processed operations remain in the devices output queue. Applications
+ * implementing a "retrieve as many processed operations as possible" policy
+ * can check this specific case and keep invoking the
+ * rte_cryptodev_dequeue_burst() function until a value less than
+ * *nb_ops* is returned.
+ *
+ * The rte_cryptodev_dequeue_burst() function does not provide any error
+ * notification to avoid the corresponding overhead.
+ *
+ * @param dev_id The symmetric crypto device identifier
+ * @param qp_id The index of the queue pair from which to
+ * retrieve processed packets. The value must be
+ * in the range [0, nb_queue_pair - 1] previously
+ * supplied to rte_cryptodev_configure().
+ * @param ops The address of an array of pointers to
+ * *rte_crypto_op* structures that must be
+ * large enough to store *nb_ops* pointers in it.
+ * @param nb_ops The maximum number of operations to dequeue.
+ *
+ * @return
+ * - The number of operations actually dequeued, which is the number
+ * of pointers to *rte_crypto_op* structures effectively supplied to the
+ * *ops* array.
+ */
+static inline uint16_t
+rte_cryptodev_dequeue_burst(uint8_t dev_id, uint16_t qp_id,
+ struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+ const struct rte_crypto_fp_ops *fp_ops;
+ void *qp;
+
+ rte_cryptodev_trace_dequeue_burst(dev_id, qp_id, (void **)ops, nb_ops);
+
+ fp_ops = &rte_crypto_fp_ops[dev_id];
+ qp = fp_ops->qp.data[qp_id];
+
+ nb_ops = fp_ops->dequeue_burst(qp, ops, nb_ops);
+
+#ifdef RTE_CRYPTO_CALLBACKS
+ if (unlikely(fp_ops->qp.deq_cb != NULL)) {
+ struct rte_cryptodev_cb_rcu *list;
+ struct rte_cryptodev_cb *cb;
+
+ /* __ATOMIC_RELEASE memory order was used when the
+ * call back was inserted into the list.
+ * Since there is a clear dependency between loading
+ * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+ * not required.
+ */
+ list = &fp_ops->qp.deq_cb[qp_id];
+ rte_rcu_qsbr_thread_online(list->qsbr, 0);
+ cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+
+ while (cb != NULL) {
+ nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
+ cb->arg);
+ cb = cb->next;
+ };
+
+ rte_rcu_qsbr_thread_offline(list->qsbr, 0);
+ }
+#endif
+ return nb_ops;
+}
+
+/**
+ * Enqueue a burst of operations for processing on a crypto device.
+ *
+ * The rte_cryptodev_enqueue_burst() function is invoked to place
+ * crypto operations on the queue *qp_id* of the device designated by
+ * its *dev_id*.
+ *
+ * The *nb_ops* parameter is the number of operations to process which are
+ * supplied in the *ops* array of *rte_crypto_op* structures.
+ *
+ * The rte_cryptodev_enqueue_burst() function returns the number of
+ * operations it actually enqueued for processing. A return value equal to
+ * *nb_ops* means that all packets have been enqueued.
+ *
+ * @param dev_id The identifier of the device.
+ * @param qp_id The index of the queue pair which packets are
+ * to be enqueued for processing. The value
+ * must be in the range [0, nb_queue_pairs - 1]
+ * previously supplied to
+ * *rte_cryptodev_configure*.
+ * @param ops The address of an array of *nb_ops* pointers
+ * to *rte_crypto_op* structures which contain
+ * the crypto operations to be processed.
+ * @param nb_ops The number of operations to process.
+ *
+ * @return
+ * The number of operations actually enqueued on the crypto device. The return
+ * value can be less than the value of the *nb_ops* parameter when the
+ * crypto devices queue is full or if invalid parameters are specified in
+ * a *rte_crypto_op*.
+ */
+static inline uint16_t
+rte_cryptodev_enqueue_burst(uint8_t dev_id, uint16_t qp_id,
+ struct rte_crypto_op **ops, uint16_t nb_ops)
+{
+ const struct rte_crypto_fp_ops *fp_ops;
+ void *qp;
+
+ fp_ops = &rte_crypto_fp_ops[dev_id];
+ qp = fp_ops->qp.data[qp_id];
+#ifdef RTE_CRYPTO_CALLBACKS
+ if (unlikely(fp_ops->qp.enq_cb != NULL)) {
+ struct rte_cryptodev_cb_rcu *list;
+ struct rte_cryptodev_cb *cb;
+
+ /* __ATOMIC_RELEASE memory order was used when the
+ * call back was inserted into the list.
+ * Since there is a clear dependency between loading
+ * cb and cb->fn/cb->next, __ATOMIC_ACQUIRE memory order is
+ * not required.
+ */
+ list = &fp_ops->qp.enq_cb[qp_id];
+ rte_rcu_qsbr_thread_online(list->qsbr, 0);
+ cb = __atomic_load_n(&list->next, __ATOMIC_RELAXED);
+
+ while (cb != NULL) {
+ nb_ops = cb->fn(dev_id, qp_id, ops, nb_ops,
+ cb->arg);
+ cb = cb->next;
+ };
+
+ rte_rcu_qsbr_thread_offline(list->qsbr, 0);
+ }
+#endif
+
+ rte_cryptodev_trace_enqueue_burst(dev_id, qp_id, (void **)ops, nb_ops);
+ return fp_ops->enqueue_burst(qp, ops, nb_ops);
+}
+
+
+
#ifdef __cplusplus
}
#endif
git.droids-corp.org / dpdk.git / blobdiff