Configure the load balanced (ldb) or directed (dir) port.
The consumer queue (CQ) and producer port (PP) are also
set up here.
Signed-off-by: Timothy McDaniel <timothy.mcdaniel@intel.com>
Reviewed-by: Gage Eads <gage.eads@intel.com>
the DLB2 does not limit the number of flows a queue can track. In the DLB2, all
load-balanced queues can use the full 16-bit flow ID range.
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events. Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size. DLB2 has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB2 device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB2 PMD, because
+the DLB2 does not limit the number of flows a queue can track. In the DLB2, all
+load-balanced queues can use the full 16-bit flow ID range.
+
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB2 ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB2 PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
Flow ID
~~~~~~~
return ret;
}
+static int
+dlb2_init_consume_qe(struct dlb2_port *qm_port, char *mz_name)
+{
+ struct dlb2_cq_pop_qe *qe;
+
+ qe = rte_zmalloc(mz_name,
+ DLB2_NUM_QES_PER_CACHE_LINE *
+ sizeof(struct dlb2_cq_pop_qe),
+ RTE_CACHE_LINE_SIZE);
+
+ if (qe == NULL) {
+ DLB2_LOG_ERR("dlb2: no memory for consume_qe\n");
+ return -ENOMEM;
+ }
+ qm_port->consume_qe = qe;
+
+ qe->qe_valid = 0;
+ qe->qe_frag = 0;
+ qe->qe_comp = 0;
+ qe->cq_token = 1;
+ /* Tokens value is 0-based; i.e. '0' returns 1 token, '1' returns 2,
+ * and so on.
+ */
+ qe->tokens = 0; /* set at run time */
+ qe->meas_lat = 0;
+ qe->no_dec = 0;
+ /* Completion IDs are disabled */
+ qe->cmp_id = 0;
+
+ return 0;
+}
+
+static int
+dlb2_init_int_arm_qe(struct dlb2_port *qm_port, char *mz_name)
+{
+ struct dlb2_enqueue_qe *qe;
+
+ qe = rte_zmalloc(mz_name,
+ DLB2_NUM_QES_PER_CACHE_LINE *
+ sizeof(struct dlb2_enqueue_qe),
+ RTE_CACHE_LINE_SIZE);
+
+ if (qe == NULL) {
+ DLB2_LOG_ERR("dlb2: no memory for complete_qe\n");
+ return -ENOMEM;
+ }
+ qm_port->int_arm_qe = qe;
+
+ /* V2 - INT ARM is CQ_TOKEN + FRAG */
+ qe->qe_valid = 0;
+ qe->qe_frag = 1;
+ qe->qe_comp = 0;
+ qe->cq_token = 1;
+ qe->meas_lat = 0;
+ qe->no_dec = 0;
+ /* Completion IDs are disabled */
+ qe->cmp_id = 0;
+
+ return 0;
+}
+
+static int
+dlb2_init_qe_mem(struct dlb2_port *qm_port, char *mz_name)
+{
+ int ret, sz;
+
+ sz = DLB2_NUM_QES_PER_CACHE_LINE * sizeof(struct dlb2_enqueue_qe);
+
+ qm_port->qe4 = rte_zmalloc(mz_name, sz, RTE_CACHE_LINE_SIZE);
+
+ if (qm_port->qe4 == NULL) {
+ DLB2_LOG_ERR("dlb2: no qe4 memory\n");
+ ret = -ENOMEM;
+ goto error_exit;
+ }
+
+ ret = dlb2_init_int_arm_qe(qm_port, mz_name);
+ if (ret < 0) {
+ DLB2_LOG_ERR("dlb2: dlb2_init_int_arm_qe ret=%d\n", ret);
+ goto error_exit;
+ }
+
+ ret = dlb2_init_consume_qe(qm_port, mz_name);
+ if (ret < 0) {
+ DLB2_LOG_ERR("dlb2: dlb2_init_consume_qe ret=%d\n", ret);
+ goto error_exit;
+ }
+
+ return 0;
+
+error_exit:
+
+ dlb2_free_qe_mem(qm_port);
+
+ return ret;
+}
+
+static int
+dlb2_hw_create_ldb_port(struct dlb2_eventdev *dlb2,
+ struct dlb2_eventdev_port *ev_port,
+ uint32_t dequeue_depth,
+ uint32_t enqueue_depth)
+{
+ struct dlb2_hw_dev *handle = &dlb2->qm_instance;
+ struct dlb2_create_ldb_port_args cfg = { {0} };
+ int ret;
+ struct dlb2_port *qm_port = NULL;
+ char mz_name[RTE_MEMZONE_NAMESIZE];
+ uint32_t qm_port_id;
+ uint16_t ldb_credit_high_watermark;
+ uint16_t dir_credit_high_watermark;
+
+ if (handle == NULL)
+ return -EINVAL;
+
+ if (dequeue_depth < DLB2_MIN_CQ_DEPTH) {
+ DLB2_LOG_ERR("dlb2: invalid enqueue_depth, must be at least %d\n",
+ DLB2_MIN_CQ_DEPTH);
+ return -EINVAL;
+ }
+
+ if (enqueue_depth < DLB2_MIN_ENQUEUE_DEPTH) {
+ DLB2_LOG_ERR("dlb2: invalid enqueue_depth, must be at least %d\n",
+ DLB2_MIN_ENQUEUE_DEPTH);
+ return -EINVAL;
+ }
+
+ rte_spinlock_lock(&handle->resource_lock);
+
+ /* We round up to the next power of 2 if necessary */
+ cfg.cq_depth = rte_align32pow2(dequeue_depth);
+ cfg.cq_depth_threshold = 1;
+
+ cfg.cq_history_list_size = DLB2_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT;
+
+ if (handle->cos_id == DLB2_COS_DEFAULT)
+ cfg.cos_id = 0;
+ else
+ cfg.cos_id = handle->cos_id;
+
+ cfg.cos_strict = 0;
+
+ /* User controls the LDB high watermark via enqueue depth. The DIR high
+ * watermark is equal, unless the directed credit pool is too small.
+ */
+ ldb_credit_high_watermark = enqueue_depth;
+
+ /* If there are no directed ports, the kernel driver will ignore this
+ * port's directed credit settings. Don't use enqueue_depth if it would
+ * require more directed credits than are available.
+ */
+ dir_credit_high_watermark =
+ RTE_MIN(enqueue_depth,
+ handle->cfg.num_dir_credits / dlb2->num_ports);
+
+ /* Per QM values */
+
+ ret = dlb2_iface_ldb_port_create(handle, &cfg, dlb2->poll_mode);
+ if (ret < 0) {
+ DLB2_LOG_ERR("dlb2: dlb2_ldb_port_create error, ret=%d (driver status: %s)\n",
+ ret, dlb2_error_strings[cfg.response.status]);
+ goto error_exit;
+ }
+
+ qm_port_id = cfg.response.id;
+
+ DLB2_LOG_DBG("dlb2: ev_port %d uses qm LB port %d <<<<<\n",
+ ev_port->id, qm_port_id);
+
+ qm_port = &ev_port->qm_port;
+ qm_port->ev_port = ev_port; /* back ptr */
+ qm_port->dlb2 = dlb2; /* back ptr */
+ /*
+ * Allocate and init local qe struct(s).
+ * Note: MOVDIR64 requires the enqueue QE (qe4) to be aligned.
+ */
+
+ snprintf(mz_name, sizeof(mz_name), "dlb2_ldb_port%d",
+ ev_port->id);
+
+ ret = dlb2_init_qe_mem(qm_port, mz_name);
+ if (ret < 0) {
+ DLB2_LOG_ERR("dlb2: init_qe_mem failed, ret=%d\n", ret);
+ goto error_exit;
+ }
+
+ qm_port->id = qm_port_id;
+
+ qm_port->cached_ldb_credits = 0;
+ qm_port->cached_dir_credits = 0;
+ /* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+ * the effective depth is smaller.
+ */
+ qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+ qm_port->cq_idx = 0;
+ qm_port->cq_idx_unmasked = 0;
+
+ if (dlb2->poll_mode == DLB2_CQ_POLL_MODE_SPARSE)
+ qm_port->cq_depth_mask = (qm_port->cq_depth * 4) - 1;
+ else
+ qm_port->cq_depth_mask = qm_port->cq_depth - 1;
+
+ qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+ /* starting value of gen bit - it toggles at wrap time */
+ qm_port->gen_bit = 1;
+
+ qm_port->int_armed = false;
+
+ /* Save off for later use in info and lookup APIs. */
+ qm_port->qid_mappings = &dlb2->qm_ldb_to_ev_queue_id[0];
+
+ qm_port->dequeue_depth = dequeue_depth;
+
+ qm_port->owed_tokens = 0;
+ qm_port->issued_releases = 0;
+
+ /* Save config message too. */
+ rte_memcpy(&qm_port->cfg.ldb, &cfg, sizeof(qm_port->cfg.ldb));
+
+ /* update state */
+ qm_port->state = PORT_STARTED; /* enabled at create time */
+ qm_port->config_state = DLB2_CONFIGURED;
+
+ qm_port->dir_credits = dir_credit_high_watermark;
+ qm_port->ldb_credits = ldb_credit_high_watermark;
+ qm_port->credit_pool[DLB2_DIR_QUEUE] = &dlb2->dir_credit_pool;
+ qm_port->credit_pool[DLB2_LDB_QUEUE] = &dlb2->ldb_credit_pool;
+
+ DLB2_LOG_DBG("dlb2: created ldb port %d, depth = %d, ldb credits=%d, dir credits=%d\n",
+ qm_port_id,
+ dequeue_depth,
+ qm_port->ldb_credits,
+ qm_port->dir_credits);
+
+ rte_spinlock_unlock(&handle->resource_lock);
+
+ return 0;
+
+error_exit:
+
+ if (qm_port)
+ dlb2_free_qe_mem(qm_port);
+
+ rte_spinlock_unlock(&handle->resource_lock);
+
+ DLB2_LOG_ERR("dlb2: create ldb port failed!\n");
+
+ return ret;
+}
+
+static void
+dlb2_port_link_teardown(struct dlb2_eventdev *dlb2,
+ struct dlb2_eventdev_port *ev_port)
+{
+ struct dlb2_eventdev_queue *ev_queue;
+ int i;
+
+ for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+ if (!ev_port->link[i].valid)
+ continue;
+
+ ev_queue = &dlb2->ev_queues[ev_port->link[i].queue_id];
+
+ ev_port->link[i].valid = false;
+ ev_port->num_links--;
+ ev_queue->num_links--;
+ }
+}
+
+static int
+dlb2_hw_create_dir_port(struct dlb2_eventdev *dlb2,
+ struct dlb2_eventdev_port *ev_port,
+ uint32_t dequeue_depth,
+ uint32_t enqueue_depth)
+{
+ struct dlb2_hw_dev *handle = &dlb2->qm_instance;
+ struct dlb2_create_dir_port_args cfg = { {0} };
+ int ret;
+ struct dlb2_port *qm_port = NULL;
+ char mz_name[RTE_MEMZONE_NAMESIZE];
+ uint32_t qm_port_id;
+ uint16_t ldb_credit_high_watermark;
+ uint16_t dir_credit_high_watermark;
+
+ if (dlb2 == NULL || handle == NULL)
+ return -EINVAL;
+
+ if (dequeue_depth < DLB2_MIN_CQ_DEPTH) {
+ DLB2_LOG_ERR("dlb2: invalid dequeue_depth, must be %d-%d\n",
+ DLB2_MIN_CQ_DEPTH, DLB2_MAX_INPUT_QUEUE_DEPTH);
+ return -EINVAL;
+ }
+
+ if (enqueue_depth < DLB2_MIN_ENQUEUE_DEPTH) {
+ DLB2_LOG_ERR("dlb2: invalid enqueue_depth, must be at least %d\n",
+ DLB2_MIN_ENQUEUE_DEPTH);
+ return -EINVAL;
+ }
+
+ rte_spinlock_lock(&handle->resource_lock);
+
+ /* Directed queues are configured at link time. */
+ cfg.queue_id = -1;
+
+ /* We round up to the next power of 2 if necessary */
+ cfg.cq_depth = rte_align32pow2(dequeue_depth);
+ cfg.cq_depth_threshold = 1;
+
+ /* User controls the LDB high watermark via enqueue depth. The DIR high
+ * watermark is equal, unless the directed credit pool is too small.
+ */
+ ldb_credit_high_watermark = enqueue_depth;
+
+ /* Don't use enqueue_depth if it would require more directed credits
+ * than are available.
+ */
+ dir_credit_high_watermark =
+ RTE_MIN(enqueue_depth,
+ handle->cfg.num_dir_credits / dlb2->num_ports);
+
+ /* Per QM values */
+
+ ret = dlb2_iface_dir_port_create(handle, &cfg, dlb2->poll_mode);
+ if (ret < 0) {
+ DLB2_LOG_ERR("dlb2: dlb2_dir_port_create error, ret=%d (driver status: %s)\n",
+ ret, dlb2_error_strings[cfg.response.status]);
+ goto error_exit;
+ }
+
+ qm_port_id = cfg.response.id;
+
+ DLB2_LOG_DBG("dlb2: ev_port %d uses qm DIR port %d <<<<<\n",
+ ev_port->id, qm_port_id);
+
+ qm_port = &ev_port->qm_port;
+ qm_port->ev_port = ev_port; /* back ptr */
+ qm_port->dlb2 = dlb2; /* back ptr */
+
+ /*
+ * Init local qe struct(s).
+ * Note: MOVDIR64 requires the enqueue QE to be aligned
+ */
+
+ snprintf(mz_name, sizeof(mz_name), "dlb2_dir_port%d",
+ ev_port->id);
+
+ ret = dlb2_init_qe_mem(qm_port, mz_name);
+
+ if (ret < 0) {
+ DLB2_LOG_ERR("dlb2: init_qe_mem failed, ret=%d\n", ret);
+ goto error_exit;
+ }
+
+ qm_port->id = qm_port_id;
+
+ qm_port->cached_ldb_credits = 0;
+ qm_port->cached_dir_credits = 0;
+ /* CQs with depth < 8 use an 8-entry queue, but withhold credits so
+ * the effective depth is smaller.
+ */
+ qm_port->cq_depth = cfg.cq_depth <= 8 ? 8 : cfg.cq_depth;
+ qm_port->cq_idx = 0;
+ qm_port->cq_idx_unmasked = 0;
+
+ if (dlb2->poll_mode == DLB2_CQ_POLL_MODE_SPARSE)
+ qm_port->cq_depth_mask = (cfg.cq_depth * 4) - 1;
+ else
+ qm_port->cq_depth_mask = cfg.cq_depth - 1;
+
+ qm_port->gen_bit_shift = __builtin_popcount(qm_port->cq_depth_mask);
+ /* starting value of gen bit - it toggles at wrap time */
+ qm_port->gen_bit = 1;
+
+ qm_port->int_armed = false;
+
+ /* Save off for later use in info and lookup APIs. */
+ qm_port->qid_mappings = &dlb2->qm_dir_to_ev_queue_id[0];
+
+ qm_port->dequeue_depth = dequeue_depth;
+
+ qm_port->owed_tokens = 0;
+ qm_port->issued_releases = 0;
+
+ /* Save config message too. */
+ rte_memcpy(&qm_port->cfg.dir, &cfg, sizeof(qm_port->cfg.dir));
+
+ /* update state */
+ qm_port->state = PORT_STARTED; /* enabled at create time */
+ qm_port->config_state = DLB2_CONFIGURED;
+
+ qm_port->dir_credits = dir_credit_high_watermark;
+ qm_port->ldb_credits = ldb_credit_high_watermark;
+ qm_port->credit_pool[DLB2_DIR_QUEUE] = &dlb2->dir_credit_pool;
+ qm_port->credit_pool[DLB2_LDB_QUEUE] = &dlb2->ldb_credit_pool;
+
+ DLB2_LOG_DBG("dlb2: created dir port %d, depth = %d cr=%d,%d\n",
+ qm_port_id,
+ dequeue_depth,
+ dir_credit_high_watermark,
+ ldb_credit_high_watermark);
+
+ rte_spinlock_unlock(&handle->resource_lock);
+
+ return 0;
+
+error_exit:
+
+ if (qm_port)
+ dlb2_free_qe_mem(qm_port);
+
+ rte_spinlock_unlock(&handle->resource_lock);
+
+ DLB2_LOG_ERR("dlb2: create dir port failed!\n");
+
+ return ret;
+}
+
+static int
+dlb2_eventdev_port_setup(struct rte_eventdev *dev,
+ uint8_t ev_port_id,
+ const struct rte_event_port_conf *port_conf)
+{
+ struct dlb2_eventdev *dlb2;
+ struct dlb2_eventdev_port *ev_port;
+ int ret;
+
+ if (dev == NULL || port_conf == NULL) {
+ DLB2_LOG_ERR("Null parameter\n");
+ return -EINVAL;
+ }
+
+ dlb2 = dlb2_pmd_priv(dev);
+
+ if (ev_port_id >= DLB2_MAX_NUM_PORTS)
+ return -EINVAL;
+
+ if (port_conf->dequeue_depth >
+ evdev_dlb2_default_info.max_event_port_dequeue_depth ||
+ port_conf->enqueue_depth >
+ evdev_dlb2_default_info.max_event_port_enqueue_depth)
+ return -EINVAL;
+
+ ev_port = &dlb2->ev_ports[ev_port_id];
+ /* configured? */
+ if (ev_port->setup_done) {
+ DLB2_LOG_ERR("evport %d is already configured\n", ev_port_id);
+ return -EINVAL;
+ }
+
+ ev_port->qm_port.is_directed = port_conf->event_port_cfg &
+ RTE_EVENT_PORT_CFG_SINGLE_LINK;
+
+ if (!ev_port->qm_port.is_directed) {
+ ret = dlb2_hw_create_ldb_port(dlb2,
+ ev_port,
+ port_conf->dequeue_depth,
+ port_conf->enqueue_depth);
+ if (ret < 0) {
+ DLB2_LOG_ERR("Failed to create the lB port ve portId=%d\n",
+ ev_port_id);
+
+ return ret;
+ }
+ } else {
+ ret = dlb2_hw_create_dir_port(dlb2,
+ ev_port,
+ port_conf->dequeue_depth,
+ port_conf->enqueue_depth);
+ if (ret < 0) {
+ DLB2_LOG_ERR("Failed to create the DIR port\n");
+ return ret;
+ }
+ }
+
+ /* Save off port config for reconfig */
+ ev_port->conf = *port_conf;
+
+ ev_port->id = ev_port_id;
+ ev_port->enq_configured = true;
+ ev_port->setup_done = true;
+ ev_port->inflight_max = port_conf->new_event_threshold;
+ ev_port->implicit_release = !(port_conf->event_port_cfg &
+ RTE_EVENT_PORT_CFG_DISABLE_IMPL_REL);
+ ev_port->outstanding_releases = 0;
+ ev_port->inflight_credits = 0;
+ ev_port->credit_update_quanta = RTE_LIBRTE_PMD_DLB2_SW_CREDIT_QUANTA;
+ ev_port->dlb2 = dlb2; /* reverse link */
+
+ /* Tear down pre-existing port->queue links */
+ if (dlb2->run_state == DLB2_RUN_STATE_STOPPED)
+ dlb2_port_link_teardown(dlb2, &dlb2->ev_ports[ev_port_id]);
+
+ dev->data->ports[ev_port_id] = &dlb2->ev_ports[ev_port_id];
+
+ return 0;
+}
+
static void
dlb2_entry_points_init(struct rte_eventdev *dev)
{
.queue_def_conf = dlb2_eventdev_queue_default_conf_get,
.queue_setup = dlb2_eventdev_queue_setup,
.port_def_conf = dlb2_eventdev_port_default_conf_get,
+ .port_setup = dlb2_eventdev_port_setup,
.dump = dlb2_eventdev_dump,
.xstats_get = dlb2_eventdev_xstats_get,
.xstats_get_names = dlb2_eventdev_xstats_get_names,
int (*dlb2_iface_get_sn_occupancy)(struct dlb2_hw_dev *handle,
struct dlb2_get_sn_occupancy_args *args);
+
+int (*dlb2_iface_ldb_port_create)(struct dlb2_hw_dev *handle,
+ struct dlb2_create_ldb_port_args *cfg,
+ enum dlb2_cq_poll_modes poll_mode);
+
+int (*dlb2_iface_dir_port_create)(struct dlb2_hw_dev *handle,
+ struct dlb2_create_dir_port_args *cfg,
+ enum dlb2_cq_poll_modes poll_mode);
extern int (*dlb2_iface_get_sn_occupancy)(struct dlb2_hw_dev *handle,
struct dlb2_get_sn_occupancy_args *args);
+
+extern int (*dlb2_iface_ldb_port_create)(struct dlb2_hw_dev *handle,
+ struct dlb2_create_ldb_port_args *cfg,
+ enum dlb2_cq_poll_modes poll_mode);
+
+extern int (*dlb2_iface_dir_port_create)(struct dlb2_hw_dev *handle,
+ struct dlb2_create_dir_port_args *cfg,
+ enum dlb2_cq_poll_modes poll_mode);
#endif /* _DLB2_IFACE_H_ */
return 0;
}
+
+static void dlb2_ldb_port_configure_pp(struct dlb2_hw *hw,
+ struct dlb2_hw_domain *domain,
+ struct dlb2_ldb_port *port,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ union dlb2_sys_ldb_pp2vas r0 = { {0} };
+ union dlb2_sys_ldb_pp_v r4 = { {0} };
+
+ r0.field.vas = domain->id.phys_id;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_LDB_PP2VAS(port->id.phys_id), r0.val);
+
+ if (vdev_req) {
+ union dlb2_sys_vf_ldb_vpp2pp r1 = { {0} };
+ union dlb2_sys_ldb_pp2vdev r2 = { {0} };
+ union dlb2_sys_vf_ldb_vpp_v r3 = { {0} };
+ unsigned int offs;
+ u32 virt_id;
+
+ /*
+ * DLB uses producer port address bits 17:12 to determine the
+ * producer port ID. In Scalable IOV mode, PP accesses come
+ * through the PF MMIO window for the physical producer port,
+ * so for translation purposes the virtual and physical port
+ * IDs are equal.
+ */
+ if (hw->virt_mode == DLB2_VIRT_SRIOV)
+ virt_id = port->id.virt_id;
+ else
+ virt_id = port->id.phys_id;
+
+ r1.field.pp = port->id.phys_id;
+
+ offs = vdev_id * DLB2_MAX_NUM_LDB_PORTS + virt_id;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_VF_LDB_VPP2PP(offs), r1.val);
+
+ r2.field.vdev = vdev_id;
+
+ DLB2_CSR_WR(hw,
+ DLB2_SYS_LDB_PP2VDEV(port->id.phys_id),
+ r2.val);
+
+ r3.field.vpp_v = 1;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_VF_LDB_VPP_V(offs), r3.val);
+ }
+
+ r4.field.pp_v = 1;
+
+ DLB2_CSR_WR(hw,
+ DLB2_SYS_LDB_PP_V(port->id.phys_id),
+ r4.val);
+}
+
+static int dlb2_ldb_port_configure_cq(struct dlb2_hw *hw,
+ struct dlb2_hw_domain *domain,
+ struct dlb2_ldb_port *port,
+ uintptr_t cq_dma_base,
+ struct dlb2_create_ldb_port_args *args,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ union dlb2_sys_ldb_cq_addr_l r0 = { {0} };
+ union dlb2_sys_ldb_cq_addr_u r1 = { {0} };
+ union dlb2_sys_ldb_cq2vf_pf_ro r2 = { {0} };
+ union dlb2_chp_ldb_cq_tkn_depth_sel r3 = { {0} };
+ union dlb2_lsp_cq_ldb_tkn_depth_sel r4 = { {0} };
+ union dlb2_chp_hist_list_lim r5 = { {0} };
+ union dlb2_chp_hist_list_base r6 = { {0} };
+ union dlb2_lsp_cq_ldb_infl_lim r7 = { {0} };
+ union dlb2_chp_hist_list_push_ptr r8 = { {0} };
+ union dlb2_chp_hist_list_pop_ptr r9 = { {0} };
+ union dlb2_sys_ldb_cq_at r10 = { {0} };
+ union dlb2_sys_ldb_cq_pasid r11 = { {0} };
+ union dlb2_chp_ldb_cq2vas r12 = { {0} };
+ union dlb2_lsp_cq2priov r13 = { {0} };
+
+ /* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+ r0.field.addr_l = cq_dma_base >> 6;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_LDB_CQ_ADDR_L(port->id.phys_id), r0.val);
+
+ r1.field.addr_u = cq_dma_base >> 32;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_LDB_CQ_ADDR_U(port->id.phys_id), r1.val);
+
+ /*
+ * 'ro' == relaxed ordering. This setting allows DLB2 to write
+ * cache lines out-of-order (but QEs within a cache line are always
+ * updated in-order).
+ */
+ r2.field.vf = vdev_id;
+ r2.field.is_pf = !vdev_req && (hw->virt_mode != DLB2_VIRT_SIOV);
+ r2.field.ro = 1;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_LDB_CQ2VF_PF_RO(port->id.phys_id), r2.val);
+
+ if (args->cq_depth <= 8) {
+ r3.field.token_depth_select = 1;
+ } else if (args->cq_depth == 16) {
+ r3.field.token_depth_select = 2;
+ } else if (args->cq_depth == 32) {
+ r3.field.token_depth_select = 3;
+ } else if (args->cq_depth == 64) {
+ r3.field.token_depth_select = 4;
+ } else if (args->cq_depth == 128) {
+ r3.field.token_depth_select = 5;
+ } else if (args->cq_depth == 256) {
+ r3.field.token_depth_select = 6;
+ } else if (args->cq_depth == 512) {
+ r3.field.token_depth_select = 7;
+ } else if (args->cq_depth == 1024) {
+ r3.field.token_depth_select = 8;
+ } else {
+ DLB2_HW_ERR(hw,
+ "[%s():%d] Internal error: invalid CQ depth\n",
+ __func__, __LINE__);
+ return -EFAULT;
+ }
+
+ DLB2_CSR_WR(hw,
+ DLB2_CHP_LDB_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+ r3.val);
+
+ /*
+ * To support CQs with depth less than 8, program the token count
+ * register with a non-zero initial value. Operations such as domain
+ * reset must take this initial value into account when quiescing the
+ * CQ.
+ */
+ port->init_tkn_cnt = 0;
+
+ if (args->cq_depth < 8) {
+ union dlb2_lsp_cq_ldb_tkn_cnt r14 = { {0} };
+
+ port->init_tkn_cnt = 8 - args->cq_depth;
+
+ r14.field.token_count = port->init_tkn_cnt;
+
+ DLB2_CSR_WR(hw,
+ DLB2_LSP_CQ_LDB_TKN_CNT(port->id.phys_id),
+ r14.val);
+ } else {
+ DLB2_CSR_WR(hw,
+ DLB2_LSP_CQ_LDB_TKN_CNT(port->id.phys_id),
+ DLB2_LSP_CQ_LDB_TKN_CNT_RST);
+ }
+
+ r4.field.token_depth_select = r3.field.token_depth_select;
+ r4.field.ignore_depth = 0;
+
+ DLB2_CSR_WR(hw,
+ DLB2_LSP_CQ_LDB_TKN_DEPTH_SEL(port->id.phys_id),
+ r4.val);
+
+ /* Reset the CQ write pointer */
+ DLB2_CSR_WR(hw,
+ DLB2_CHP_LDB_CQ_WPTR(port->id.phys_id),
+ DLB2_CHP_LDB_CQ_WPTR_RST);
+
+ r5.field.limit = port->hist_list_entry_limit - 1;
+
+ DLB2_CSR_WR(hw, DLB2_CHP_HIST_LIST_LIM(port->id.phys_id), r5.val);
+
+ r6.field.base = port->hist_list_entry_base;
+
+ DLB2_CSR_WR(hw, DLB2_CHP_HIST_LIST_BASE(port->id.phys_id), r6.val);
+
+ /*
+ * The inflight limit sets a cap on the number of QEs for which this CQ
+ * can owe completions at one time.
+ */
+ r7.field.limit = args->cq_history_list_size;
+
+ DLB2_CSR_WR(hw, DLB2_LSP_CQ_LDB_INFL_LIM(port->id.phys_id), r7.val);
+
+ r8.field.push_ptr = r6.field.base;
+ r8.field.generation = 0;
+
+ DLB2_CSR_WR(hw,
+ DLB2_CHP_HIST_LIST_PUSH_PTR(port->id.phys_id),
+ r8.val);
+
+ r9.field.pop_ptr = r6.field.base;
+ r9.field.generation = 0;
+
+ DLB2_CSR_WR(hw, DLB2_CHP_HIST_LIST_POP_PTR(port->id.phys_id), r9.val);
+
+ /*
+ * Address translation (AT) settings: 0: untranslated, 2: translated
+ * (see ATS spec regarding Address Type field for more details)
+ */
+ r10.field.cq_at = 0;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_LDB_CQ_AT(port->id.phys_id), r10.val);
+
+ if (vdev_req && hw->virt_mode == DLB2_VIRT_SIOV) {
+ r11.field.pasid = hw->pasid[vdev_id];
+ r11.field.fmt2 = 1;
+ }
+
+ DLB2_CSR_WR(hw,
+ DLB2_SYS_LDB_CQ_PASID(port->id.phys_id),
+ r11.val);
+
+ r12.field.cq2vas = domain->id.phys_id;
+
+ DLB2_CSR_WR(hw, DLB2_CHP_LDB_CQ2VAS(port->id.phys_id), r12.val);
+
+ /* Disable the port's QID mappings */
+ r13.field.v = 0;
+
+ DLB2_CSR_WR(hw, DLB2_LSP_CQ2PRIOV(port->id.phys_id), r13.val);
+
+ return 0;
+}
+
+static int dlb2_configure_ldb_port(struct dlb2_hw *hw,
+ struct dlb2_hw_domain *domain,
+ struct dlb2_ldb_port *port,
+ uintptr_t cq_dma_base,
+ struct dlb2_create_ldb_port_args *args,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ int ret, i;
+
+ port->hist_list_entry_base = domain->hist_list_entry_base +
+ domain->hist_list_entry_offset;
+ port->hist_list_entry_limit = port->hist_list_entry_base +
+ args->cq_history_list_size;
+
+ domain->hist_list_entry_offset += args->cq_history_list_size;
+ domain->avail_hist_list_entries -= args->cq_history_list_size;
+
+ ret = dlb2_ldb_port_configure_cq(hw,
+ domain,
+ port,
+ cq_dma_base,
+ args,
+ vdev_req,
+ vdev_id);
+ if (ret < 0)
+ return ret;
+
+ dlb2_ldb_port_configure_pp(hw,
+ domain,
+ port,
+ vdev_req,
+ vdev_id);
+
+ dlb2_ldb_port_cq_enable(hw, port);
+
+ for (i = 0; i < DLB2_MAX_NUM_QIDS_PER_LDB_CQ; i++)
+ port->qid_map[i].state = DLB2_QUEUE_UNMAPPED;
+ port->num_mappings = 0;
+
+ port->enabled = true;
+
+ port->configured = true;
+
+ return 0;
+}
+
+static void
+dlb2_log_create_ldb_port_args(struct dlb2_hw *hw,
+ u32 domain_id,
+ uintptr_t cq_dma_base,
+ struct dlb2_create_ldb_port_args *args,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ DLB2_HW_DBG(hw, "DLB2 create load-balanced port arguments:\n");
+ if (vdev_req)
+ DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
+ DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
+ domain_id);
+ DLB2_HW_DBG(hw, "\tCQ depth: %d\n",
+ args->cq_depth);
+ DLB2_HW_DBG(hw, "\tCQ hist list size: %d\n",
+ args->cq_history_list_size);
+ DLB2_HW_DBG(hw, "\tCQ base address: 0x%lx\n",
+ cq_dma_base);
+ DLB2_HW_DBG(hw, "\tCoS ID: %u\n", args->cos_id);
+ DLB2_HW_DBG(hw, "\tStrict CoS allocation: %u\n",
+ args->cos_strict);
+}
+
+static int
+dlb2_verify_create_ldb_port_args(struct dlb2_hw *hw,
+ u32 domain_id,
+ uintptr_t cq_dma_base,
+ struct dlb2_create_ldb_port_args *args,
+ struct dlb2_cmd_response *resp,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ struct dlb2_hw_domain *domain;
+ int i;
+
+ domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
+
+ if (domain == NULL) {
+ resp->status = DLB2_ST_INVALID_DOMAIN_ID;
+ return -EINVAL;
+ }
+
+ if (!domain->configured) {
+ resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
+ return -EINVAL;
+ }
+
+ if (domain->started) {
+ resp->status = DLB2_ST_DOMAIN_STARTED;
+ return -EINVAL;
+ }
+
+ if (args->cos_id >= DLB2_NUM_COS_DOMAINS) {
+ resp->status = DLB2_ST_INVALID_COS_ID;
+ return -EINVAL;
+ }
+
+ if (args->cos_strict) {
+ if (dlb2_list_empty(&domain->avail_ldb_ports[args->cos_id])) {
+ resp->status = DLB2_ST_LDB_PORTS_UNAVAILABLE;
+ return -EINVAL;
+ }
+ } else {
+ for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
+ if (!dlb2_list_empty(&domain->avail_ldb_ports[i]))
+ break;
+ }
+
+ if (i == DLB2_NUM_COS_DOMAINS) {
+ resp->status = DLB2_ST_LDB_PORTS_UNAVAILABLE;
+ return -EINVAL;
+ }
+ }
+
+ /* Check cache-line alignment */
+ if ((cq_dma_base & 0x3F) != 0) {
+ resp->status = DLB2_ST_INVALID_CQ_VIRT_ADDR;
+ return -EINVAL;
+ }
+
+ if (args->cq_depth != 1 &&
+ args->cq_depth != 2 &&
+ args->cq_depth != 4 &&
+ args->cq_depth != 8 &&
+ args->cq_depth != 16 &&
+ args->cq_depth != 32 &&
+ args->cq_depth != 64 &&
+ args->cq_depth != 128 &&
+ args->cq_depth != 256 &&
+ args->cq_depth != 512 &&
+ args->cq_depth != 1024) {
+ resp->status = DLB2_ST_INVALID_CQ_DEPTH;
+ return -EINVAL;
+ }
+
+ /* The history list size must be >= 1 */
+ if (!args->cq_history_list_size) {
+ resp->status = DLB2_ST_INVALID_HIST_LIST_DEPTH;
+ return -EINVAL;
+ }
+
+ if (args->cq_history_list_size > domain->avail_hist_list_entries) {
+ resp->status = DLB2_ST_HIST_LIST_ENTRIES_UNAVAILABLE;
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+
+/**
+ * dlb2_hw_create_ldb_port() - Allocate and initialize a load-balanced port and
+ * its resources.
+ * @hw: Contains the current state of the DLB2 hardware.
+ * @domain_id: Domain ID
+ * @args: User-provided arguments.
+ * @cq_dma_base: Base DMA address for consumer queue memory
+ * @resp: Response to user.
+ * @vdev_req: Request came from a virtual device.
+ * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb2_hw_create_ldb_port(struct dlb2_hw *hw,
+ u32 domain_id,
+ struct dlb2_create_ldb_port_args *args,
+ uintptr_t cq_dma_base,
+ struct dlb2_cmd_response *resp,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ struct dlb2_hw_domain *domain;
+ struct dlb2_ldb_port *port;
+ int ret, cos_id, i;
+
+ dlb2_log_create_ldb_port_args(hw,
+ domain_id,
+ cq_dma_base,
+ args,
+ vdev_req,
+ vdev_id);
+
+ /*
+ * Verify that hardware resources are available before attempting to
+ * satisfy the request. This simplifies the error unwinding code.
+ */
+ ret = dlb2_verify_create_ldb_port_args(hw,
+ domain_id,
+ cq_dma_base,
+ args,
+ resp,
+ vdev_req,
+ vdev_id);
+ if (ret)
+ return ret;
+
+ domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
+ if (domain == NULL) {
+ DLB2_HW_ERR(hw,
+ "[%s():%d] Internal error: domain not found\n",
+ __func__, __LINE__);
+ return -EFAULT;
+ }
+
+ if (args->cos_strict) {
+ cos_id = args->cos_id;
+
+ port = DLB2_DOM_LIST_HEAD(domain->avail_ldb_ports[cos_id],
+ typeof(*port));
+ } else {
+ int idx;
+
+ for (i = 0; i < DLB2_NUM_COS_DOMAINS; i++) {
+ idx = (args->cos_id + i) % DLB2_NUM_COS_DOMAINS;
+
+ port = DLB2_DOM_LIST_HEAD(domain->avail_ldb_ports[idx],
+ typeof(*port));
+ if (port)
+ break;
+ }
+
+ cos_id = idx;
+ }
+
+ if (port == NULL) {
+ DLB2_HW_ERR(hw,
+ "[%s():%d] Internal error: no available ldb ports\n",
+ __func__, __LINE__);
+ return -EFAULT;
+ }
+
+ if (port->configured) {
+ DLB2_HW_ERR(hw,
+ "[%s()] Internal error: avail_ldb_ports contains configured ports.\n",
+ __func__);
+ return -EFAULT;
+ }
+
+ ret = dlb2_configure_ldb_port(hw,
+ domain,
+ port,
+ cq_dma_base,
+ args,
+ vdev_req,
+ vdev_id);
+ if (ret < 0)
+ return ret;
+
+ /*
+ * Configuration succeeded, so move the resource from the 'avail' to
+ * the 'used' list.
+ */
+ dlb2_list_del(&domain->avail_ldb_ports[cos_id], &port->domain_list);
+
+ dlb2_list_add(&domain->used_ldb_ports[cos_id], &port->domain_list);
+
+ resp->status = 0;
+ resp->id = (vdev_req) ? port->id.virt_id : port->id.phys_id;
+
+ return 0;
+}
+
+static void
+dlb2_log_create_dir_port_args(struct dlb2_hw *hw,
+ u32 domain_id,
+ uintptr_t cq_dma_base,
+ struct dlb2_create_dir_port_args *args,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ DLB2_HW_DBG(hw, "DLB2 create directed port arguments:\n");
+ if (vdev_req)
+ DLB2_HW_DBG(hw, "(Request from vdev %d)\n", vdev_id);
+ DLB2_HW_DBG(hw, "\tDomain ID: %d\n",
+ domain_id);
+ DLB2_HW_DBG(hw, "\tCQ depth: %d\n",
+ args->cq_depth);
+ DLB2_HW_DBG(hw, "\tCQ base address: 0x%lx\n",
+ cq_dma_base);
+}
+
+static struct dlb2_dir_pq_pair *
+dlb2_get_domain_used_dir_pq(u32 id,
+ bool vdev_req,
+ struct dlb2_hw_domain *domain)
+{
+ struct dlb2_list_entry *iter;
+ struct dlb2_dir_pq_pair *port;
+ RTE_SET_USED(iter);
+
+ if (id >= DLB2_MAX_NUM_DIR_PORTS)
+ return NULL;
+
+ DLB2_DOM_LIST_FOR(domain->used_dir_pq_pairs, port, iter)
+ if ((!vdev_req && port->id.phys_id == id) ||
+ (vdev_req && port->id.virt_id == id))
+ return port;
+
+ return NULL;
+}
+
+static int
+dlb2_verify_create_dir_port_args(struct dlb2_hw *hw,
+ u32 domain_id,
+ uintptr_t cq_dma_base,
+ struct dlb2_create_dir_port_args *args,
+ struct dlb2_cmd_response *resp,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ struct dlb2_hw_domain *domain;
+
+ domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
+
+ if (domain == NULL) {
+ resp->status = DLB2_ST_INVALID_DOMAIN_ID;
+ return -EINVAL;
+ }
+
+ if (!domain->configured) {
+ resp->status = DLB2_ST_DOMAIN_NOT_CONFIGURED;
+ return -EINVAL;
+ }
+
+ if (domain->started) {
+ resp->status = DLB2_ST_DOMAIN_STARTED;
+ return -EINVAL;
+ }
+
+ /*
+ * If the user claims the queue is already configured, validate
+ * the queue ID, its domain, and whether the queue is configured.
+ */
+ if (args->queue_id != -1) {
+ struct dlb2_dir_pq_pair *queue;
+
+ queue = dlb2_get_domain_used_dir_pq(args->queue_id,
+ vdev_req,
+ domain);
+
+ if (queue == NULL || queue->domain_id.phys_id !=
+ domain->id.phys_id ||
+ !queue->queue_configured) {
+ resp->status = DLB2_ST_INVALID_DIR_QUEUE_ID;
+ return -EINVAL;
+ }
+ }
+
+ /*
+ * If the port's queue is not configured, validate that a free
+ * port-queue pair is available.
+ */
+ if (args->queue_id == -1 &&
+ dlb2_list_empty(&domain->avail_dir_pq_pairs)) {
+ resp->status = DLB2_ST_DIR_PORTS_UNAVAILABLE;
+ return -EINVAL;
+ }
+
+ /* Check cache-line alignment */
+ if ((cq_dma_base & 0x3F) != 0) {
+ resp->status = DLB2_ST_INVALID_CQ_VIRT_ADDR;
+ return -EINVAL;
+ }
+
+ if (args->cq_depth != 1 &&
+ args->cq_depth != 2 &&
+ args->cq_depth != 4 &&
+ args->cq_depth != 8 &&
+ args->cq_depth != 16 &&
+ args->cq_depth != 32 &&
+ args->cq_depth != 64 &&
+ args->cq_depth != 128 &&
+ args->cq_depth != 256 &&
+ args->cq_depth != 512 &&
+ args->cq_depth != 1024) {
+ resp->status = DLB2_ST_INVALID_CQ_DEPTH;
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void dlb2_dir_port_configure_pp(struct dlb2_hw *hw,
+ struct dlb2_hw_domain *domain,
+ struct dlb2_dir_pq_pair *port,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ union dlb2_sys_dir_pp2vas r0 = { {0} };
+ union dlb2_sys_dir_pp_v r4 = { {0} };
+
+ r0.field.vas = domain->id.phys_id;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_DIR_PP2VAS(port->id.phys_id), r0.val);
+
+ if (vdev_req) {
+ union dlb2_sys_vf_dir_vpp2pp r1 = { {0} };
+ union dlb2_sys_dir_pp2vdev r2 = { {0} };
+ union dlb2_sys_vf_dir_vpp_v r3 = { {0} };
+ unsigned int offs;
+ u32 virt_id;
+
+ /*
+ * DLB uses producer port address bits 17:12 to determine the
+ * producer port ID. In Scalable IOV mode, PP accesses come
+ * through the PF MMIO window for the physical producer port,
+ * so for translation purposes the virtual and physical port
+ * IDs are equal.
+ */
+ if (hw->virt_mode == DLB2_VIRT_SRIOV)
+ virt_id = port->id.virt_id;
+ else
+ virt_id = port->id.phys_id;
+
+ r1.field.pp = port->id.phys_id;
+
+ offs = vdev_id * DLB2_MAX_NUM_DIR_PORTS + virt_id;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VPP2PP(offs), r1.val);
+
+ r2.field.vdev = vdev_id;
+
+ DLB2_CSR_WR(hw,
+ DLB2_SYS_DIR_PP2VDEV(port->id.phys_id),
+ r2.val);
+
+ r3.field.vpp_v = 1;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_VF_DIR_VPP_V(offs), r3.val);
+ }
+
+ r4.field.pp_v = 1;
+
+ DLB2_CSR_WR(hw,
+ DLB2_SYS_DIR_PP_V(port->id.phys_id),
+ r4.val);
+}
+
+static int dlb2_dir_port_configure_cq(struct dlb2_hw *hw,
+ struct dlb2_hw_domain *domain,
+ struct dlb2_dir_pq_pair *port,
+ uintptr_t cq_dma_base,
+ struct dlb2_create_dir_port_args *args,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ union dlb2_sys_dir_cq_addr_l r0 = { {0} };
+ union dlb2_sys_dir_cq_addr_u r1 = { {0} };
+ union dlb2_sys_dir_cq2vf_pf_ro r2 = { {0} };
+ union dlb2_chp_dir_cq_tkn_depth_sel r3 = { {0} };
+ union dlb2_lsp_cq_dir_tkn_depth_sel_dsi r4 = { {0} };
+ union dlb2_sys_dir_cq_fmt r9 = { {0} };
+ union dlb2_sys_dir_cq_at r10 = { {0} };
+ union dlb2_sys_dir_cq_pasid r11 = { {0} };
+ union dlb2_chp_dir_cq2vas r12 = { {0} };
+
+ /* The CQ address is 64B-aligned, and the DLB only wants bits [63:6] */
+ r0.field.addr_l = cq_dma_base >> 6;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_ADDR_L(port->id.phys_id), r0.val);
+
+ r1.field.addr_u = cq_dma_base >> 32;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_ADDR_U(port->id.phys_id), r1.val);
+
+ /*
+ * 'ro' == relaxed ordering. This setting allows DLB2 to write
+ * cache lines out-of-order (but QEs within a cache line are always
+ * updated in-order).
+ */
+ r2.field.vf = vdev_id;
+ r2.field.is_pf = !vdev_req && (hw->virt_mode != DLB2_VIRT_SIOV);
+ r2.field.ro = 1;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ2VF_PF_RO(port->id.phys_id), r2.val);
+
+ if (args->cq_depth <= 8) {
+ r3.field.token_depth_select = 1;
+ } else if (args->cq_depth == 16) {
+ r3.field.token_depth_select = 2;
+ } else if (args->cq_depth == 32) {
+ r3.field.token_depth_select = 3;
+ } else if (args->cq_depth == 64) {
+ r3.field.token_depth_select = 4;
+ } else if (args->cq_depth == 128) {
+ r3.field.token_depth_select = 5;
+ } else if (args->cq_depth == 256) {
+ r3.field.token_depth_select = 6;
+ } else if (args->cq_depth == 512) {
+ r3.field.token_depth_select = 7;
+ } else if (args->cq_depth == 1024) {
+ r3.field.token_depth_select = 8;
+ } else {
+ DLB2_HW_ERR(hw,
+ "[%s():%d] Internal error: invalid CQ depth\n",
+ __func__, __LINE__);
+ return -EFAULT;
+ }
+
+ DLB2_CSR_WR(hw,
+ DLB2_CHP_DIR_CQ_TKN_DEPTH_SEL(port->id.phys_id),
+ r3.val);
+
+ /*
+ * To support CQs with depth less than 8, program the token count
+ * register with a non-zero initial value. Operations such as domain
+ * reset must take this initial value into account when quiescing the
+ * CQ.
+ */
+ port->init_tkn_cnt = 0;
+
+ if (args->cq_depth < 8) {
+ union dlb2_lsp_cq_dir_tkn_cnt r13 = { {0} };
+
+ port->init_tkn_cnt = 8 - args->cq_depth;
+
+ r13.field.count = port->init_tkn_cnt;
+
+ DLB2_CSR_WR(hw,
+ DLB2_LSP_CQ_DIR_TKN_CNT(port->id.phys_id),
+ r13.val);
+ } else {
+ DLB2_CSR_WR(hw,
+ DLB2_LSP_CQ_DIR_TKN_CNT(port->id.phys_id),
+ DLB2_LSP_CQ_DIR_TKN_CNT_RST);
+ }
+
+ r4.field.token_depth_select = r3.field.token_depth_select;
+ r4.field.disable_wb_opt = 0;
+ r4.field.ignore_depth = 0;
+
+ DLB2_CSR_WR(hw,
+ DLB2_LSP_CQ_DIR_TKN_DEPTH_SEL_DSI(port->id.phys_id),
+ r4.val);
+
+ /* Reset the CQ write pointer */
+ DLB2_CSR_WR(hw,
+ DLB2_CHP_DIR_CQ_WPTR(port->id.phys_id),
+ DLB2_CHP_DIR_CQ_WPTR_RST);
+
+ /* Virtualize the PPID */
+ r9.field.keep_pf_ppid = 0;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_FMT(port->id.phys_id), r9.val);
+
+ /*
+ * Address translation (AT) settings: 0: untranslated, 2: translated
+ * (see ATS spec regarding Address Type field for more details)
+ */
+ r10.field.cq_at = 0;
+
+ DLB2_CSR_WR(hw, DLB2_SYS_DIR_CQ_AT(port->id.phys_id), r10.val);
+
+ if (vdev_req && hw->virt_mode == DLB2_VIRT_SIOV) {
+ r11.field.pasid = hw->pasid[vdev_id];
+ r11.field.fmt2 = 1;
+ }
+
+ DLB2_CSR_WR(hw,
+ DLB2_SYS_DIR_CQ_PASID(port->id.phys_id),
+ r11.val);
+
+ r12.field.cq2vas = domain->id.phys_id;
+
+ DLB2_CSR_WR(hw, DLB2_CHP_DIR_CQ2VAS(port->id.phys_id), r12.val);
+
+ return 0;
+}
+
+static int dlb2_configure_dir_port(struct dlb2_hw *hw,
+ struct dlb2_hw_domain *domain,
+ struct dlb2_dir_pq_pair *port,
+ uintptr_t cq_dma_base,
+ struct dlb2_create_dir_port_args *args,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ int ret;
+
+ ret = dlb2_dir_port_configure_cq(hw,
+ domain,
+ port,
+ cq_dma_base,
+ args,
+ vdev_req,
+ vdev_id);
+
+ if (ret < 0)
+ return ret;
+
+ dlb2_dir_port_configure_pp(hw,
+ domain,
+ port,
+ vdev_req,
+ vdev_id);
+
+ dlb2_dir_port_cq_enable(hw, port);
+
+ port->enabled = true;
+
+ port->port_configured = true;
+
+ return 0;
+}
+
+/**
+ * dlb2_hw_create_dir_port() - Allocate and initialize a DLB directed port
+ * and queue. The port/queue pair have the same ID and name.
+ * @hw: Contains the current state of the DLB2 hardware.
+ * @domain_id: Domain ID
+ * @args: User-provided arguments.
+ * @cq_dma_base: Base DMA address for consumer queue memory
+ * @resp: Response to user.
+ * @vdev_req: Request came from a virtual device.
+ * @vdev_id: If vdev_req is true, this contains the virtual device's ID.
+ *
+ * Return: returns < 0 on error, 0 otherwise. If the driver is unable to
+ * satisfy a request, resp->status will be set accordingly.
+ */
+int dlb2_hw_create_dir_port(struct dlb2_hw *hw,
+ u32 domain_id,
+ struct dlb2_create_dir_port_args *args,
+ uintptr_t cq_dma_base,
+ struct dlb2_cmd_response *resp,
+ bool vdev_req,
+ unsigned int vdev_id)
+{
+ struct dlb2_dir_pq_pair *port;
+ struct dlb2_hw_domain *domain;
+ int ret;
+
+ dlb2_log_create_dir_port_args(hw,
+ domain_id,
+ cq_dma_base,
+ args,
+ vdev_req,
+ vdev_id);
+
+ /*
+ * Verify that hardware resources are available before attempting to
+ * satisfy the request. This simplifies the error unwinding code.
+ */
+ ret = dlb2_verify_create_dir_port_args(hw,
+ domain_id,
+ cq_dma_base,
+ args,
+ resp,
+ vdev_req,
+ vdev_id);
+ if (ret)
+ return ret;
+
+ domain = dlb2_get_domain_from_id(hw, domain_id, vdev_req, vdev_id);
+
+ if (args->queue_id != -1)
+ port = dlb2_get_domain_used_dir_pq(args->queue_id,
+ vdev_req,
+ domain);
+ else
+ port = DLB2_DOM_LIST_HEAD(domain->avail_dir_pq_pairs,
+ typeof(*port));
+ if (port == NULL) {
+ DLB2_HW_ERR(hw,
+ "[%s():%d] Internal error: no available dir ports\n",
+ __func__, __LINE__);
+ return -EFAULT;
+ }
+
+ ret = dlb2_configure_dir_port(hw,
+ domain,
+ port,
+ cq_dma_base,
+ args,
+ vdev_req,
+ vdev_id);
+ if (ret < 0)
+ return ret;
+
+ /*
+ * Configuration succeeded, so move the resource from the 'avail' to
+ * the 'used' list (if it's not already there).
+ */
+ if (args->queue_id == -1) {
+ dlb2_list_del(&domain->avail_dir_pq_pairs, &port->domain_list);
+
+ dlb2_list_add(&domain->used_dir_pq_pairs, &port->domain_list);
+ }
+
+ resp->status = 0;
+ resp->id = (vdev_req) ? port->id.virt_id : port->id.phys_id;
+
+ return 0;
+}
return dlb2_hw_create_ldb_queue(hw, id, args, resp, NOT_VF_REQ,
PF_ID_ZERO);
}
+
+int
+dlb2_pf_create_ldb_port(struct dlb2_hw *hw,
+ u32 id,
+ struct dlb2_create_ldb_port_args *args,
+ uintptr_t cq_dma_base,
+ struct dlb2_cmd_response *resp)
+{
+ return dlb2_hw_create_ldb_port(hw, id, args,
+ cq_dma_base,
+ resp,
+ NOT_VF_REQ,
+ PF_ID_ZERO);
+}
+
+int
+dlb2_pf_create_dir_port(struct dlb2_hw *hw,
+ u32 id,
+ struct dlb2_create_dir_port_args *args,
+ uintptr_t cq_dma_base,
+ struct dlb2_cmd_response *resp)
+{
+ return dlb2_hw_create_dir_port(hw, id, args,
+ cq_dma_base,
+ resp,
+ NOT_VF_REQ,
+ PF_ID_ZERO);
+}
return ret;
}
+static void *
+dlb2_alloc_coherent_aligned(const struct rte_memzone **mz, uintptr_t *phys,
+ size_t size, int align)
+{
+ char mz_name[RTE_MEMZONE_NAMESIZE];
+ uint32_t core_id = rte_lcore_id();
+ unsigned int socket_id;
+
+ snprintf(mz_name, sizeof(mz_name) - 1, "event_dlb2_pf_%lx",
+ (unsigned long)rte_get_timer_cycles());
+ if (core_id == (unsigned int)LCORE_ID_ANY)
+ core_id = rte_get_main_lcore();
+ socket_id = rte_lcore_to_socket_id(core_id);
+ *mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+ RTE_MEMZONE_IOVA_CONTIG, align);
+ if (*mz == NULL) {
+ DLB2_LOG_DBG("Unable to allocate DMA memory of size %zu bytes - %s\n",
+ size, rte_strerror(rte_errno));
+ *phys = 0;
+ return NULL;
+ }
+ *phys = (*mz)->iova;
+ return (*mz)->addr;
+}
+
+static int
+dlb2_pf_ldb_port_create(struct dlb2_hw_dev *handle,
+ struct dlb2_create_ldb_port_args *cfg,
+ enum dlb2_cq_poll_modes poll_mode)
+{
+ struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
+ struct dlb2_cmd_response response = {0};
+ struct dlb2_port_memory port_memory;
+ int ret, cq_alloc_depth;
+ uint8_t *port_base;
+ const struct rte_memzone *mz;
+ int alloc_sz, qe_sz;
+ phys_addr_t cq_base;
+ phys_addr_t pp_base;
+ int is_dir = false;
+
+ DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
+
+ if (poll_mode == DLB2_CQ_POLL_MODE_STD)
+ qe_sz = sizeof(struct dlb2_dequeue_qe);
+ else
+ qe_sz = RTE_CACHE_LINE_SIZE;
+
+ /* Calculate the port memory required, and round up to the nearest
+ * cache line.
+ */
+ cq_alloc_depth = RTE_MAX(cfg->cq_depth, DLB2_MIN_HARDWARE_CQ_DEPTH);
+ alloc_sz = cq_alloc_depth * qe_sz;
+ alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+ port_base = dlb2_alloc_coherent_aligned(&mz, &cq_base, alloc_sz,
+ PAGE_SIZE);
+ if (port_base == NULL)
+ return -ENOMEM;
+
+ /* Lock the page in memory */
+ ret = rte_mem_lock_page(port_base);
+ if (ret < 0) {
+ DLB2_LOG_ERR("dlb2 pf pmd could not lock page for device i/o\n");
+ goto create_port_err;
+ }
+
+ memset(port_base, 0, alloc_sz);
+
+ ret = dlb2_pf_create_ldb_port(&dlb2_dev->hw,
+ handle->domain_id,
+ cfg,
+ cq_base,
+ &response);
+ if (ret)
+ goto create_port_err;
+
+ pp_base = (uintptr_t)dlb2_dev->hw.func_kva + PP_BASE(is_dir);
+ dlb2_port[response.id][DLB2_LDB_PORT].pp_addr =
+ (void *)(pp_base + (PAGE_SIZE * response.id));
+
+ dlb2_port[response.id][DLB2_LDB_PORT].cq_base = (void *)(port_base);
+ memset(&port_memory, 0, sizeof(port_memory));
+
+ dlb2_port[response.id][DLB2_LDB_PORT].mz = mz;
+
+ dlb2_list_init_head(&port_memory.list);
+
+ cfg->response = response;
+
+ return 0;
+
+create_port_err:
+
+ rte_memzone_free(mz);
+
+ DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
+ __func__, ret);
+ return ret;
+}
+
+static int
+dlb2_pf_dir_port_create(struct dlb2_hw_dev *handle,
+ struct dlb2_create_dir_port_args *cfg,
+ enum dlb2_cq_poll_modes poll_mode)
+{
+ struct dlb2_dev *dlb2_dev = (struct dlb2_dev *)handle->pf_dev;
+ struct dlb2_cmd_response response = {0};
+ struct dlb2_port_memory port_memory;
+ int ret;
+ uint8_t *port_base;
+ const struct rte_memzone *mz;
+ int alloc_sz, qe_sz;
+ phys_addr_t cq_base;
+ phys_addr_t pp_base;
+ int is_dir = true;
+
+ DLB2_INFO(dev->dlb2_device, "Entering %s()\n", __func__);
+
+ if (poll_mode == DLB2_CQ_POLL_MODE_STD)
+ qe_sz = sizeof(struct dlb2_dequeue_qe);
+ else
+ qe_sz = RTE_CACHE_LINE_SIZE;
+
+ /* Calculate the port memory required, and round up to the nearest
+ * cache line.
+ */
+ alloc_sz = cfg->cq_depth * qe_sz;
+ alloc_sz = RTE_CACHE_LINE_ROUNDUP(alloc_sz);
+
+ port_base = dlb2_alloc_coherent_aligned(&mz, &cq_base, alloc_sz,
+ PAGE_SIZE);
+ if (port_base == NULL)
+ return -ENOMEM;
+
+ /* Lock the page in memory */
+ ret = rte_mem_lock_page(port_base);
+ if (ret < 0) {
+ DLB2_LOG_ERR("dlb2 pf pmd could not lock page for device i/o\n");
+ goto create_port_err;
+ }
+
+ memset(port_base, 0, alloc_sz);
+
+ ret = dlb2_pf_create_dir_port(&dlb2_dev->hw,
+ handle->domain_id,
+ cfg,
+ cq_base,
+ &response);
+ if (ret)
+ goto create_port_err;
+
+ pp_base = (uintptr_t)dlb2_dev->hw.func_kva + PP_BASE(is_dir);
+ dlb2_port[response.id][DLB2_DIR_PORT].pp_addr =
+ (void *)(pp_base + (PAGE_SIZE * response.id));
+
+ dlb2_port[response.id][DLB2_DIR_PORT].cq_base =
+ (void *)(port_base);
+ memset(&port_memory, 0, sizeof(port_memory));
+
+ dlb2_port[response.id][DLB2_DIR_PORT].mz = mz;
+
+ dlb2_list_init_head(&port_memory.list);
+
+ cfg->response = response;
+
+ return 0;
+
+create_port_err:
+
+ rte_memzone_free(mz);
+
+ DLB2_INFO(dev->dlb2_device, "Exiting %s() with ret=%d\n",
+ __func__, ret);
+
+ return ret;
+}
+
static void
dlb2_pf_iface_fn_ptrs_init(void)
{
dlb2_iface_get_cq_poll_mode = dlb2_pf_get_cq_poll_mode;
dlb2_iface_sched_domain_create = dlb2_pf_sched_domain_create;
dlb2_iface_ldb_queue_create = dlb2_pf_ldb_queue_create;
+ dlb2_iface_ldb_port_create = dlb2_pf_ldb_port_create;
+ dlb2_iface_dir_port_create = dlb2_pf_dir_port_create;
dlb2_iface_get_sn_allocation = dlb2_pf_get_sn_allocation;
dlb2_iface_set_sn_allocation = dlb2_pf_set_sn_allocation;
dlb2_iface_get_sn_occupancy = dlb2_pf_get_sn_occupancy;
git.droids-corp.org / dpdk.git / commitdiff