-/*-
-* BSD LICENSE
-*
-* Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates.
-* All rights reserved.
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions
-* are met:
-*
-* * Redistributions of source code must retain the above copyright
-* notice, this list of conditions and the following disclaimer.
-* * Redistributions in binary form must reproduce the above copyright
-* notice, this list of conditions and the following disclaimer in
-* the documentation and/or other materials provided with the
-* distribution.
-* * Neither the name of copyright holder nor the names of its
-* contributors may be used to endorse or promote products derived
-* from this software without specific prior written permission.
-*
-* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates.
+ * All rights reserved.
+ */
#include "ena_com.h"
#define ENA_ASYNC_QUEUE_DEPTH 16
#define ENA_ADMIN_QUEUE_DEPTH 32
-
#define ENA_CTRL_MAJOR 0
#define ENA_CTRL_MINOR 0
#define ENA_CTRL_SUB_MINOR 1
struct ena_admin_acq_get_stats_resp get_resp;
};
-static inline int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
+static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
struct ena_common_mem_addr *ena_addr,
dma_addr_t addr)
{
}
ena_addr->mem_addr_low = lower_32_bits(addr);
- ena_addr->mem_addr_high = (u16)upper_32_bits(addr);
+ ena_addr->mem_addr_high = upper_32_bits(addr);
return 0;
}
sq->mem_handle);
if (!sq->entries) {
- ena_trc_err("memory allocation failed");
+ ena_trc_err("memory allocation failed\n");
return ENA_COM_NO_MEM;
}
cq->mem_handle);
if (!cq->entries) {
- ena_trc_err("memory allocation failed");
+ ena_trc_err("memory allocation failed\n");
return ENA_COM_NO_MEM;
}
aenq->mem_handle);
if (!aenq->entries) {
- ena_trc_err("memory allocation failed");
+ ena_trc_err("memory allocation failed\n");
return ENA_COM_NO_MEM;
}
return 0;
}
-static inline void comp_ctxt_release(struct ena_com_admin_queue *queue,
+static void comp_ctxt_release(struct ena_com_admin_queue *queue,
struct ena_comp_ctx *comp_ctx)
{
comp_ctx->occupied = false;
return NULL;
}
+ if (unlikely(!queue->comp_ctx)) {
+ ena_trc_err("Completion context is NULL\n");
+ return NULL;
+ }
+
if (unlikely(queue->comp_ctx[command_id].occupied && capture)) {
ena_trc_err("Completion context is occupied\n");
return NULL;
return comp_ctx;
}
-static inline int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
+static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *queue)
{
size_t size = queue->q_depth * sizeof(struct ena_comp_ctx);
struct ena_comp_ctx *comp_ctx;
queue->comp_ctx = ENA_MEM_ALLOC(queue->q_dmadev, size);
if (unlikely(!queue->comp_ctx)) {
- ena_trc_err("memory allocation failed");
+ ena_trc_err("memory allocation failed\n");
return ENA_COM_NO_MEM;
}
}
if (!io_sq->desc_addr.virt_addr) {
- ena_trc_err("memory allocation failed");
+ ena_trc_err("memory allocation failed\n");
return ENA_COM_NO_MEM;
}
}
if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
/* Allocate bounce buffers */
- io_sq->bounce_buf_ctrl.buffer_size = ena_dev->llq_info.desc_list_entry_size;
- io_sq->bounce_buf_ctrl.buffers_num = ENA_COM_BOUNCE_BUFFER_CNTRL_CNT;
+ io_sq->bounce_buf_ctrl.buffer_size =
+ ena_dev->llq_info.desc_list_entry_size;
+ io_sq->bounce_buf_ctrl.buffers_num =
+ ENA_COM_BOUNCE_BUFFER_CNTRL_CNT;
io_sq->bounce_buf_ctrl.next_to_use = 0;
- size = io_sq->bounce_buf_ctrl.buffer_size * io_sq->bounce_buf_ctrl.buffers_num;
+ size = io_sq->bounce_buf_ctrl.buffer_size *
+ io_sq->bounce_buf_ctrl.buffers_num;
ENA_MEM_ALLOC_NODE(ena_dev->dmadev,
size,
io_sq->bounce_buf_ctrl.base_buffer = ENA_MEM_ALLOC(ena_dev->dmadev, size);
if (!io_sq->bounce_buf_ctrl.base_buffer) {
- ena_trc_err("bounce buffer memory allocation failed");
+ ena_trc_err("bounce buffer memory allocation failed\n");
return ENA_COM_NO_MEM;
}
- memcpy(&io_sq->llq_info, &ena_dev->llq_info, sizeof(io_sq->llq_info));
+ memcpy(&io_sq->llq_info, &ena_dev->llq_info,
+ sizeof(io_sq->llq_info));
/* Initiate the first bounce buffer */
io_sq->llq_buf_ctrl.curr_bounce_buf =
}
if (!io_cq->cdesc_addr.virt_addr) {
- ena_trc_err("memory allocation failed");
+ ena_trc_err("memory allocation failed\n");
return ENA_COM_NO_MEM;
}
if (unlikely(comp_status != 0))
ena_trc_err("admin command failed[%u]\n", comp_status);
- if (unlikely(comp_status > ENA_ADMIN_UNKNOWN_ERROR))
- return ENA_COM_INVAL;
-
switch (comp_status) {
case ENA_ADMIN_SUCCESS:
- return 0;
+ return ENA_COM_OK;
case ENA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
return ENA_COM_NO_MEM;
case ENA_ADMIN_UNSUPPORTED_OPCODE:
return ENA_COM_INVAL;
}
- return 0;
+ return ENA_COM_INVAL;
}
static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_ctx,
struct ena_com_admin_queue *admin_queue)
{
unsigned long flags = 0;
- uint64_t timeout;
+ ena_time_t timeout;
int ret;
timeout = ENA_GET_SYSTEM_TIMEOUT(admin_queue->completion_timeout);
/**
* Set the LLQ configurations of the firmware
*
- * The driver provides only the enabled feature values to the FW,
+ * The driver provides only the enabled feature values to the device,
* which in turn, checks if they are supported.
*/
static int ena_com_set_llq(struct ena_com_dev *ena_dev)
supported_feat = llq_features->header_location_ctrl_supported;
if (likely(supported_feat & llq_default_cfg->llq_header_location)) {
- llq_info->header_location_ctrl = llq_default_cfg->llq_header_location;
+ llq_info->header_location_ctrl =
+ llq_default_cfg->llq_header_location;
} else {
ena_trc_err("Invalid header location control, supported: 0x%x\n",
supported_feat);
}
if (likely(llq_info->header_location_ctrl == ENA_ADMIN_INLINE_HEADER)) {
- llq_info->inline_header = true;
-
supported_feat = llq_features->descriptors_stride_ctrl_supported;
if (likely(supported_feat & llq_default_cfg->llq_stride_ctrl)) {
llq_info->desc_stride_ctrl = llq_default_cfg->llq_stride_ctrl;
return -EINVAL;
}
- ena_trc_err("Default llq stride ctrl is not supported, performing fallback,"
- "default: 0x%x, supported: 0x%x, used: 0x%x\n",
+ ena_trc_err("Default llq stride ctrl is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_stride_ctrl,
supported_feat,
llq_info->desc_stride_ctrl);
}
} else {
- llq_info->inline_header = false;
llq_info->desc_stride_ctrl = 0;
}
return -EINVAL;
}
- ena_trc_err("Default llq ring entry size is not supported, performing fallback,"
- "default: 0x%x, supported: 0x%x, used: 0x%x\n",
+ ena_trc_err("Default llq ring entry size is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_ring_entry_size,
supported_feat,
llq_info->desc_list_entry_size);
return -EINVAL;
}
- ena_trc_err("Default llq num descs before header is not supported, performing fallback,"
- "default: 0x%x, supported: 0x%x, used: 0x%x\n",
+ ena_trc_err("Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n",
llq_default_cfg->llq_num_decs_before_header,
supported_feat,
llq_info->descs_num_before_header);
if (rc)
ena_trc_err("Cannot set LLQ configuration: %d\n", rc);
- return 0;
+ return rc;
}
-
-
static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *comp_ctx,
struct ena_com_admin_queue *admin_queue)
{
admin_queue->stats.no_completion++;
ENA_SPINLOCK_UNLOCK(admin_queue->q_lock, flags);
- if (comp_ctx->status == ENA_CMD_COMPLETED)
- ena_trc_err("The ena device have completion but the driver didn't receive any MSI-X interrupt (cmd %d)\n",
- comp_ctx->cmd_opcode);
- else
- ena_trc_err("The ena device doesn't send any completion for the admin cmd %d status %d\n",
+ if (comp_ctx->status == ENA_CMD_COMPLETED) {
+ ena_trc_err("The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n",
+ comp_ctx->cmd_opcode, admin_queue->auto_polling ? "ON" : "OFF");
+ /* Check if fallback to polling is enabled */
+ if (admin_queue->auto_polling)
+ admin_queue->polling = true;
+ } else {
+ ena_trc_err("The ena device didn't send a completion for the admin cmd %d status %d\n",
comp_ctx->cmd_opcode, comp_ctx->status);
-
- admin_queue->running_state = false;
- ret = ENA_COM_TIMER_EXPIRED;
- goto err;
+ }
+ /* Check if shifted to polling mode.
+ * This will happen if there is a completion without an interrupt
+ * and autopolling mode is enabled. Continuing normal execution in such case
+ */
+ if (!admin_queue->polling) {
+ admin_queue->running_state = false;
+ ret = ENA_COM_TIMER_EXPIRED;
+ goto err;
+ }
}
ret = ena_com_comp_status_to_errno(comp_ctx->comp_status);
}
if (read_resp->reg_off != offset) {
- ena_trc_err("Read failure: wrong offset provided");
+ ena_trc_err("Read failure: wrong offset provided\n");
ret = ENA_MMIO_READ_TIMEOUT;
} else {
ret = read_resp->reg_val;
}
if (io_sq->bounce_buf_ctrl.base_buffer) {
- size = io_sq->llq_info.desc_list_entry_size * ENA_COM_BOUNCE_BUFFER_CNTRL_CNT;
- ENA_MEM_FREE(ena_dev->dmadev, io_sq->bounce_buf_ctrl.base_buffer);
+ ENA_MEM_FREE(ena_dev->dmadev,
+ io_sq->bounce_buf_ctrl.base_buffer,
+ (io_sq->llq_info.desc_list_entry_size * ENA_COM_BOUNCE_BUFFER_CNTRL_CNT));
io_sq->bounce_buf_ctrl.base_buffer = NULL;
}
}
rss->rss_ind_tbl = NULL;
if (rss->host_rss_ind_tbl)
- ENA_MEM_FREE(ena_dev->dmadev, rss->host_rss_ind_tbl);
+ ENA_MEM_FREE(ena_dev->dmadev,
+ rss->host_rss_ind_tbl,
+ ((1ULL << rss->tbl_log_size) * sizeof(u16)));
rss->host_rss_ind_tbl = NULL;
}
ENA_WAIT_EVENT_DESTROY(admin_queue->comp_ctx->wait_event);
if (admin_queue->comp_ctx)
- ENA_MEM_FREE(ena_dev->dmadev, admin_queue->comp_ctx);
+ ENA_MEM_FREE(ena_dev->dmadev,
+ admin_queue->comp_ctx,
+ (admin_queue->q_depth * sizeof(struct ena_comp_ctx)));
admin_queue->comp_ctx = NULL;
size = ADMIN_SQ_SIZE(admin_queue->q_depth);
if (sq->entries)
ena_dev->admin_queue.polling = polling;
}
+bool ena_com_get_admin_polling_mode(struct ena_com_dev * ena_dev)
+{
+ return ena_dev->admin_queue.polling;
+}
+
+void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev,
+ bool polling)
+{
+ ena_dev->admin_queue.auto_polling = polling;
+}
+
int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev)
{
struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read;
struct ena_admin_aenq_entry *aenq_e;
struct ena_admin_aenq_common_desc *aenq_common;
struct ena_com_aenq *aenq = &dev->aenq;
- unsigned long long timestamp;
+ u64 timestamp;
ena_aenq_handler handler_cb;
u16 masked_head, processed = 0;
u8 phase;
*/
dma_rmb();
- timestamp = (unsigned long long)aenq_common->timestamp_low |
- ((unsigned long long)aenq_common->timestamp_high << 32);
+ timestamp = (u64)aenq_common->timestamp_low |
+ ((u64)aenq_common->timestamp_high << 32);
ENA_TOUCH(timestamp); /* In case debug is disabled */
- ena_trc_dbg("AENQ! Group[%x] Syndrom[%x] timestamp: [%llus]\n",
+ ena_trc_dbg("AENQ! Group[%x] Syndrom[%x] timestamp: [%"PRIu64"]\n",
aenq_common->group,
aenq_common->syndrom,
timestamp);
mb();
ENA_REG_WRITE32_RELAXED(dev->bus, (u32)aenq->head,
dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF);
+#ifndef MMIOWB_NOT_DEFINED
mmiowb();
+#endif
}
int ena_com_dev_reset(struct ena_com_dev *ena_dev,
if (unlikely(ret))
return ret;
- if (get_resp.u.flow_hash_func.supported_func & (1 << rss->hash_func)) {
+ if (!(get_resp.u.flow_hash_func.supported_func & BIT(rss->hash_func))) {
ena_trc_err("Func hash %d isn't supported by device, abort\n",
rss->hash_func);
return ENA_COM_UNSUPPORTED;
return ENA_COM_INVAL;
}
+ rss->hash_func = func;
rc = ena_com_set_hash_function(ena_dev);
/* Restore the old function */
void ena_com_destroy_interrupt_moderation(struct ena_com_dev *ena_dev)
{
if (ena_dev->intr_moder_tbl)
- ENA_MEM_FREE(ena_dev->dmadev, ena_dev->intr_moder_tbl);
+ ENA_MEM_FREE(ena_dev->dmadev,
+ ena_dev->intr_moder_tbl,
+ (sizeof(struct ena_intr_moder_entry) * ENA_INTR_MAX_NUM_OF_LEVELS));
ena_dev->intr_moder_tbl = NULL;
}
/* if moderation is supported by device we set adaptive moderation */
delay_resolution = get_resp.u.intr_moderation.intr_delay_resolution;
ena_com_update_intr_delay_resolution(ena_dev, delay_resolution);
- ena_com_enable_adaptive_moderation(ena_dev);
+
+ /* Disable adaptive moderation by default - can be enabled later */
+ ena_com_disable_adaptive_moderation(ena_dev);
return 0;
err:
struct ena_llq_configurations *llq_default_cfg)
{
int rc;
- int size;
+ struct ena_com_llq_info *llq_info = &(ena_dev->llq_info);;
if (!llq_features->max_llq_num) {
ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
if (rc)
return rc;
- /* Validate the descriptor is not too big */
- size = ena_dev->tx_max_header_size;
- size += ena_dev->llq_info.descs_num_before_header *
- sizeof(struct ena_eth_io_tx_desc);
+ ena_dev->tx_max_header_size = llq_info->desc_list_entry_size -
+ (llq_info->descs_num_before_header * sizeof(struct ena_eth_io_tx_desc));
- if (unlikely(ena_dev->llq_info.desc_list_entry_size < size)) {
+ if (ena_dev->tx_max_header_size == 0) {
ena_trc_err("the size of the LLQ entry is smaller than needed\n");
- return ENA_COM_INVAL;
+ return -EINVAL;
}
ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_DEV;
git.droids-corp.org / dpdk.git / blobdiff