drivers/net/ena/ena_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
   3  * All rights reserved.
   4  */
   5
   6 #include <rte_string_fns.h>
   7 #include <rte_errno.h>
   8 #include <rte_version.h>
   9 #include <rte_net.h>
  10 #include <rte_kvargs.h>
  11
  12 #include "ena_ethdev.h"
  13 #include "ena_logs.h"
  14 #include "ena_platform.h"
  15 #include "ena_com.h"
  16 #include "ena_eth_com.h"
  17
  18 #include <ena_common_defs.h>
  19 #include <ena_regs_defs.h>
  20 #include <ena_admin_defs.h>
  21 #include <ena_eth_io_defs.h>
  22
  23 #define DRV_MODULE_VER_MAJOR    2
  24 #define DRV_MODULE_VER_MINOR    5
  25 #define DRV_MODULE_VER_SUBMINOR 0
  26
  27 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l)
  28
  29 #define GET_L4_HDR_LEN(mbuf)                                    \
  30         ((rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *,   \
  31                 mbuf->l3_len + mbuf->l2_len)->data_off) >> 4)
  32
  33 #define ETH_GSTRING_LEN 32
  34
  35 #define ARRAY_SIZE(x) RTE_DIM(x)
  36
  37 #define ENA_MIN_RING_DESC       128
  38
  39 #define ENA_PTYPE_HAS_HASH      (RTE_PTYPE_L4_TCP | RTE_PTYPE_L4_UDP)
  40
  41 struct ena_stats {
  42         char name[ETH_GSTRING_LEN];
  43         int stat_offset;
  44 };
  45
  46 #define ENA_STAT_ENTRY(stat, stat_type) { \
  47         .name = #stat, \
  48         .stat_offset = offsetof(struct ena_stats_##stat_type, stat) \
  49 }
  50
  51 #define ENA_STAT_RX_ENTRY(stat) \
  52         ENA_STAT_ENTRY(stat, rx)
  53
  54 #define ENA_STAT_TX_ENTRY(stat) \
  55         ENA_STAT_ENTRY(stat, tx)
  56
  57 #define ENA_STAT_ENI_ENTRY(stat) \
  58         ENA_STAT_ENTRY(stat, eni)
  59
  60 #define ENA_STAT_GLOBAL_ENTRY(stat) \
  61         ENA_STAT_ENTRY(stat, dev)
  62
  63 /* Device arguments */
  64 #define ENA_DEVARG_LARGE_LLQ_HDR "large_llq_hdr"
  65
  66 /*
  67  * Each rte_memzone should have unique name.
  68  * To satisfy it, count number of allocation and add it to name.
  69  */
  70 rte_atomic64_t ena_alloc_cnt;
  71
  72 static const struct ena_stats ena_stats_global_strings[] = {
  73         ENA_STAT_GLOBAL_ENTRY(wd_expired),
  74         ENA_STAT_GLOBAL_ENTRY(dev_start),
  75         ENA_STAT_GLOBAL_ENTRY(dev_stop),
  76         ENA_STAT_GLOBAL_ENTRY(tx_drops),
  77 };
  78
  79 static const struct ena_stats ena_stats_eni_strings[] = {
  80         ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded),
  81         ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded),
  82         ENA_STAT_ENI_ENTRY(pps_allowance_exceeded),
  83         ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded),
  84         ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded),
  85 };
  86
  87 static const struct ena_stats ena_stats_tx_strings[] = {
  88         ENA_STAT_TX_ENTRY(cnt),
  89         ENA_STAT_TX_ENTRY(bytes),
  90         ENA_STAT_TX_ENTRY(prepare_ctx_err),
  91         ENA_STAT_TX_ENTRY(tx_poll),
  92         ENA_STAT_TX_ENTRY(doorbells),
  93         ENA_STAT_TX_ENTRY(bad_req_id),
  94         ENA_STAT_TX_ENTRY(available_desc),
  95         ENA_STAT_TX_ENTRY(missed_tx),
  96 };
  97
  98 static const struct ena_stats ena_stats_rx_strings[] = {
  99         ENA_STAT_RX_ENTRY(cnt),
 100         ENA_STAT_RX_ENTRY(bytes),
 101         ENA_STAT_RX_ENTRY(refill_partial),
 102         ENA_STAT_RX_ENTRY(l3_csum_bad),
 103         ENA_STAT_RX_ENTRY(l4_csum_bad),
 104         ENA_STAT_RX_ENTRY(l4_csum_good),
 105         ENA_STAT_RX_ENTRY(mbuf_alloc_fail),
 106         ENA_STAT_RX_ENTRY(bad_desc_num),
 107         ENA_STAT_RX_ENTRY(bad_req_id),
 108 };
 109
 110 #define ENA_STATS_ARRAY_GLOBAL  ARRAY_SIZE(ena_stats_global_strings)
 111 #define ENA_STATS_ARRAY_ENI     ARRAY_SIZE(ena_stats_eni_strings)
 112 #define ENA_STATS_ARRAY_TX      ARRAY_SIZE(ena_stats_tx_strings)
 113 #define ENA_STATS_ARRAY_RX      ARRAY_SIZE(ena_stats_rx_strings)
 114
 115 #define QUEUE_OFFLOADS (RTE_ETH_TX_OFFLOAD_TCP_CKSUM |\
 116                         RTE_ETH_TX_OFFLOAD_UDP_CKSUM |\
 117                         RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |\
 118                         RTE_ETH_TX_OFFLOAD_TCP_TSO)
 119 #define MBUF_OFFLOADS (RTE_MBUF_F_TX_L4_MASK |\
 120                        RTE_MBUF_F_TX_IP_CKSUM |\
 121                        RTE_MBUF_F_TX_TCP_SEG)
 122
 123 /** Vendor ID used by Amazon devices */
 124 #define PCI_VENDOR_ID_AMAZON 0x1D0F
 125 /** Amazon devices */
 126 #define PCI_DEVICE_ID_ENA_VF            0xEC20
 127 #define PCI_DEVICE_ID_ENA_VF_RSERV0     0xEC21
 128
 129 #define ENA_TX_OFFLOAD_MASK     (RTE_MBUF_F_TX_L4_MASK |         \
 130         RTE_MBUF_F_TX_IPV6 |            \
 131         RTE_MBUF_F_TX_IPV4 |            \
 132         RTE_MBUF_F_TX_IP_CKSUM |        \
 133         RTE_MBUF_F_TX_TCP_SEG)
 134
 135 #define ENA_TX_OFFLOAD_NOTSUP_MASK      \
 136         (RTE_MBUF_F_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK)
 137
 138 /** HW specific offloads capabilities. */
 139 /* IPv4 checksum offload. */
 140 #define ENA_L3_IPV4_CSUM                0x0001
 141 /* TCP/UDP checksum offload for IPv4 packets. */
 142 #define ENA_L4_IPV4_CSUM                0x0002
 143 /* TCP/UDP checksum offload for IPv4 packets with pseudo header checksum. */
 144 #define ENA_L4_IPV4_CSUM_PARTIAL        0x0004
 145 /* TCP/UDP checksum offload for IPv6 packets. */
 146 #define ENA_L4_IPV6_CSUM                0x0008
 147 /* TCP/UDP checksum offload for IPv6 packets with pseudo header checksum. */
 148 #define ENA_L4_IPV6_CSUM_PARTIAL        0x0010
 149 /* TSO support for IPv4 packets. */
 150 #define ENA_IPV4_TSO                    0x0020
 151
 152 /* Device supports setting RSS hash. */
 153 #define ENA_RX_RSS_HASH                 0x0040
 154
 155 static const struct rte_pci_id pci_id_ena_map[] = {
 156         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) },
 157         { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF_RSERV0) },
 158         { .device_id = 0 },
 159 };
 160
 161 static struct ena_aenq_handlers aenq_handlers;
 162
 163 static int ena_device_init(struct ena_adapter *adapter,
 164                            struct rte_pci_device *pdev,
 165                            struct ena_com_dev_get_features_ctx *get_feat_ctx);
 166 static int ena_dev_configure(struct rte_eth_dev *dev);
 167 static void ena_tx_map_mbuf(struct ena_ring *tx_ring,
 168         struct ena_tx_buffer *tx_info,
 169         struct rte_mbuf *mbuf,
 170         void **push_header,
 171         uint16_t *header_len);
 172 static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf);
 173 static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt);
 174 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 175                                   uint16_t nb_pkts);
 176 static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 177                 uint16_t nb_pkts);
 178 static int ena_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 179                               uint16_t nb_desc, unsigned int socket_id,
 180                               const struct rte_eth_txconf *tx_conf);
 181 static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 182                               uint16_t nb_desc, unsigned int socket_id,
 183                               const struct rte_eth_rxconf *rx_conf,
 184                               struct rte_mempool *mp);
 185 static inline void ena_init_rx_mbuf(struct rte_mbuf *mbuf, uint16_t len);
 186 static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring,
 187                                     struct ena_com_rx_buf_info *ena_bufs,
 188                                     uint32_t descs,
 189                                     uint16_t *next_to_clean,
 190                                     uint8_t offset);
 191 static uint16_t eth_ena_recv_pkts(void *rx_queue,
 192                                   struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
 193 static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq,
 194                                   struct rte_mbuf *mbuf, uint16_t id);
 195 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count);
 196 static void ena_init_rings(struct ena_adapter *adapter,
 197                            bool disable_meta_caching);
 198 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 199 static int ena_start(struct rte_eth_dev *dev);
 200 static int ena_stop(struct rte_eth_dev *dev);
 201 static int ena_close(struct rte_eth_dev *dev);
 202 static int ena_dev_reset(struct rte_eth_dev *dev);
 203 static int ena_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
 204 static void ena_rx_queue_release_all(struct rte_eth_dev *dev);
 205 static void ena_tx_queue_release_all(struct rte_eth_dev *dev);
 206 static void ena_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
 207 static void ena_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid);
 208 static void ena_rx_queue_release_bufs(struct ena_ring *ring);
 209 static void ena_tx_queue_release_bufs(struct ena_ring *ring);
 210 static int ena_link_update(struct rte_eth_dev *dev,
 211                            int wait_to_complete);
 212 static int ena_create_io_queue(struct rte_eth_dev *dev, struct ena_ring *ring);
 213 static void ena_queue_stop(struct ena_ring *ring);
 214 static void ena_queue_stop_all(struct rte_eth_dev *dev,
 215                               enum ena_ring_type ring_type);
 216 static int ena_queue_start(struct rte_eth_dev *dev, struct ena_ring *ring);
 217 static int ena_queue_start_all(struct rte_eth_dev *dev,
 218                                enum ena_ring_type ring_type);
 219 static void ena_stats_restart(struct rte_eth_dev *dev);
 220 static uint64_t ena_get_rx_port_offloads(struct ena_adapter *adapter);
 221 static uint64_t ena_get_tx_port_offloads(struct ena_adapter *adapter);
 222 static uint64_t ena_get_rx_queue_offloads(struct ena_adapter *adapter);
 223 static uint64_t ena_get_tx_queue_offloads(struct ena_adapter *adapter);
 224 static int ena_infos_get(struct rte_eth_dev *dev,
 225                          struct rte_eth_dev_info *dev_info);
 226 static void ena_interrupt_handler_rte(void *cb_arg);
 227 static void ena_timer_wd_callback(struct rte_timer *timer, void *arg);
 228 static void ena_destroy_device(struct rte_eth_dev *eth_dev);
 229 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev);
 230 static int ena_xstats_get_names(struct rte_eth_dev *dev,
 231                                 struct rte_eth_xstat_name *xstats_names,
 232                                 unsigned int n);
 233 static int ena_xstats_get_names_by_id(struct rte_eth_dev *dev,
 234                                       const uint64_t *ids,
 235                                       struct rte_eth_xstat_name *xstats_names,
 236                                       unsigned int size);
 237 static int ena_xstats_get(struct rte_eth_dev *dev,
 238                           struct rte_eth_xstat *stats,
 239                           unsigned int n);
 240 static int ena_xstats_get_by_id(struct rte_eth_dev *dev,
 241                                 const uint64_t *ids,
 242                                 uint64_t *values,
 243                                 unsigned int n);
 244 static int ena_process_bool_devarg(const char *key,
 245                                    const char *value,
 246                                    void *opaque);
 247 static int ena_parse_devargs(struct ena_adapter *adapter,
 248                              struct rte_devargs *devargs);
 249 static int ena_copy_eni_stats(struct ena_adapter *adapter,
 250                               struct ena_stats_eni *stats);
 251 static int ena_setup_rx_intr(struct rte_eth_dev *dev);
 252 static int ena_rx_queue_intr_enable(struct rte_eth_dev *dev,
 253                                     uint16_t queue_id);
 254 static int ena_rx_queue_intr_disable(struct rte_eth_dev *dev,
 255                                      uint16_t queue_id);
 256 static int ena_configure_aenq(struct ena_adapter *adapter);
 257 static int ena_mp_primary_handle(const struct rte_mp_msg *mp_msg,
 258                                  const void *peer);
 259
 260 static const struct eth_dev_ops ena_dev_ops = {
 261         .dev_configure          = ena_dev_configure,
 262         .dev_infos_get          = ena_infos_get,
 263         .rx_queue_setup         = ena_rx_queue_setup,
 264         .tx_queue_setup         = ena_tx_queue_setup,
 265         .dev_start              = ena_start,
 266         .dev_stop               = ena_stop,
 267         .link_update            = ena_link_update,
 268         .stats_get              = ena_stats_get,
 269         .xstats_get_names       = ena_xstats_get_names,
 270         .xstats_get_names_by_id = ena_xstats_get_names_by_id,
 271         .xstats_get             = ena_xstats_get,
 272         .xstats_get_by_id       = ena_xstats_get_by_id,
 273         .mtu_set                = ena_mtu_set,
 274         .rx_queue_release       = ena_rx_queue_release,
 275         .tx_queue_release       = ena_tx_queue_release,
 276         .dev_close              = ena_close,
 277         .dev_reset              = ena_dev_reset,
 278         .reta_update            = ena_rss_reta_update,
 279         .reta_query             = ena_rss_reta_query,
 280         .rx_queue_intr_enable   = ena_rx_queue_intr_enable,
 281         .rx_queue_intr_disable  = ena_rx_queue_intr_disable,
 282         .rss_hash_update        = ena_rss_hash_update,
 283         .rss_hash_conf_get      = ena_rss_hash_conf_get,
 284         .tx_done_cleanup        = ena_tx_cleanup,
 285 };
 286
 287 /*********************************************************************
 288  *  Multi-Process communication bits
 289  *********************************************************************/
 290 /* rte_mp IPC message name */
 291 #define ENA_MP_NAME     "net_ena_mp"
 292 /* Request timeout in seconds */
 293 #define ENA_MP_REQ_TMO  5
 294
 295 /** Proxy request type */
 296 enum ena_mp_req {
 297         ENA_MP_DEV_STATS_GET,
 298         ENA_MP_ENI_STATS_GET,
 299         ENA_MP_MTU_SET,
 300         ENA_MP_IND_TBL_GET,
 301         ENA_MP_IND_TBL_SET
 302 };
 303
 304 /** Proxy message body. Shared between requests and responses. */
 305 struct ena_mp_body {
 306         /* Message type */
 307         enum ena_mp_req type;
 308         int port_id;
 309         /* Processing result. Set in replies. 0 if message succeeded, negative
 310          * error code otherwise.
 311          */
 312         int result;
 313         union {
 314                 int mtu; /* For ENA_MP_MTU_SET */
 315         } args;
 316 };
 317
 318 /**
 319  * Initialize IPC message.
 320  *
 321  * @param[out] msg
 322  *   Pointer to the message to initialize.
 323  * @param[in] type
 324  *   Message type.
 325  * @param[in] port_id
 326  *   Port ID of target device.
 327  *
 328  */
 329 static void
 330 mp_msg_init(struct rte_mp_msg *msg, enum ena_mp_req type, int port_id)
 331 {
 332         struct ena_mp_body *body = (struct ena_mp_body *)&msg->param;
 333
 334         memset(msg, 0, sizeof(*msg));
 335         strlcpy(msg->name, ENA_MP_NAME, sizeof(msg->name));
 336         msg->len_param = sizeof(*body);
 337         body->type = type;
 338         body->port_id = port_id;
 339 }
 340
 341 /*********************************************************************
 342  *  Multi-Process communication PMD API
 343  *********************************************************************/
 344 /**
 345  * Define proxy request descriptor
 346  *
 347  * Used to define all structures and functions required for proxying a given
 348  * function to the primary process including the code to perform to prepare the
 349  * request and process the response.
 350  *
 351  * @param[in] f
 352  *   Name of the function to proxy
 353  * @param[in] t
 354  *   Message type to use
 355  * @param[in] prep
 356  *   Body of a function to prepare the request in form of a statement
 357  *   expression. It is passed all the original function arguments along with two
 358  *   extra ones:
 359  *   - struct ena_adapter *adapter - PMD data of the device calling the proxy.
 360  *   - struct ena_mp_body *req - body of a request to prepare.
 361  * @param[in] proc
 362  *   Body of a function to process the response in form of a statement
 363  *   expression. It is passed all the original function arguments along with two
 364  *   extra ones:
 365  *   - struct ena_adapter *adapter - PMD data of the device calling the proxy.
 366  *   - struct ena_mp_body *rsp - body of a response to process.
 367  * @param ...
 368  *   Proxied function's arguments
 369  *
 370  * @note Inside prep and proc any parameters which aren't used should be marked
 371  *       as such (with ENA_TOUCH or __rte_unused).
 372  */
 373 #define ENA_PROXY_DESC(f, t, prep, proc, ...)                   \
 374         static const enum ena_mp_req mp_type_ ## f =  t;        \
 375         static const char *mp_name_ ## f = #t;                  \
 376         static void mp_prep_ ## f(struct ena_adapter *adapter,  \
 377                                   struct ena_mp_body *req,      \
 378                                   __VA_ARGS__)                  \
 379         {                                                       \
 380                 prep;                                           \
 381         }                                                       \
 382         static void mp_proc_ ## f(struct ena_adapter *adapter,  \
 383                                   struct ena_mp_body *rsp,      \
 384                                   __VA_ARGS__)                  \
 385         {                                                       \
 386                 proc;                                           \
 387         }
 388
 389 /**
 390  * Proxy wrapper for calling primary functions in a secondary process.
 391  *
 392  * Depending on whether called in primary or secondary process, calls the
 393  * @p func directly or proxies the call to the primary process via rte_mp IPC.
 394  * This macro requires a proxy request descriptor to be defined for @p func
 395  * using ENA_PROXY_DESC() macro.
 396  *
 397  * @param[in/out] a
 398  *   Device PMD data. Used for sending the message and sharing message results
 399  *   between primary and secondary.
 400  * @param[in] f
 401  *   Function to proxy.
 402  * @param ...
 403  *   Arguments of @p func.
 404  *
 405  * @return
 406  *   - 0: Processing succeeded and response handler was called.
 407  *   - -EPERM: IPC is unavailable on this platform. This means only primary
 408  *             process may call the proxied function.
 409  *   - -EIO:   IPC returned error on request send. Inspect rte_errno detailed
 410  *             error code.
 411  *   - Negative error code from the proxied function.
 412  *
 413  * @note This mechanism is geared towards control-path tasks. Avoid calling it
 414  *       in fast-path unless unbound delays are allowed. This is due to the IPC
 415  *       mechanism itself (socket based).
 416  * @note Due to IPC parameter size limitations the proxy logic shares call
 417  *       results through the struct ena_adapter shared memory. This makes the
 418  *       proxy mechanism strictly single-threaded. Therefore be sure to make all
 419  *       calls to the same proxied function under the same lock.
 420  */
 421 #define ENA_PROXY(a, f, ...)                                            \
 422 ({                                                                      \
 423         struct ena_adapter *_a = (a);                                   \
 424         struct timespec ts = { .tv_sec = ENA_MP_REQ_TMO };              \
 425         struct ena_mp_body *req, *rsp;                                  \
 426         struct rte_mp_reply mp_rep;                                     \
 427         struct rte_mp_msg mp_req;                                       \
 428         int ret;                                                        \
 429                                                                         \
 430         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {               \
 431                 ret = f(__VA_ARGS__);                                   \
 432         } else {                                                        \
 433                 /* Prepare and send request */                          \
 434                 req = (struct ena_mp_body *)&mp_req.param;              \
 435                 mp_msg_init(&mp_req, mp_type_ ## f, _a->edev_data->port_id); \
 436                 mp_prep_ ## f(_a, req, ## __VA_ARGS__);                 \
 437                                                                         \
 438                 ret = rte_mp_request_sync(&mp_req, &mp_rep, &ts);       \
 439                 if (likely(!ret)) {                                     \
 440                         RTE_ASSERT(mp_rep.nb_received == 1);            \
 441                         rsp = (struct ena_mp_body *)&mp_rep.msgs[0].param; \
 442                         ret = rsp->result;                              \
 443                         if (ret == 0) {                                 \
 444                                 mp_proc_##f(_a, rsp, ## __VA_ARGS__);   \
 445                         } else {                                        \
 446                                 PMD_DRV_LOG(ERR,                        \
 447                                             "%s returned error: %d\n",  \
 448                                             mp_name_ ## f, rsp->result);\
 449                         }                                               \
 450                         free(mp_rep.msgs);                              \
 451                 } else if (rte_errno == ENOTSUP) {                      \
 452                         PMD_DRV_LOG(ERR,                                \
 453                                     "No IPC, can't proxy to primary\n");\
 454                         ret = -rte_errno;                               \
 455                 } else {                                                \
 456                         PMD_DRV_LOG(ERR, "Request %s failed: %s\n",     \
 457                                     mp_name_ ## f,                      \
 458                                     rte_strerror(rte_errno));           \
 459                         ret = -EIO;                                     \
 460                 }                                                       \
 461         }                                                               \
 462         ret;                                                            \
 463 })
 464
 465 /*********************************************************************
 466  *  Multi-Process communication request descriptors
 467  *********************************************************************/
 468
 469 ENA_PROXY_DESC(ena_com_get_dev_basic_stats, ENA_MP_DEV_STATS_GET,
 470 ({
 471         ENA_TOUCH(adapter);
 472         ENA_TOUCH(req);
 473         ENA_TOUCH(ena_dev);
 474         ENA_TOUCH(stats);
 475 }),
 476 ({
 477         ENA_TOUCH(rsp);
 478         ENA_TOUCH(ena_dev);
 479         if (stats != &adapter->basic_stats)
 480                 rte_memcpy(stats, &adapter->basic_stats, sizeof(*stats));
 481 }),
 482         struct ena_com_dev *ena_dev, struct ena_admin_basic_stats *stats);
 483
 484 ENA_PROXY_DESC(ena_com_get_eni_stats, ENA_MP_ENI_STATS_GET,
 485 ({
 486         ENA_TOUCH(adapter);
 487         ENA_TOUCH(req);
 488         ENA_TOUCH(ena_dev);
 489         ENA_TOUCH(stats);
 490 }),
 491 ({
 492         ENA_TOUCH(rsp);
 493         ENA_TOUCH(ena_dev);
 494         if (stats != (struct ena_admin_eni_stats *)&adapter->eni_stats)
 495                 rte_memcpy(stats, &adapter->eni_stats, sizeof(*stats));
 496 }),
 497         struct ena_com_dev *ena_dev, struct ena_admin_eni_stats *stats);
 498
 499 ENA_PROXY_DESC(ena_com_set_dev_mtu, ENA_MP_MTU_SET,
 500 ({
 501         ENA_TOUCH(adapter);
 502         ENA_TOUCH(ena_dev);
 503         req->args.mtu = mtu;
 504 }),
 505 ({
 506         ENA_TOUCH(adapter);
 507         ENA_TOUCH(rsp);
 508         ENA_TOUCH(ena_dev);
 509         ENA_TOUCH(mtu);
 510 }),
 511         struct ena_com_dev *ena_dev, int mtu);
 512
 513 ENA_PROXY_DESC(ena_com_indirect_table_set, ENA_MP_IND_TBL_SET,
 514 ({
 515         ENA_TOUCH(adapter);
 516         ENA_TOUCH(req);
 517         ENA_TOUCH(ena_dev);
 518 }),
 519 ({
 520         ENA_TOUCH(adapter);
 521         ENA_TOUCH(rsp);
 522         ENA_TOUCH(ena_dev);
 523 }),
 524         struct ena_com_dev *ena_dev);
 525
 526 ENA_PROXY_DESC(ena_com_indirect_table_get, ENA_MP_IND_TBL_GET,
 527 ({
 528         ENA_TOUCH(adapter);
 529         ENA_TOUCH(req);
 530         ENA_TOUCH(ena_dev);
 531         ENA_TOUCH(ind_tbl);
 532 }),
 533 ({
 534         ENA_TOUCH(rsp);
 535         ENA_TOUCH(ena_dev);
 536         if (ind_tbl != adapter->indirect_table)
 537                 rte_memcpy(ind_tbl, adapter->indirect_table,
 538                            sizeof(adapter->indirect_table));
 539 }),
 540         struct ena_com_dev *ena_dev, u32 *ind_tbl);
 541
 542 static inline void ena_rx_mbuf_prepare(struct ena_ring *rx_ring,
 543                                        struct rte_mbuf *mbuf,
 544                                        struct ena_com_rx_ctx *ena_rx_ctx,
 545                                        bool fill_hash)
 546 {
 547         struct ena_stats_rx *rx_stats = &rx_ring->rx_stats;
 548         uint64_t ol_flags = 0;
 549         uint32_t packet_type = 0;
 550
 551         if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP)
 552                 packet_type |= RTE_PTYPE_L4_TCP;
 553         else if (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)
 554                 packet_type |= RTE_PTYPE_L4_UDP;
 555
 556         if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) {
 557                 packet_type |= RTE_PTYPE_L3_IPV4;
 558                 if (unlikely(ena_rx_ctx->l3_csum_err)) {
 559                         ++rx_stats->l3_csum_bad;
 560                         ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD;
 561                 } else {
 562                         ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
 563                 }
 564         } else if (ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV6) {
 565                 packet_type |= RTE_PTYPE_L3_IPV6;
 566         }
 567
 568         if (!ena_rx_ctx->l4_csum_checked || ena_rx_ctx->frag) {
 569                 ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN;
 570         } else {
 571                 if (unlikely(ena_rx_ctx->l4_csum_err)) {
 572                         ++rx_stats->l4_csum_bad;
 573                         ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
 574                 } else {
 575                         ++rx_stats->l4_csum_good;
 576                         ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
 577                 }
 578         }
 579
 580         if (fill_hash &&
 581             likely((packet_type & ENA_PTYPE_HAS_HASH) && !ena_rx_ctx->frag)) {
 582                 ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
 583                 mbuf->hash.rss = ena_rx_ctx->hash;
 584         }
 585
 586         mbuf->ol_flags = ol_flags;
 587         mbuf->packet_type = packet_type;
 588 }
 589
 590 static inline void ena_tx_mbuf_prepare(struct rte_mbuf *mbuf,
 591                                        struct ena_com_tx_ctx *ena_tx_ctx,
 592                                        uint64_t queue_offloads,
 593                                        bool disable_meta_caching)
 594 {
 595         struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
 596
 597         if ((mbuf->ol_flags & MBUF_OFFLOADS) &&
 598             (queue_offloads & QUEUE_OFFLOADS)) {
 599                 /* check if TSO is required */
 600                 if ((mbuf->ol_flags & RTE_MBUF_F_TX_TCP_SEG) &&
 601                     (queue_offloads & RTE_ETH_TX_OFFLOAD_TCP_TSO)) {
 602                         ena_tx_ctx->tso_enable = true;
 603
 604                         ena_meta->l4_hdr_len = GET_L4_HDR_LEN(mbuf);
 605                 }
 606
 607                 /* check if L3 checksum is needed */
 608                 if ((mbuf->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) &&
 609                     (queue_offloads & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM))
 610                         ena_tx_ctx->l3_csum_enable = true;
 611
 612                 if (mbuf->ol_flags & RTE_MBUF_F_TX_IPV6) {
 613                         ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
 614                 } else {
 615                         ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
 616
 617                         /* set don't fragment (DF) flag */
 618                         if (mbuf->packet_type &
 619                                 (RTE_PTYPE_L4_NONFRAG
 620                                  | RTE_PTYPE_INNER_L4_NONFRAG))
 621                                 ena_tx_ctx->df = true;
 622                 }
 623
 624                 /* check if L4 checksum is needed */
 625                 if (((mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) == RTE_MBUF_F_TX_TCP_CKSUM) &&
 626                     (queue_offloads & RTE_ETH_TX_OFFLOAD_TCP_CKSUM)) {
 627                         ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
 628                         ena_tx_ctx->l4_csum_enable = true;
 629                 } else if (((mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK) ==
 630                                 RTE_MBUF_F_TX_UDP_CKSUM) &&
 631                                 (queue_offloads & RTE_ETH_TX_OFFLOAD_UDP_CKSUM)) {
 632                         ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
 633                         ena_tx_ctx->l4_csum_enable = true;
 634                 } else {
 635                         ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN;
 636                         ena_tx_ctx->l4_csum_enable = false;
 637                 }
 638
 639                 ena_meta->mss = mbuf->tso_segsz;
 640                 ena_meta->l3_hdr_len = mbuf->l3_len;
 641                 ena_meta->l3_hdr_offset = mbuf->l2_len;
 642
 643                 ena_tx_ctx->meta_valid = true;
 644         } else if (disable_meta_caching) {
 645                 memset(ena_meta, 0, sizeof(*ena_meta));
 646                 ena_tx_ctx->meta_valid = true;
 647         } else {
 648                 ena_tx_ctx->meta_valid = false;
 649         }
 650 }
 651
 652 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
 653 {
 654         struct ena_tx_buffer *tx_info = NULL;
 655
 656         if (likely(req_id < tx_ring->ring_size)) {
 657                 tx_info = &tx_ring->tx_buffer_info[req_id];
 658                 if (likely(tx_info->mbuf))
 659                         return 0;
 660         }
 661
 662         if (tx_info)
 663                 PMD_TX_LOG(ERR, "tx_info doesn't have valid mbuf\n");
 664         else
 665                 PMD_TX_LOG(ERR, "Invalid req_id: %hu\n", req_id);
 666
 667         /* Trigger device reset */
 668         ++tx_ring->tx_stats.bad_req_id;
 669         tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
 670         tx_ring->adapter->trigger_reset = true;
 671         return -EFAULT;
 672 }
 673
 674 static void ena_config_host_info(struct ena_com_dev *ena_dev)
 675 {
 676         struct ena_admin_host_info *host_info;
 677         int rc;
 678
 679         /* Allocate only the host info */
 680         rc = ena_com_allocate_host_info(ena_dev);
 681         if (rc) {
 682                 PMD_DRV_LOG(ERR, "Cannot allocate host info\n");
 683                 return;
 684         }
 685
 686         host_info = ena_dev->host_attr.host_info;
 687
 688         host_info->os_type = ENA_ADMIN_OS_DPDK;
 689         host_info->kernel_ver = RTE_VERSION;
 690         strlcpy((char *)host_info->kernel_ver_str, rte_version(),
 691                 sizeof(host_info->kernel_ver_str));
 692         host_info->os_dist = RTE_VERSION;
 693         strlcpy((char *)host_info->os_dist_str, rte_version(),
 694                 sizeof(host_info->os_dist_str));
 695         host_info->driver_version =
 696                 (DRV_MODULE_VER_MAJOR) |
 697                 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
 698                 (DRV_MODULE_VER_SUBMINOR <<
 699                         ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
 700         host_info->num_cpus = rte_lcore_count();
 701
 702         host_info->driver_supported_features =
 703                 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |
 704                 ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK;
 705
 706         rc = ena_com_set_host_attributes(ena_dev);
 707         if (rc) {
 708                 if (rc == -ENA_COM_UNSUPPORTED)
 709                         PMD_DRV_LOG(WARNING, "Cannot set host attributes\n");
 710                 else
 711                         PMD_DRV_LOG(ERR, "Cannot set host attributes\n");
 712
 713                 goto err;
 714         }
 715
 716         return;
 717
 718 err:
 719         ena_com_delete_host_info(ena_dev);
 720 }
 721
 722 /* This function calculates the number of xstats based on the current config */
 723 static unsigned int ena_xstats_calc_num(struct rte_eth_dev_data *data)
 724 {
 725         return ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENI +
 726                 (data->nb_tx_queues * ENA_STATS_ARRAY_TX) +
 727                 (data->nb_rx_queues * ENA_STATS_ARRAY_RX);
 728 }
 729
 730 static void ena_config_debug_area(struct ena_adapter *adapter)
 731 {
 732         u32 debug_area_size;
 733         int rc, ss_count;
 734
 735         ss_count = ena_xstats_calc_num(adapter->edev_data);
 736
 737         /* allocate 32 bytes for each string and 64bit for the value */
 738         debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;
 739
 740         rc = ena_com_allocate_debug_area(&adapter->ena_dev, debug_area_size);
 741         if (rc) {
 742                 PMD_DRV_LOG(ERR, "Cannot allocate debug area\n");
 743                 return;
 744         }
 745
 746         rc = ena_com_set_host_attributes(&adapter->ena_dev);
 747         if (rc) {
 748                 if (rc == -ENA_COM_UNSUPPORTED)
 749                         PMD_DRV_LOG(WARNING, "Cannot set host attributes\n");
 750                 else
 751                         PMD_DRV_LOG(ERR, "Cannot set host attributes\n");
 752
 753                 goto err;
 754         }
 755
 756         return;
 757 err:
 758         ena_com_delete_debug_area(&adapter->ena_dev);
 759 }
 760
 761 static int ena_close(struct rte_eth_dev *dev)
 762 {
 763         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 764         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 765         struct ena_adapter *adapter = dev->data->dev_private;
 766         int ret = 0;
 767
 768         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 769                 return 0;
 770
 771         if (adapter->state == ENA_ADAPTER_STATE_RUNNING)
 772                 ret = ena_stop(dev);
 773         adapter->state = ENA_ADAPTER_STATE_CLOSED;
 774
 775         ena_rx_queue_release_all(dev);
 776         ena_tx_queue_release_all(dev);
 777
 778         rte_free(adapter->drv_stats);
 779         adapter->drv_stats = NULL;
 780
 781         rte_intr_disable(intr_handle);
 782         rte_intr_callback_unregister(intr_handle,
 783                                      ena_interrupt_handler_rte,
 784                                      dev);
 785
 786         /*
 787          * MAC is not allocated dynamically. Setting NULL should prevent from
 788          * release of the resource in the rte_eth_dev_release_port().
 789          */
 790         dev->data->mac_addrs = NULL;
 791
 792         return ret;
 793 }
 794
 795 static int
 796 ena_dev_reset(struct rte_eth_dev *dev)
 797 {
 798         int rc = 0;
 799
 800         /* Cannot release memory in secondary process */
 801         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 802                 PMD_DRV_LOG(WARNING, "dev_reset not supported in secondary.\n");
 803                 return -EPERM;
 804         }
 805
 806         ena_destroy_device(dev);
 807         rc = eth_ena_dev_init(dev);
 808         if (rc)
 809                 PMD_INIT_LOG(CRIT, "Cannot initialize device\n");
 810
 811         return rc;
 812 }
 813
 814 static void ena_rx_queue_release_all(struct rte_eth_dev *dev)
 815 {
 816         int nb_queues = dev->data->nb_rx_queues;
 817         int i;
 818
 819         for (i = 0; i < nb_queues; i++)
 820                 ena_rx_queue_release(dev, i);
 821 }
 822
 823 static void ena_tx_queue_release_all(struct rte_eth_dev *dev)
 824 {
 825         int nb_queues = dev->data->nb_tx_queues;
 826         int i;
 827
 828         for (i = 0; i < nb_queues; i++)
 829                 ena_tx_queue_release(dev, i);
 830 }
 831
 832 static void ena_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 833 {
 834         struct ena_ring *ring = dev->data->rx_queues[qid];
 835
 836         /* Free ring resources */
 837         rte_free(ring->rx_buffer_info);
 838         ring->rx_buffer_info = NULL;
 839
 840         rte_free(ring->rx_refill_buffer);
 841         ring->rx_refill_buffer = NULL;
 842
 843         rte_free(ring->empty_rx_reqs);
 844         ring->empty_rx_reqs = NULL;
 845
 846         ring->configured = 0;
 847
 848         PMD_DRV_LOG(NOTICE, "Rx queue %d:%d released\n",
 849                 ring->port_id, ring->id);
 850 }
 851
 852 static void ena_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 853 {
 854         struct ena_ring *ring = dev->data->tx_queues[qid];
 855
 856         /* Free ring resources */
 857         rte_free(ring->push_buf_intermediate_buf);
 858
 859         rte_free(ring->tx_buffer_info);
 860
 861         rte_free(ring->empty_tx_reqs);
 862
 863         ring->empty_tx_reqs = NULL;
 864         ring->tx_buffer_info = NULL;
 865         ring->push_buf_intermediate_buf = NULL;
 866
 867         ring->configured = 0;
 868
 869         PMD_DRV_LOG(NOTICE, "Tx queue %d:%d released\n",
 870                 ring->port_id, ring->id);
 871 }
 872
 873 static void ena_rx_queue_release_bufs(struct ena_ring *ring)
 874 {
 875         unsigned int i;
 876
 877         for (i = 0; i < ring->ring_size; ++i) {
 878                 struct ena_rx_buffer *rx_info = &ring->rx_buffer_info[i];
 879                 if (rx_info->mbuf) {
 880                         rte_mbuf_raw_free(rx_info->mbuf);
 881                         rx_info->mbuf = NULL;
 882                 }
 883         }
 884 }
 885
 886 static void ena_tx_queue_release_bufs(struct ena_ring *ring)
 887 {
 888         unsigned int i;
 889
 890         for (i = 0; i < ring->ring_size; ++i) {
 891                 struct ena_tx_buffer *tx_buf = &ring->tx_buffer_info[i];
 892
 893                 if (tx_buf->mbuf) {
 894                         rte_pktmbuf_free(tx_buf->mbuf);
 895                         tx_buf->mbuf = NULL;
 896                 }
 897         }
 898 }
 899
 900 static int ena_link_update(struct rte_eth_dev *dev,
 901                            __rte_unused int wait_to_complete)
 902 {
 903         struct rte_eth_link *link = &dev->data->dev_link;
 904         struct ena_adapter *adapter = dev->data->dev_private;
 905
 906         link->link_status = adapter->link_status ? RTE_ETH_LINK_UP : RTE_ETH_LINK_DOWN;
 907         link->link_speed = RTE_ETH_SPEED_NUM_NONE;
 908         link->link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
 909
 910         return 0;
 911 }
 912
 913 static int ena_queue_start_all(struct rte_eth_dev *dev,
 914                                enum ena_ring_type ring_type)
 915 {
 916         struct ena_adapter *adapter = dev->data->dev_private;
 917         struct ena_ring *queues = NULL;
 918         int nb_queues;
 919         int i = 0;
 920         int rc = 0;
 921
 922         if (ring_type == ENA_RING_TYPE_RX) {
 923                 queues = adapter->rx_ring;
 924                 nb_queues = dev->data->nb_rx_queues;
 925         } else {
 926                 queues = adapter->tx_ring;
 927                 nb_queues = dev->data->nb_tx_queues;
 928         }
 929         for (i = 0; i < nb_queues; i++) {
 930                 if (queues[i].configured) {
 931                         if (ring_type == ENA_RING_TYPE_RX) {
 932                                 ena_assert_msg(
 933                                         dev->data->rx_queues[i] == &queues[i],
 934                                         "Inconsistent state of Rx queues\n");
 935                         } else {
 936                                 ena_assert_msg(
 937                                         dev->data->tx_queues[i] == &queues[i],
 938                                         "Inconsistent state of Tx queues\n");
 939                         }
 940
 941                         rc = ena_queue_start(dev, &queues[i]);
 942
 943                         if (rc) {
 944                                 PMD_INIT_LOG(ERR,
 945                                         "Failed to start queue[%d] of type(%d)\n",
 946                                         i, ring_type);
 947                                 goto err;
 948                         }
 949                 }
 950         }
 951
 952         return 0;
 953
 954 err:
 955         while (i--)
 956                 if (queues[i].configured)
 957                         ena_queue_stop(&queues[i]);
 958
 959         return rc;
 960 }
 961
 962 static int ena_check_valid_conf(struct ena_adapter *adapter)
 963 {
 964         uint32_t mtu = adapter->edev_data->mtu;
 965
 966         if (mtu > adapter->max_mtu || mtu < ENA_MIN_MTU) {
 967                 PMD_INIT_LOG(ERR,
 968                         "Unsupported MTU of %d. Max MTU: %d, min MTU: %d\n",
 969                         mtu, adapter->max_mtu, ENA_MIN_MTU);
 970                 return ENA_COM_UNSUPPORTED;
 971         }
 972
 973         return 0;
 974 }
 975
 976 static int
 977 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx,
 978                        bool use_large_llq_hdr)
 979 {
 980         struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq;
 981         struct ena_com_dev *ena_dev = ctx->ena_dev;
 982         uint32_t max_tx_queue_size;
 983         uint32_t max_rx_queue_size;
 984
 985         if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
 986                 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
 987                         &ctx->get_feat_ctx->max_queue_ext.max_queue_ext;
 988                 max_rx_queue_size = RTE_MIN(max_queue_ext->max_rx_cq_depth,
 989                         max_queue_ext->max_rx_sq_depth);
 990                 max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
 991
 992                 if (ena_dev->tx_mem_queue_type ==
 993                     ENA_ADMIN_PLACEMENT_POLICY_DEV) {
 994                         max_tx_queue_size = RTE_MIN(max_tx_queue_size,
 995                                 llq->max_llq_depth);
 996                 } else {
 997                         max_tx_queue_size = RTE_MIN(max_tx_queue_size,
 998                                 max_queue_ext->max_tx_sq_depth);
 999                 }
1000
1001                 ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
1002                         max_queue_ext->max_per_packet_rx_descs);
1003                 ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
1004                         max_queue_ext->max_per_packet_tx_descs);
1005         } else {
1006                 struct ena_admin_queue_feature_desc *max_queues =
1007                         &ctx->get_feat_ctx->max_queues;
1008                 max_rx_queue_size = RTE_MIN(max_queues->max_cq_depth,
1009                         max_queues->max_sq_depth);
1010                 max_tx_queue_size = max_queues->max_cq_depth;
1011
1012                 if (ena_dev->tx_mem_queue_type ==
1013                     ENA_ADMIN_PLACEMENT_POLICY_DEV) {
1014                         max_tx_queue_size = RTE_MIN(max_tx_queue_size,
1015                                 llq->max_llq_depth);
1016                 } else {
1017                         max_tx_queue_size = RTE_MIN(max_tx_queue_size,
1018                                 max_queues->max_sq_depth);
1019                 }
1020
1021                 ctx->max_rx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
1022                         max_queues->max_packet_rx_descs);
1023                 ctx->max_tx_sgl_size = RTE_MIN(ENA_PKT_MAX_BUFS,
1024                         max_queues->max_packet_tx_descs);
1025         }
1026
1027         /* Round down to the nearest power of 2 */
1028         max_rx_queue_size = rte_align32prevpow2(max_rx_queue_size);
1029         max_tx_queue_size = rte_align32prevpow2(max_tx_queue_size);
1030
1031         if (use_large_llq_hdr) {
1032                 if ((llq->entry_size_ctrl_supported &
1033                      ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&
1034                     (ena_dev->tx_mem_queue_type ==
1035                      ENA_ADMIN_PLACEMENT_POLICY_DEV)) {
1036                         max_tx_queue_size /= 2;
1037                         PMD_INIT_LOG(INFO,
1038                                 "Forcing large headers and decreasing maximum Tx queue size to %d\n",
1039                                 max_tx_queue_size);
1040                 } else {
1041                         PMD_INIT_LOG(ERR,
1042                                 "Forcing large headers failed: LLQ is disabled or device does not support large headers\n");
1043                 }
1044         }
1045
1046         if (unlikely(max_rx_queue_size == 0 || max_tx_queue_size == 0)) {
1047                 PMD_INIT_LOG(ERR, "Invalid queue size\n");
1048                 return -EFAULT;
1049         }
1050
1051         ctx->max_tx_queue_size = max_tx_queue_size;
1052         ctx->max_rx_queue_size = max_rx_queue_size;
1053
1054         return 0;
1055 }
1056
1057 static void ena_stats_restart(struct rte_eth_dev *dev)
1058 {
1059         struct ena_adapter *adapter = dev->data->dev_private;
1060
1061         rte_atomic64_init(&adapter->drv_stats->ierrors);
1062         rte_atomic64_init(&adapter->drv_stats->oerrors);
1063         rte_atomic64_init(&adapter->drv_stats->rx_nombuf);
1064         adapter->drv_stats->rx_drops = 0;
1065 }
1066
1067 static int ena_stats_get(struct rte_eth_dev *dev,
1068                           struct rte_eth_stats *stats)
1069 {
1070         struct ena_admin_basic_stats ena_stats;
1071         struct ena_adapter *adapter = dev->data->dev_private;
1072         struct ena_com_dev *ena_dev = &adapter->ena_dev;
1073         int rc;
1074         int i;
1075         int max_rings_stats;
1076
1077         memset(&ena_stats, 0, sizeof(ena_stats));
1078
1079         rte_spinlock_lock(&adapter->admin_lock);
1080         rc = ENA_PROXY(adapter, ena_com_get_dev_basic_stats, ena_dev,
1081                        &ena_stats);
1082         rte_spinlock_unlock(&adapter->admin_lock);
1083         if (unlikely(rc)) {
1084                 PMD_DRV_LOG(ERR, "Could not retrieve statistics from ENA\n");
1085                 return rc;
1086         }
1087
1088         /* Set of basic statistics from ENA */
1089         stats->ipackets = __MERGE_64B_H_L(ena_stats.rx_pkts_high,
1090                                           ena_stats.rx_pkts_low);
1091         stats->opackets = __MERGE_64B_H_L(ena_stats.tx_pkts_high,
1092                                           ena_stats.tx_pkts_low);
1093         stats->ibytes = __MERGE_64B_H_L(ena_stats.rx_bytes_high,
1094                                         ena_stats.rx_bytes_low);
1095         stats->obytes = __MERGE_64B_H_L(ena_stats.tx_bytes_high,
1096                                         ena_stats.tx_bytes_low);
1097
1098         /* Driver related stats */
1099         stats->imissed = adapter->drv_stats->rx_drops;
1100         stats->ierrors = rte_atomic64_read(&adapter->drv_stats->ierrors);
1101         stats->oerrors = rte_atomic64_read(&adapter->drv_stats->oerrors);
1102         stats->rx_nombuf = rte_atomic64_read(&adapter->drv_stats->rx_nombuf);
1103
1104         max_rings_stats = RTE_MIN(dev->data->nb_rx_queues,
1105                 RTE_ETHDEV_QUEUE_STAT_CNTRS);
1106         for (i = 0; i < max_rings_stats; ++i) {
1107                 struct ena_stats_rx *rx_stats = &adapter->rx_ring[i].rx_stats;
1108
1109                 stats->q_ibytes[i] = rx_stats->bytes;
1110                 stats->q_ipackets[i] = rx_stats->cnt;
1111                 stats->q_errors[i] = rx_stats->bad_desc_num +
1112                         rx_stats->bad_req_id;
1113         }
1114
1115         max_rings_stats = RTE_MIN(dev->data->nb_tx_queues,
1116                 RTE_ETHDEV_QUEUE_STAT_CNTRS);
1117         for (i = 0; i < max_rings_stats; ++i) {
1118                 struct ena_stats_tx *tx_stats = &adapter->tx_ring[i].tx_stats;
1119
1120                 stats->q_obytes[i] = tx_stats->bytes;
1121                 stats->q_opackets[i] = tx_stats->cnt;
1122         }
1123
1124         return 0;
1125 }
1126
1127 static int ena_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1128 {
1129         struct ena_adapter *adapter;
1130         struct ena_com_dev *ena_dev;
1131         int rc = 0;
1132
1133         ena_assert_msg(dev->data != NULL, "Uninitialized device\n");
1134         ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n");
1135         adapter = dev->data->dev_private;
1136
1137         ena_dev = &adapter->ena_dev;
1138         ena_assert_msg(ena_dev != NULL, "Uninitialized device\n");
1139
1140         if (mtu > adapter->max_mtu || mtu < ENA_MIN_MTU) {
1141                 PMD_DRV_LOG(ERR,
1142                         "Invalid MTU setting. New MTU: %d, max MTU: %d, min MTU: %d\n",
1143                         mtu, adapter->max_mtu, ENA_MIN_MTU);
1144                 return -EINVAL;
1145         }
1146
1147         rc = ENA_PROXY(adapter, ena_com_set_dev_mtu, ena_dev, mtu);
1148         if (rc)
1149                 PMD_DRV_LOG(ERR, "Could not set MTU: %d\n", mtu);
1150         else
1151                 PMD_DRV_LOG(NOTICE, "MTU set to: %d\n", mtu);
1152
1153         return rc;
1154 }
1155
1156 static int ena_start(struct rte_eth_dev *dev)
1157 {
1158         struct ena_adapter *adapter = dev->data->dev_private;
1159         uint64_t ticks;
1160         int rc = 0;
1161
1162         /* Cannot allocate memory in secondary process */
1163         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1164                 PMD_DRV_LOG(WARNING, "dev_start not supported in secondary.\n");
1165                 return -EPERM;
1166         }
1167
1168         rc = ena_check_valid_conf(adapter);
1169         if (rc)
1170                 return rc;
1171
1172         rc = ena_setup_rx_intr(dev);
1173         if (rc)
1174                 return rc;
1175
1176         rc = ena_queue_start_all(dev, ENA_RING_TYPE_RX);
1177         if (rc)
1178                 return rc;
1179
1180         rc = ena_queue_start_all(dev, ENA_RING_TYPE_TX);
1181         if (rc)
1182                 goto err_start_tx;
1183
1184         if (adapter->edev_data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) {
1185                 rc = ena_rss_configure(adapter);
1186                 if (rc)
1187                         goto err_rss_init;
1188         }
1189
1190         ena_stats_restart(dev);
1191
1192         adapter->timestamp_wd = rte_get_timer_cycles();
1193         adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;
1194
1195         ticks = rte_get_timer_hz();
1196         rte_timer_reset(&adapter->timer_wd, ticks, PERIODICAL, rte_lcore_id(),
1197                         ena_timer_wd_callback, dev);
1198
1199         ++adapter->dev_stats.dev_start;
1200         adapter->state = ENA_ADAPTER_STATE_RUNNING;
1201
1202         return 0;
1203
1204 err_rss_init:
1205         ena_queue_stop_all(dev, ENA_RING_TYPE_TX);
1206 err_start_tx:
1207         ena_queue_stop_all(dev, ENA_RING_TYPE_RX);
1208         return rc;
1209 }
1210
1211 static int ena_stop(struct rte_eth_dev *dev)
1212 {
1213         struct ena_adapter *adapter = dev->data->dev_private;
1214         struct ena_com_dev *ena_dev = &adapter->ena_dev;
1215         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1216         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1217         int rc;
1218
1219         /* Cannot free memory in secondary process */
1220         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1221                 PMD_DRV_LOG(WARNING, "dev_stop not supported in secondary.\n");
1222                 return -EPERM;
1223         }
1224
1225         rte_timer_stop_sync(&adapter->timer_wd);
1226         ena_queue_stop_all(dev, ENA_RING_TYPE_TX);
1227         ena_queue_stop_all(dev, ENA_RING_TYPE_RX);
1228
1229         if (adapter->trigger_reset) {
1230                 rc = ena_com_dev_reset(ena_dev, adapter->reset_reason);
1231                 if (rc)
1232                         PMD_DRV_LOG(ERR, "Device reset failed, rc: %d\n", rc);
1233         }
1234
1235         rte_intr_disable(intr_handle);
1236
1237         rte_intr_efd_disable(intr_handle);
1238
1239         /* Cleanup vector list */
1240         rte_intr_vec_list_free(intr_handle);
1241
1242         rte_intr_enable(intr_handle);
1243
1244         ++adapter->dev_stats.dev_stop;
1245         adapter->state = ENA_ADAPTER_STATE_STOPPED;
1246         dev->data->dev_started = 0;
1247
1248         return 0;
1249 }
1250
1251 static int ena_create_io_queue(struct rte_eth_dev *dev, struct ena_ring *ring)
1252 {
1253         struct ena_adapter *adapter = ring->adapter;
1254         struct ena_com_dev *ena_dev = &adapter->ena_dev;
1255         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1256         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1257         struct ena_com_create_io_ctx ctx =
1258                 /* policy set to _HOST just to satisfy icc compiler */
1259                 { ENA_ADMIN_PLACEMENT_POLICY_HOST,
1260                   0, 0, 0, 0, 0 };
1261         uint16_t ena_qid;
1262         unsigned int i;
1263         int rc;
1264
1265         ctx.msix_vector = -1;
1266         if (ring->type == ENA_RING_TYPE_TX) {
1267                 ena_qid = ENA_IO_TXQ_IDX(ring->id);
1268                 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1269                 ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1270                 for (i = 0; i < ring->ring_size; i++)
1271                         ring->empty_tx_reqs[i] = i;
1272         } else {
1273                 ena_qid = ENA_IO_RXQ_IDX(ring->id);
1274                 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1275                 if (rte_intr_dp_is_en(intr_handle))
1276                         ctx.msix_vector =
1277                                 rte_intr_vec_list_index_get(intr_handle,
1278                                                                    ring->id);
1279
1280                 for (i = 0; i < ring->ring_size; i++)
1281                         ring->empty_rx_reqs[i] = i;
1282         }
1283         ctx.queue_size = ring->ring_size;
1284         ctx.qid = ena_qid;
1285         ctx.numa_node = ring->numa_socket_id;
1286
1287         rc = ena_com_create_io_queue(ena_dev, &ctx);
1288         if (rc) {
1289                 PMD_DRV_LOG(ERR,
1290                         "Failed to create IO queue[%d] (qid:%d), rc: %d\n",
1291                         ring->id, ena_qid, rc);
1292                 return rc;
1293         }
1294
1295         rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1296                                      &ring->ena_com_io_sq,
1297                                      &ring->ena_com_io_cq);
1298         if (rc) {
1299                 PMD_DRV_LOG(ERR,
1300                         "Failed to get IO queue[%d] handlers, rc: %d\n",
1301                         ring->id, rc);
1302                 ena_com_destroy_io_queue(ena_dev, ena_qid);
1303                 return rc;
1304         }
1305
1306         if (ring->type == ENA_RING_TYPE_TX)
1307                 ena_com_update_numa_node(ring->ena_com_io_cq, ctx.numa_node);
1308
1309         /* Start with Rx interrupts being masked. */
1310         if (ring->type == ENA_RING_TYPE_RX && rte_intr_dp_is_en(intr_handle))
1311                 ena_rx_queue_intr_disable(dev, ring->id);
1312
1313         return 0;
1314 }
1315
1316 static void ena_queue_stop(struct ena_ring *ring)
1317 {
1318         struct ena_com_dev *ena_dev = &ring->adapter->ena_dev;
1319
1320         if (ring->type == ENA_RING_TYPE_RX) {
1321                 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(ring->id));
1322                 ena_rx_queue_release_bufs(ring);
1323         } else {
1324                 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(ring->id));
1325                 ena_tx_queue_release_bufs(ring);
1326         }
1327 }
1328
1329 static void ena_queue_stop_all(struct rte_eth_dev *dev,
1330                               enum ena_ring_type ring_type)
1331 {
1332         struct ena_adapter *adapter = dev->data->dev_private;
1333         struct ena_ring *queues = NULL;
1334         uint16_t nb_queues, i;
1335
1336         if (ring_type == ENA_RING_TYPE_RX) {
1337                 queues = adapter->rx_ring;
1338                 nb_queues = dev->data->nb_rx_queues;
1339         } else {
1340                 queues = adapter->tx_ring;
1341                 nb_queues = dev->data->nb_tx_queues;
1342         }
1343
1344         for (i = 0; i < nb_queues; ++i)
1345                 if (queues[i].configured)
1346                         ena_queue_stop(&queues[i]);
1347 }
1348
1349 static int ena_queue_start(struct rte_eth_dev *dev, struct ena_ring *ring)
1350 {
1351         int rc, bufs_num;
1352
1353         ena_assert_msg(ring->configured == 1,
1354                        "Trying to start unconfigured queue\n");
1355
1356         rc = ena_create_io_queue(dev, ring);
1357         if (rc) {
1358                 PMD_INIT_LOG(ERR, "Failed to create IO queue\n");
1359                 return rc;
1360         }
1361
1362         ring->next_to_clean = 0;
1363         ring->next_to_use = 0;
1364
1365         if (ring->type == ENA_RING_TYPE_TX) {
1366                 ring->tx_stats.available_desc =
1367                         ena_com_free_q_entries(ring->ena_com_io_sq);
1368                 return 0;
1369         }
1370
1371         bufs_num = ring->ring_size - 1;
1372         rc = ena_populate_rx_queue(ring, bufs_num);
1373         if (rc != bufs_num) {
1374                 ena_com_destroy_io_queue(&ring->adapter->ena_dev,
1375                                          ENA_IO_RXQ_IDX(ring->id));
1376                 PMD_INIT_LOG(ERR, "Failed to populate Rx ring\n");
1377                 return ENA_COM_FAULT;
1378         }
1379         /* Flush per-core RX buffers pools cache as they can be used on other
1380          * cores as well.
1381          */
1382         rte_mempool_cache_flush(NULL, ring->mb_pool);
1383
1384         return 0;
1385 }
1386
1387 static int ena_tx_queue_setup(struct rte_eth_dev *dev,
1388                               uint16_t queue_idx,
1389                               uint16_t nb_desc,
1390                               unsigned int socket_id,
1391                               const struct rte_eth_txconf *tx_conf)
1392 {
1393         struct ena_ring *txq = NULL;
1394         struct ena_adapter *adapter = dev->data->dev_private;
1395         unsigned int i;
1396         uint16_t dyn_thresh;
1397
1398         txq = &adapter->tx_ring[queue_idx];
1399
1400         if (txq->configured) {
1401                 PMD_DRV_LOG(CRIT,
1402                         "API violation. Queue[%d] is already configured\n",
1403                         queue_idx);
1404                 return ENA_COM_FAULT;
1405         }
1406
1407         if (!rte_is_power_of_2(nb_desc)) {
1408                 PMD_DRV_LOG(ERR,
1409                         "Unsupported size of Tx queue: %d is not a power of 2.\n",
1410                         nb_desc);
1411                 return -EINVAL;
1412         }
1413
1414         if (nb_desc > adapter->max_tx_ring_size) {
1415                 PMD_DRV_LOG(ERR,
1416                         "Unsupported size of Tx queue (max size: %d)\n",
1417                         adapter->max_tx_ring_size);
1418                 return -EINVAL;
1419         }
1420
1421         txq->port_id = dev->data->port_id;
1422         txq->next_to_clean = 0;
1423         txq->next_to_use = 0;
1424         txq->ring_size = nb_desc;
1425         txq->size_mask = nb_desc - 1;
1426         txq->numa_socket_id = socket_id;
1427         txq->pkts_without_db = false;
1428         txq->last_cleanup_ticks = 0;
1429
1430         txq->tx_buffer_info = rte_zmalloc_socket("txq->tx_buffer_info",
1431                 sizeof(struct ena_tx_buffer) * txq->ring_size,
1432                 RTE_CACHE_LINE_SIZE,
1433                 socket_id);
1434         if (!txq->tx_buffer_info) {
1435                 PMD_DRV_LOG(ERR,
1436                         "Failed to allocate memory for Tx buffer info\n");
1437                 return -ENOMEM;
1438         }
1439
1440         txq->empty_tx_reqs = rte_zmalloc_socket("txq->empty_tx_reqs",
1441                 sizeof(uint16_t) * txq->ring_size,
1442                 RTE_CACHE_LINE_SIZE,
1443                 socket_id);
1444         if (!txq->empty_tx_reqs) {
1445                 PMD_DRV_LOG(ERR,
1446                         "Failed to allocate memory for empty Tx requests\n");
1447                 rte_free(txq->tx_buffer_info);
1448                 return -ENOMEM;
1449         }
1450
1451         txq->push_buf_intermediate_buf =
1452                 rte_zmalloc_socket("txq->push_buf_intermediate_buf",
1453                         txq->tx_max_header_size,
1454                         RTE_CACHE_LINE_SIZE,
1455                         socket_id);
1456         if (!txq->push_buf_intermediate_buf) {
1457                 PMD_DRV_LOG(ERR, "Failed to alloc push buffer for LLQ\n");
1458                 rte_free(txq->tx_buffer_info);
1459                 rte_free(txq->empty_tx_reqs);
1460                 return -ENOMEM;
1461         }
1462
1463         for (i = 0; i < txq->ring_size; i++)
1464                 txq->empty_tx_reqs[i] = i;
1465
1466         txq->offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
1467
1468         /* Check if caller provided the Tx cleanup threshold value. */
1469         if (tx_conf->tx_free_thresh != 0) {
1470                 txq->tx_free_thresh = tx_conf->tx_free_thresh;
1471         } else {
1472                 dyn_thresh = txq->ring_size -
1473                         txq->ring_size / ENA_REFILL_THRESH_DIVIDER;
1474                 txq->tx_free_thresh = RTE_MAX(dyn_thresh,
1475                         txq->ring_size - ENA_REFILL_THRESH_PACKET);
1476         }
1477
1478         txq->missing_tx_completion_threshold =
1479                 RTE_MIN(txq->ring_size / 2, ENA_DEFAULT_MISSING_COMP);
1480
1481         /* Store pointer to this queue in upper layer */
1482         txq->configured = 1;
1483         dev->data->tx_queues[queue_idx] = txq;
1484
1485         return 0;
1486 }
1487
1488 static int ena_rx_queue_setup(struct rte_eth_dev *dev,
1489                               uint16_t queue_idx,
1490                               uint16_t nb_desc,
1491                               unsigned int socket_id,
1492                               const struct rte_eth_rxconf *rx_conf,
1493                               struct rte_mempool *mp)
1494 {
1495         struct ena_adapter *adapter = dev->data->dev_private;
1496         struct ena_ring *rxq = NULL;
1497         size_t buffer_size;
1498         int i;
1499         uint16_t dyn_thresh;
1500
1501         rxq = &adapter->rx_ring[queue_idx];
1502         if (rxq->configured) {
1503                 PMD_DRV_LOG(CRIT,
1504                         "API violation. Queue[%d] is already configured\n",
1505                         queue_idx);
1506                 return ENA_COM_FAULT;
1507         }
1508
1509         if (!rte_is_power_of_2(nb_desc)) {
1510                 PMD_DRV_LOG(ERR,
1511                         "Unsupported size of Rx queue: %d is not a power of 2.\n",
1512                         nb_desc);
1513                 return -EINVAL;
1514         }
1515
1516         if (nb_desc > adapter->max_rx_ring_size) {
1517                 PMD_DRV_LOG(ERR,
1518                         "Unsupported size of Rx queue (max size: %d)\n",
1519                         adapter->max_rx_ring_size);
1520                 return -EINVAL;
1521         }
1522
1523         /* ENA isn't supporting buffers smaller than 1400 bytes */
1524         buffer_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
1525         if (buffer_size < ENA_RX_BUF_MIN_SIZE) {
1526                 PMD_DRV_LOG(ERR,
1527                         "Unsupported size of Rx buffer: %zu (min size: %d)\n",
1528                         buffer_size, ENA_RX_BUF_MIN_SIZE);
1529                 return -EINVAL;
1530         }
1531
1532         rxq->port_id = dev->data->port_id;
1533         rxq->next_to_clean = 0;
1534         rxq->next_to_use = 0;
1535         rxq->ring_size = nb_desc;
1536         rxq->size_mask = nb_desc - 1;
1537         rxq->numa_socket_id = socket_id;
1538         rxq->mb_pool = mp;
1539
1540         rxq->rx_buffer_info = rte_zmalloc_socket("rxq->buffer_info",
1541                 sizeof(struct ena_rx_buffer) * nb_desc,
1542                 RTE_CACHE_LINE_SIZE,
1543                 socket_id);
1544         if (!rxq->rx_buffer_info) {
1545                 PMD_DRV_LOG(ERR,
1546                         "Failed to allocate memory for Rx buffer info\n");
1547                 return -ENOMEM;
1548         }
1549
1550         rxq->rx_refill_buffer = rte_zmalloc_socket("rxq->rx_refill_buffer",
1551                 sizeof(struct rte_mbuf *) * nb_desc,
1552                 RTE_CACHE_LINE_SIZE,
1553                 socket_id);
1554         if (!rxq->rx_refill_buffer) {
1555                 PMD_DRV_LOG(ERR,
1556                         "Failed to allocate memory for Rx refill buffer\n");
1557                 rte_free(rxq->rx_buffer_info);
1558                 rxq->rx_buffer_info = NULL;
1559                 return -ENOMEM;
1560         }
1561
1562         rxq->empty_rx_reqs = rte_zmalloc_socket("rxq->empty_rx_reqs",
1563                 sizeof(uint16_t) * nb_desc,
1564                 RTE_CACHE_LINE_SIZE,
1565                 socket_id);
1566         if (!rxq->empty_rx_reqs) {
1567                 PMD_DRV_LOG(ERR,
1568                         "Failed to allocate memory for empty Rx requests\n");
1569                 rte_free(rxq->rx_buffer_info);
1570                 rxq->rx_buffer_info = NULL;
1571                 rte_free(rxq->rx_refill_buffer);
1572                 rxq->rx_refill_buffer = NULL;
1573                 return -ENOMEM;
1574         }
1575
1576         for (i = 0; i < nb_desc; i++)
1577                 rxq->empty_rx_reqs[i] = i;
1578
1579         rxq->offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
1580
1581         if (rx_conf->rx_free_thresh != 0) {
1582                 rxq->rx_free_thresh = rx_conf->rx_free_thresh;
1583         } else {
1584                 dyn_thresh = rxq->ring_size / ENA_REFILL_THRESH_DIVIDER;
1585                 rxq->rx_free_thresh = RTE_MIN(dyn_thresh,
1586                         (uint16_t)(ENA_REFILL_THRESH_PACKET));
1587         }
1588
1589         /* Store pointer to this queue in upper layer */
1590         rxq->configured = 1;
1591         dev->data->rx_queues[queue_idx] = rxq;
1592
1593         return 0;
1594 }
1595
1596 static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq,
1597                                   struct rte_mbuf *mbuf, uint16_t id)
1598 {
1599         struct ena_com_buf ebuf;
1600         int rc;
1601
1602         /* prepare physical address for DMA transaction */
1603         ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM;
1604         ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM;
1605
1606         /* pass resource to device */
1607         rc = ena_com_add_single_rx_desc(io_sq, &ebuf, id);
1608         if (unlikely(rc != 0))
1609                 PMD_RX_LOG(WARNING, "Failed adding Rx desc\n");
1610
1611         return rc;
1612 }
1613
1614 static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count)
1615 {
1616         unsigned int i;
1617         int rc;
1618         uint16_t next_to_use = rxq->next_to_use;
1619         uint16_t req_id;
1620 #ifdef RTE_ETHDEV_DEBUG_RX
1621         uint16_t in_use;
1622 #endif
1623         struct rte_mbuf **mbufs = rxq->rx_refill_buffer;
1624
1625         if (unlikely(!count))
1626                 return 0;
1627
1628 #ifdef RTE_ETHDEV_DEBUG_RX
1629         in_use = rxq->ring_size - 1 -
1630                 ena_com_free_q_entries(rxq->ena_com_io_sq);
1631         if (unlikely((in_use + count) >= rxq->ring_size))
1632                 PMD_RX_LOG(ERR, "Bad Rx ring state\n");
1633 #endif
1634
1635         /* get resources for incoming packets */
1636         rc = rte_pktmbuf_alloc_bulk(rxq->mb_pool, mbufs, count);
1637         if (unlikely(rc < 0)) {
1638                 rte_atomic64_inc(&rxq->adapter->drv_stats->rx_nombuf);
1639                 ++rxq->rx_stats.mbuf_alloc_fail;
1640                 PMD_RX_LOG(DEBUG, "There are not enough free buffers\n");
1641                 return 0;
1642         }
1643
1644         for (i = 0; i < count; i++) {
1645                 struct rte_mbuf *mbuf = mbufs[i];
1646                 struct ena_rx_buffer *rx_info;
1647
1648                 if (likely((i + 4) < count))
1649                         rte_prefetch0(mbufs[i + 4]);
1650
1651                 req_id = rxq->empty_rx_reqs[next_to_use];
1652                 rx_info = &rxq->rx_buffer_info[req_id];
1653
1654                 rc = ena_add_single_rx_desc(rxq->ena_com_io_sq, mbuf, req_id);
1655                 if (unlikely(rc != 0))
1656                         break;
1657
1658                 rx_info->mbuf = mbuf;
1659                 next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, rxq->size_mask);
1660         }
1661
1662         if (unlikely(i < count)) {
1663                 PMD_RX_LOG(WARNING,
1664                         "Refilled Rx queue[%d] with only %d/%d buffers\n",
1665                         rxq->id, i, count);
1666                 rte_pktmbuf_free_bulk(&mbufs[i], count - i);
1667                 ++rxq->rx_stats.refill_partial;
1668         }
1669
1670         /* When we submitted free resources to device... */
1671         if (likely(i > 0)) {
1672                 /* ...let HW know that it can fill buffers with data. */
1673                 ena_com_write_sq_doorbell(rxq->ena_com_io_sq);
1674
1675                 rxq->next_to_use = next_to_use;
1676         }
1677
1678         return i;
1679 }
1680
1681 static int ena_device_init(struct ena_adapter *adapter,
1682                            struct rte_pci_device *pdev,
1683                            struct ena_com_dev_get_features_ctx *get_feat_ctx)
1684 {
1685         struct ena_com_dev *ena_dev = &adapter->ena_dev;
1686         uint32_t aenq_groups;
1687         int rc;
1688         bool readless_supported;
1689
1690         /* Initialize mmio registers */
1691         rc = ena_com_mmio_reg_read_request_init(ena_dev);
1692         if (rc) {
1693                 PMD_DRV_LOG(ERR, "Failed to init MMIO read less\n");
1694                 return rc;
1695         }
1696
1697         /* The PCIe configuration space revision id indicate if mmio reg
1698          * read is disabled.
1699          */
1700         readless_supported = !(pdev->id.class_id & ENA_MMIO_DISABLE_REG_READ);
1701         ena_com_set_mmio_read_mode(ena_dev, readless_supported);
1702
1703         /* reset device */
1704         rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
1705         if (rc) {
1706                 PMD_DRV_LOG(ERR, "Cannot reset device\n");
1707                 goto err_mmio_read_less;
1708         }
1709
1710         /* check FW version */
1711         rc = ena_com_validate_version(ena_dev);
1712         if (rc) {
1713                 PMD_DRV_LOG(ERR, "Device version is too low\n");
1714                 goto err_mmio_read_less;
1715         }
1716
1717         ena_dev->dma_addr_bits = ena_com_get_dma_width(ena_dev);
1718
1719         /* ENA device administration layer init */
1720         rc = ena_com_admin_init(ena_dev, &aenq_handlers);
1721         if (rc) {
1722                 PMD_DRV_LOG(ERR,
1723                         "Cannot initialize ENA admin queue\n");
1724                 goto err_mmio_read_less;
1725         }
1726
1727         /* To enable the msix interrupts the driver needs to know the number
1728          * of queues. So the driver uses polling mode to retrieve this
1729          * information.
1730          */
1731         ena_com_set_admin_polling_mode(ena_dev, true);
1732
1733         ena_config_host_info(ena_dev);
1734
1735         /* Get Device Attributes and features */
1736         rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
1737         if (rc) {
1738                 PMD_DRV_LOG(ERR,
1739                         "Cannot get attribute for ENA device, rc: %d\n", rc);
1740                 goto err_admin_init;
1741         }
1742
1743         aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
1744                       BIT(ENA_ADMIN_NOTIFICATION) |
1745                       BIT(ENA_ADMIN_KEEP_ALIVE) |
1746                       BIT(ENA_ADMIN_FATAL_ERROR) |
1747                       BIT(ENA_ADMIN_WARNING);
1748
1749         aenq_groups &= get_feat_ctx->aenq.supported_groups;
1750
1751         adapter->all_aenq_groups = aenq_groups;
1752
1753         return 0;
1754
1755 err_admin_init:
1756         ena_com_admin_destroy(ena_dev);
1757
1758 err_mmio_read_less:
1759         ena_com_mmio_reg_read_request_destroy(ena_dev);
1760
1761         return rc;
1762 }
1763
1764 static void ena_interrupt_handler_rte(void *cb_arg)
1765 {
1766         struct rte_eth_dev *dev = cb_arg;
1767         struct ena_adapter *adapter = dev->data->dev_private;
1768         struct ena_com_dev *ena_dev = &adapter->ena_dev;
1769
1770         ena_com_admin_q_comp_intr_handler(ena_dev);
1771         if (likely(adapter->state != ENA_ADAPTER_STATE_CLOSED))
1772                 ena_com_aenq_intr_handler(ena_dev, dev);
1773 }
1774
1775 static void check_for_missing_keep_alive(struct ena_adapter *adapter)
1776 {
1777         if (!(adapter->active_aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)))
1778                 return;
1779
1780         if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
1781                 return;
1782
1783         if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >=
1784             adapter->keep_alive_timeout)) {
1785                 PMD_DRV_LOG(ERR, "Keep alive timeout\n");
1786                 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
1787                 adapter->trigger_reset = true;
1788                 ++adapter->dev_stats.wd_expired;
1789         }
1790 }
1791
1792 /* Check if admin queue is enabled */
1793 static void check_for_admin_com_state(struct ena_adapter *adapter)
1794 {
1795         if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) {
1796                 PMD_DRV_LOG(ERR, "ENA admin queue is not in running state\n");
1797                 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
1798                 adapter->trigger_reset = true;
1799         }
1800 }
1801
1802 static int check_for_tx_completion_in_queue(struct ena_adapter *adapter,
1803                                             struct ena_ring *tx_ring)
1804 {
1805         struct ena_tx_buffer *tx_buf;
1806         uint64_t timestamp;
1807         uint64_t completion_delay;
1808         uint32_t missed_tx = 0;
1809         unsigned int i;
1810         int rc = 0;
1811
1812         for (i = 0; i < tx_ring->ring_size; ++i) {
1813                 tx_buf = &tx_ring->tx_buffer_info[i];
1814                 timestamp = tx_buf->timestamp;
1815
1816                 if (timestamp == 0)
1817                         continue;
1818
1819                 completion_delay = rte_get_timer_cycles() - timestamp;
1820                 if (completion_delay > adapter->missing_tx_completion_to) {
1821                         if (unlikely(!tx_buf->print_once)) {
1822                                 PMD_TX_LOG(WARNING,
1823                                         "Found a Tx that wasn't completed on time, qid %d, index %d. "
1824                                         "Missing Tx outstanding for %" PRIu64 " msecs.\n",
1825                                         tx_ring->id, i, completion_delay /
1826                                         rte_get_timer_hz() * 1000);
1827                                 tx_buf->print_once = true;
1828                         }
1829                         ++missed_tx;
1830                 }
1831         }
1832
1833         if (unlikely(missed_tx > tx_ring->missing_tx_completion_threshold)) {
1834                 PMD_DRV_LOG(ERR,
1835                         "The number of lost Tx completions is above the threshold (%d > %d). "
1836                         "Trigger the device reset.\n",
1837                         missed_tx,
1838                         tx_ring->missing_tx_completion_threshold);
1839                 adapter->reset_reason = ENA_REGS_RESET_MISS_TX_CMPL;
1840                 adapter->trigger_reset = true;
1841                 rc = -EIO;
1842         }
1843
1844         tx_ring->tx_stats.missed_tx += missed_tx;
1845
1846         return rc;
1847 }
1848
1849 static void check_for_tx_completions(struct ena_adapter *adapter)
1850 {
1851         struct ena_ring *tx_ring;
1852         uint64_t tx_cleanup_delay;
1853         size_t qid;
1854         int budget;
1855         uint16_t nb_tx_queues = adapter->edev_data->nb_tx_queues;
1856
1857         if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT)
1858                 return;
1859
1860         nb_tx_queues = adapter->edev_data->nb_tx_queues;
1861         budget = adapter->missing_tx_completion_budget;
1862
1863         qid = adapter->last_tx_comp_qid;
1864         while (budget-- > 0) {
1865                 tx_ring = &adapter->tx_ring[qid];
1866
1867                 /* Tx cleanup is called only by the burst function and can be
1868                  * called dynamically by the application. Also cleanup is
1869                  * limited by the threshold. To avoid false detection of the
1870                  * missing HW Tx completion, get the delay since last cleanup
1871                  * function was called.
1872                  */
1873                 tx_cleanup_delay = rte_get_timer_cycles() -
1874                         tx_ring->last_cleanup_ticks;
1875                 if (tx_cleanup_delay < adapter->tx_cleanup_stall_delay)
1876                         check_for_tx_completion_in_queue(adapter, tx_ring);
1877                 qid = (qid + 1) % nb_tx_queues;
1878         }
1879
1880         adapter->last_tx_comp_qid = qid;
1881 }
1882
1883 static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer,
1884                                   void *arg)
1885 {
1886         struct rte_eth_dev *dev = arg;
1887         struct ena_adapter *adapter = dev->data->dev_private;
1888
1889         if (unlikely(adapter->trigger_reset))
1890                 return;
1891
1892         check_for_missing_keep_alive(adapter);
1893         check_for_admin_com_state(adapter);
1894         check_for_tx_completions(adapter);
1895
1896         if (unlikely(adapter->trigger_reset)) {
1897                 PMD_DRV_LOG(ERR, "Trigger reset is on\n");
1898                 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET,
1899                         NULL);
1900         }
1901 }
1902
1903 static inline void
1904 set_default_llq_configurations(struct ena_llq_configurations *llq_config,
1905                                struct ena_admin_feature_llq_desc *llq,
1906                                bool use_large_llq_hdr)
1907 {
1908         llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
1909         llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
1910         llq_config->llq_num_decs_before_header =
1911                 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
1912
1913         if (use_large_llq_hdr &&
1914             (llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B)) {
1915                 llq_config->llq_ring_entry_size =
1916                         ENA_ADMIN_LIST_ENTRY_SIZE_256B;
1917                 llq_config->llq_ring_entry_size_value = 256;
1918         } else {
1919                 llq_config->llq_ring_entry_size =
1920                         ENA_ADMIN_LIST_ENTRY_SIZE_128B;
1921                 llq_config->llq_ring_entry_size_value = 128;
1922         }
1923 }
1924
1925 static int
1926 ena_set_queues_placement_policy(struct ena_adapter *adapter,
1927                                 struct ena_com_dev *ena_dev,
1928                                 struct ena_admin_feature_llq_desc *llq,
1929                                 struct ena_llq_configurations *llq_default_configurations)
1930 {
1931         int rc;
1932         u32 llq_feature_mask;
1933
1934         llq_feature_mask = 1 << ENA_ADMIN_LLQ;
1935         if (!(ena_dev->supported_features & llq_feature_mask)) {
1936                 PMD_DRV_LOG(INFO,
1937                         "LLQ is not supported. Fallback to host mode policy.\n");
1938                 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1939                 return 0;
1940         }
1941
1942         rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
1943         if (unlikely(rc)) {
1944                 PMD_INIT_LOG(WARNING,
1945                         "Failed to config dev mode. Fallback to host mode policy.\n");
1946                 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1947                 return 0;
1948         }
1949
1950         /* Nothing to config, exit */
1951         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
1952                 return 0;
1953
1954         if (!adapter->dev_mem_base) {
1955                 PMD_DRV_LOG(ERR,
1956                         "Unable to access LLQ BAR resource. Fallback to host mode policy.\n");
1957                 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1958                 return 0;
1959         }
1960
1961         ena_dev->mem_bar = adapter->dev_mem_base;
1962
1963         return 0;
1964 }
1965
1966 static uint32_t ena_calc_max_io_queue_num(struct ena_com_dev *ena_dev,
1967         struct ena_com_dev_get_features_ctx *get_feat_ctx)
1968 {
1969         uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
1970
1971         /* Regular queues capabilities */
1972         if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
1973                 struct ena_admin_queue_ext_feature_fields *max_queue_ext =
1974                         &get_feat_ctx->max_queue_ext.max_queue_ext;
1975                 io_rx_num = RTE_MIN(max_queue_ext->max_rx_sq_num,
1976                                     max_queue_ext->max_rx_cq_num);
1977                 io_tx_sq_num = max_queue_ext->max_tx_sq_num;
1978                 io_tx_cq_num = max_queue_ext->max_tx_cq_num;
1979         } else {
1980                 struct ena_admin_queue_feature_desc *max_queues =
1981                         &get_feat_ctx->max_queues;
1982                 io_tx_sq_num = max_queues->max_sq_num;
1983                 io_tx_cq_num = max_queues->max_cq_num;
1984                 io_rx_num = RTE_MIN(io_tx_sq_num, io_tx_cq_num);
1985         }
1986
1987         /* In case of LLQ use the llq number in the get feature cmd */
1988         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
1989                 io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
1990
1991         max_num_io_queues = RTE_MIN(ENA_MAX_NUM_IO_QUEUES, io_rx_num);
1992         max_num_io_queues = RTE_MIN(max_num_io_queues, io_tx_sq_num);
1993         max_num_io_queues = RTE_MIN(max_num_io_queues, io_tx_cq_num);
1994
1995         if (unlikely(max_num_io_queues == 0)) {
1996                 PMD_DRV_LOG(ERR, "Number of IO queues cannot not be 0\n");
1997                 return -EFAULT;
1998         }
1999
2000         return max_num_io_queues;
2001 }
2002
2003 static void
2004 ena_set_offloads(struct ena_offloads *offloads,
2005                  struct ena_admin_feature_offload_desc *offload_desc)
2006 {
2007         if (offload_desc->tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
2008                 offloads->tx_offloads |= ENA_IPV4_TSO;
2009
2010         /* Tx IPv4 checksum offloads */
2011         if (offload_desc->tx &
2012             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)
2013                 offloads->tx_offloads |= ENA_L3_IPV4_CSUM;
2014         if (offload_desc->tx &
2015             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK)
2016                 offloads->tx_offloads |= ENA_L4_IPV4_CSUM;
2017         if (offload_desc->tx &
2018             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
2019                 offloads->tx_offloads |= ENA_L4_IPV4_CSUM_PARTIAL;
2020
2021         /* Tx IPv6 checksum offloads */
2022         if (offload_desc->tx &
2023             ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK)
2024                 offloads->tx_offloads |= ENA_L4_IPV6_CSUM;
2025         if (offload_desc->tx &
2026              ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
2027                 offloads->tx_offloads |= ENA_L4_IPV6_CSUM_PARTIAL;
2028
2029         /* Rx IPv4 checksum offloads */
2030         if (offload_desc->rx_supported &
2031             ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)
2032                 offloads->rx_offloads |= ENA_L3_IPV4_CSUM;
2033         if (offload_desc->rx_supported &
2034             ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
2035                 offloads->rx_offloads |= ENA_L4_IPV4_CSUM;
2036
2037         /* Rx IPv6 checksum offloads */
2038         if (offload_desc->rx_supported &
2039             ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
2040                 offloads->rx_offloads |= ENA_L4_IPV6_CSUM;
2041
2042         if (offload_desc->rx_supported &
2043             ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_HASH_MASK)
2044                 offloads->rx_offloads |= ENA_RX_RSS_HASH;
2045 }
2046
2047 static int ena_init_once(void)
2048 {
2049         static bool init_done;
2050
2051         if (init_done)
2052                 return 0;
2053
2054         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2055                 /* Init timer subsystem for the ENA timer service. */
2056                 rte_timer_subsystem_init();
2057                 /* Register handler for requests from secondary processes. */
2058                 rte_mp_action_register(ENA_MP_NAME, ena_mp_primary_handle);
2059         }
2060
2061         init_done = true;
2062         return 0;
2063 }
2064
2065 static int eth_ena_dev_init(struct rte_eth_dev *eth_dev)
2066 {
2067         struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 };
2068         struct rte_pci_device *pci_dev;
2069         struct rte_intr_handle *intr_handle;
2070         struct ena_adapter *adapter = eth_dev->data->dev_private;
2071         struct ena_com_dev *ena_dev = &adapter->ena_dev;
2072         struct ena_com_dev_get_features_ctx get_feat_ctx;
2073         struct ena_llq_configurations llq_config;
2074         const char *queue_type_str;
2075         uint32_t max_num_io_queues;
2076         int rc;
2077         static int adapters_found;
2078         bool disable_meta_caching;
2079
2080         eth_dev->dev_ops = &ena_dev_ops;
2081         eth_dev->rx_pkt_burst = &eth_ena_recv_pkts;
2082         eth_dev->tx_pkt_burst = &eth_ena_xmit_pkts;
2083         eth_dev->tx_pkt_prepare = &eth_ena_prep_pkts;
2084
2085         rc = ena_init_once();
2086         if (rc != 0)
2087                 return rc;
2088
2089         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2090                 return 0;
2091
2092         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
2093
2094         memset(adapter, 0, sizeof(struct ena_adapter));
2095         ena_dev = &adapter->ena_dev;
2096
2097         adapter->edev_data = eth_dev->data;
2098
2099         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2100
2101         PMD_INIT_LOG(INFO, "Initializing %x:%x:%x.%d\n",
2102                      pci_dev->addr.domain,
2103                      pci_dev->addr.bus,
2104                      pci_dev->addr.devid,
2105                      pci_dev->addr.function);
2106
2107         intr_handle = pci_dev->intr_handle;
2108
2109         adapter->regs = pci_dev->mem_resource[ENA_REGS_BAR].addr;
2110         adapter->dev_mem_base = pci_dev->mem_resource[ENA_MEM_BAR].addr;
2111
2112         if (!adapter->regs) {
2113                 PMD_INIT_LOG(CRIT, "Failed to access registers BAR(%d)\n",
2114                              ENA_REGS_BAR);
2115                 return -ENXIO;
2116         }
2117
2118         ena_dev->reg_bar = adapter->regs;
2119         /* Pass device data as a pointer which can be passed to the IO functions
2120          * by the ena_com (for example - the memory allocation).
2121          */
2122         ena_dev->dmadev = eth_dev->data;
2123
2124         adapter->id_number = adapters_found;
2125
2126         snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d",
2127                  adapter->id_number);
2128
2129         rc = ena_parse_devargs(adapter, pci_dev->device.devargs);
2130         if (rc != 0) {
2131                 PMD_INIT_LOG(CRIT, "Failed to parse devargs\n");
2132                 goto err;
2133         }
2134
2135         /* device specific initialization routine */
2136         rc = ena_device_init(adapter, pci_dev, &get_feat_ctx);
2137         if (rc) {
2138                 PMD_INIT_LOG(CRIT, "Failed to init ENA device\n");
2139                 goto err;
2140         }
2141
2142         /* Check if device supports LSC */
2143         if (!(adapter->all_aenq_groups & BIT(ENA_ADMIN_LINK_CHANGE)))
2144                 adapter->edev_data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
2145
2146         set_default_llq_configurations(&llq_config, &get_feat_ctx.llq,
2147                 adapter->use_large_llq_hdr);
2148         rc = ena_set_queues_placement_policy(adapter, ena_dev,
2149                                              &get_feat_ctx.llq, &llq_config);
2150         if (unlikely(rc)) {
2151                 PMD_INIT_LOG(CRIT, "Failed to set placement policy\n");
2152                 return rc;
2153         }
2154
2155         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
2156                 queue_type_str = "Regular";
2157         else
2158                 queue_type_str = "Low latency";
2159         PMD_DRV_LOG(INFO, "Placement policy: %s\n", queue_type_str);
2160
2161         calc_queue_ctx.ena_dev = ena_dev;
2162         calc_queue_ctx.get_feat_ctx = &get_feat_ctx;
2163
2164         max_num_io_queues = ena_calc_max_io_queue_num(ena_dev, &get_feat_ctx);
2165         rc = ena_calc_io_queue_size(&calc_queue_ctx,
2166                 adapter->use_large_llq_hdr);
2167         if (unlikely((rc != 0) || (max_num_io_queues == 0))) {
2168                 rc = -EFAULT;
2169                 goto err_device_destroy;
2170         }
2171
2172         adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size;
2173         adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size;
2174         adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size;
2175         adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size;
2176         adapter->max_num_io_queues = max_num_io_queues;
2177
2178         if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2179                 disable_meta_caching =
2180                         !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &
2181                         BIT(ENA_ADMIN_DISABLE_META_CACHING));
2182         } else {
2183                 disable_meta_caching = false;
2184         }
2185
2186         /* prepare ring structures */
2187         ena_init_rings(adapter, disable_meta_caching);
2188
2189         ena_config_debug_area(adapter);
2190
2191         /* Set max MTU for this device */
2192         adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu;
2193
2194         ena_set_offloads(&adapter->offloads, &get_feat_ctx.offload);
2195
2196         /* Copy MAC address and point DPDK to it */
2197         eth_dev->data->mac_addrs = (struct rte_ether_addr *)adapter->mac_addr;
2198         rte_ether_addr_copy((struct rte_ether_addr *)
2199                         get_feat_ctx.dev_attr.mac_addr,
2200                         (struct rte_ether_addr *)adapter->mac_addr);
2201
2202         rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
2203         if (unlikely(rc != 0)) {
2204                 PMD_DRV_LOG(ERR, "Failed to initialize RSS in ENA device\n");
2205                 goto err_delete_debug_area;
2206         }
2207
2208         adapter->drv_stats = rte_zmalloc("adapter stats",
2209                                          sizeof(*adapter->drv_stats),
2210                                          RTE_CACHE_LINE_SIZE);
2211         if (!adapter->drv_stats) {
2212                 PMD_DRV_LOG(ERR,
2213                         "Failed to allocate memory for adapter statistics\n");
2214                 rc = -ENOMEM;
2215                 goto err_rss_destroy;
2216         }
2217
2218         rte_spinlock_init(&adapter->admin_lock);
2219
2220         rte_intr_callback_register(intr_handle,
2221                                    ena_interrupt_handler_rte,
2222                                    eth_dev);
2223         rte_intr_enable(intr_handle);
2224         ena_com_set_admin_polling_mode(ena_dev, false);
2225         ena_com_admin_aenq_enable(ena_dev);
2226
2227         rte_timer_init(&adapter->timer_wd);
2228
2229         adapters_found++;
2230         adapter->state = ENA_ADAPTER_STATE_INIT;
2231
2232         return 0;
2233
2234 err_rss_destroy:
2235         ena_com_rss_destroy(ena_dev);
2236 err_delete_debug_area:
2237         ena_com_delete_debug_area(ena_dev);
2238
2239 err_device_destroy:
2240         ena_com_delete_host_info(ena_dev);
2241         ena_com_admin_destroy(ena_dev);
2242
2243 err:
2244         return rc;
2245 }
2246
2247 static void ena_destroy_device(struct rte_eth_dev *eth_dev)
2248 {
2249         struct ena_adapter *adapter = eth_dev->data->dev_private;
2250         struct ena_com_dev *ena_dev = &adapter->ena_dev;
2251
2252         if (adapter->state == ENA_ADAPTER_STATE_FREE)
2253                 return;
2254
2255         ena_com_set_admin_running_state(ena_dev, false);
2256
2257         if (adapter->state != ENA_ADAPTER_STATE_CLOSED)
2258                 ena_close(eth_dev);
2259
2260         ena_com_rss_destroy(ena_dev);
2261
2262         ena_com_delete_debug_area(ena_dev);
2263         ena_com_delete_host_info(ena_dev);
2264
2265         ena_com_abort_admin_commands(ena_dev);
2266         ena_com_wait_for_abort_completion(ena_dev);
2267         ena_com_admin_destroy(ena_dev);
2268         ena_com_mmio_reg_read_request_destroy(ena_dev);
2269
2270         adapter->state = ENA_ADAPTER_STATE_FREE;
2271 }
2272
2273 static int eth_ena_dev_uninit(struct rte_eth_dev *eth_dev)
2274 {
2275         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2276                 return 0;
2277
2278         ena_destroy_device(eth_dev);
2279
2280         return 0;
2281 }
2282
2283 static int ena_dev_configure(struct rte_eth_dev *dev)
2284 {
2285         struct ena_adapter *adapter = dev->data->dev_private;
2286         int rc;
2287
2288         adapter->state = ENA_ADAPTER_STATE_CONFIG;
2289
2290         if (dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
2291                 dev->data->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
2292         dev->data->dev_conf.txmode.offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
2293
2294         /* Scattered Rx cannot be turned off in the HW, so this capability must
2295          * be forced.
2296          */
2297         dev->data->scattered_rx = 1;
2298
2299         adapter->last_tx_comp_qid = 0;
2300
2301         adapter->missing_tx_completion_budget =
2302                 RTE_MIN(ENA_MONITORED_TX_QUEUES, dev->data->nb_tx_queues);
2303
2304         adapter->missing_tx_completion_to = ENA_TX_TIMEOUT;
2305         /* To avoid detection of the spurious Tx completion timeout due to
2306          * application not calling the Tx cleanup function, set timeout for the
2307          * Tx queue which should be half of the missing completion timeout for a
2308          * safety. If there will be a lot of missing Tx completions in the
2309          * queue, they will be detected sooner or later.
2310          */
2311         adapter->tx_cleanup_stall_delay = adapter->missing_tx_completion_to / 2;
2312
2313         rc = ena_configure_aenq(adapter);
2314
2315         return rc;
2316 }
2317
2318 static void ena_init_rings(struct ena_adapter *adapter,
2319                            bool disable_meta_caching)
2320 {
2321         size_t i;
2322
2323         for (i = 0; i < adapter->max_num_io_queues; i++) {
2324                 struct ena_ring *ring = &adapter->tx_ring[i];
2325
2326                 ring->configured = 0;
2327                 ring->type = ENA_RING_TYPE_TX;
2328                 ring->adapter = adapter;
2329                 ring->id = i;
2330                 ring->tx_mem_queue_type = adapter->ena_dev.tx_mem_queue_type;
2331                 ring->tx_max_header_size = adapter->ena_dev.tx_max_header_size;
2332                 ring->sgl_size = adapter->max_tx_sgl_size;
2333                 ring->disable_meta_caching = disable_meta_caching;
2334         }
2335
2336         for (i = 0; i < adapter->max_num_io_queues; i++) {
2337                 struct ena_ring *ring = &adapter->rx_ring[i];
2338
2339                 ring->configured = 0;
2340                 ring->type = ENA_RING_TYPE_RX;
2341                 ring->adapter = adapter;
2342                 ring->id = i;
2343                 ring->sgl_size = adapter->max_rx_sgl_size;
2344         }
2345 }
2346
2347 static uint64_t ena_get_rx_port_offloads(struct ena_adapter *adapter)
2348 {
2349         uint64_t port_offloads = 0;
2350
2351         if (adapter->offloads.rx_offloads & ENA_L3_IPV4_CSUM)
2352                 port_offloads |= RTE_ETH_RX_OFFLOAD_IPV4_CKSUM;
2353
2354         if (adapter->offloads.rx_offloads &
2355             (ENA_L4_IPV4_CSUM | ENA_L4_IPV6_CSUM))
2356                 port_offloads |=
2357                         RTE_ETH_RX_OFFLOAD_UDP_CKSUM | RTE_ETH_RX_OFFLOAD_TCP_CKSUM;
2358
2359         if (adapter->offloads.rx_offloads & ENA_RX_RSS_HASH)
2360                 port_offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
2361
2362         port_offloads |= RTE_ETH_RX_OFFLOAD_SCATTER;
2363
2364         return port_offloads;
2365 }
2366
2367 static uint64_t ena_get_tx_port_offloads(struct ena_adapter *adapter)
2368 {
2369         uint64_t port_offloads = 0;
2370
2371         if (adapter->offloads.tx_offloads & ENA_IPV4_TSO)
2372                 port_offloads |= RTE_ETH_TX_OFFLOAD_TCP_TSO;
2373
2374         if (adapter->offloads.tx_offloads & ENA_L3_IPV4_CSUM)
2375                 port_offloads |= RTE_ETH_TX_OFFLOAD_IPV4_CKSUM;
2376         if (adapter->offloads.tx_offloads &
2377             (ENA_L4_IPV4_CSUM_PARTIAL | ENA_L4_IPV4_CSUM |
2378              ENA_L4_IPV6_CSUM | ENA_L4_IPV6_CSUM_PARTIAL))
2379                 port_offloads |=
2380                         RTE_ETH_TX_OFFLOAD_UDP_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_CKSUM;
2381
2382         port_offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS;
2383
2384         return port_offloads;
2385 }
2386
2387 static uint64_t ena_get_rx_queue_offloads(struct ena_adapter *adapter)
2388 {
2389         RTE_SET_USED(adapter);
2390
2391         return 0;
2392 }
2393
2394 static uint64_t ena_get_tx_queue_offloads(struct ena_adapter *adapter)
2395 {
2396         RTE_SET_USED(adapter);
2397
2398         return 0;
2399 }
2400
2401 static int ena_infos_get(struct rte_eth_dev *dev,
2402                           struct rte_eth_dev_info *dev_info)
2403 {
2404         struct ena_adapter *adapter;
2405         struct ena_com_dev *ena_dev;
2406
2407         ena_assert_msg(dev->data != NULL, "Uninitialized device\n");
2408         ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n");
2409         adapter = dev->data->dev_private;
2410
2411         ena_dev = &adapter->ena_dev;
2412         ena_assert_msg(ena_dev != NULL, "Uninitialized device\n");
2413
2414         dev_info->speed_capa =
2415                         RTE_ETH_LINK_SPEED_1G   |
2416                         RTE_ETH_LINK_SPEED_2_5G |
2417                         RTE_ETH_LINK_SPEED_5G   |
2418                         RTE_ETH_LINK_SPEED_10G  |
2419                         RTE_ETH_LINK_SPEED_25G  |
2420                         RTE_ETH_LINK_SPEED_40G  |
2421                         RTE_ETH_LINK_SPEED_50G  |
2422                         RTE_ETH_LINK_SPEED_100G;
2423
2424         /* Inform framework about available features */
2425         dev_info->rx_offload_capa = ena_get_rx_port_offloads(adapter);
2426         dev_info->tx_offload_capa = ena_get_tx_port_offloads(adapter);
2427         dev_info->rx_queue_offload_capa = ena_get_rx_queue_offloads(adapter);
2428         dev_info->tx_queue_offload_capa = ena_get_tx_queue_offloads(adapter);
2429
2430         dev_info->flow_type_rss_offloads = ENA_ALL_RSS_HF;
2431         dev_info->hash_key_size = ENA_HASH_KEY_SIZE;
2432
2433         dev_info->min_rx_bufsize = ENA_MIN_FRAME_LEN;
2434         dev_info->max_rx_pktlen  = adapter->max_mtu + RTE_ETHER_HDR_LEN +
2435                 RTE_ETHER_CRC_LEN;
2436         dev_info->min_mtu = ENA_MIN_MTU;
2437         dev_info->max_mtu = adapter->max_mtu;
2438         dev_info->max_mac_addrs = 1;
2439
2440         dev_info->max_rx_queues = adapter->max_num_io_queues;
2441         dev_info->max_tx_queues = adapter->max_num_io_queues;
2442         dev_info->reta_size = ENA_RX_RSS_TABLE_SIZE;
2443
2444         dev_info->rx_desc_lim.nb_max = adapter->max_rx_ring_size;
2445         dev_info->rx_desc_lim.nb_min = ENA_MIN_RING_DESC;
2446         dev_info->rx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS,
2447                                         adapter->max_rx_sgl_size);
2448         dev_info->rx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS,
2449                                         adapter->max_rx_sgl_size);
2450
2451         dev_info->tx_desc_lim.nb_max = adapter->max_tx_ring_size;
2452         dev_info->tx_desc_lim.nb_min = ENA_MIN_RING_DESC;
2453         dev_info->tx_desc_lim.nb_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS,
2454                                         adapter->max_tx_sgl_size);
2455         dev_info->tx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS,
2456                                         adapter->max_tx_sgl_size);
2457
2458         dev_info->default_rxportconf.ring_size = ENA_DEFAULT_RING_SIZE;
2459         dev_info->default_txportconf.ring_size = ENA_DEFAULT_RING_SIZE;
2460
2461         return 0;
2462 }
2463
2464 static inline void ena_init_rx_mbuf(struct rte_mbuf *mbuf, uint16_t len)
2465 {
2466         mbuf->data_len = len;
2467         mbuf->data_off = RTE_PKTMBUF_HEADROOM;
2468         mbuf->refcnt = 1;
2469         mbuf->next = NULL;
2470 }
2471
2472 static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring,
2473                                     struct ena_com_rx_buf_info *ena_bufs,
2474                                     uint32_t descs,
2475                                     uint16_t *next_to_clean,
2476                                     uint8_t offset)
2477 {
2478         struct rte_mbuf *mbuf;
2479         struct rte_mbuf *mbuf_head;
2480         struct ena_rx_buffer *rx_info;
2481         int rc;
2482         uint16_t ntc, len, req_id, buf = 0;
2483
2484         if (unlikely(descs == 0))
2485                 return NULL;
2486
2487         ntc = *next_to_clean;
2488
2489         len = ena_bufs[buf].len;
2490         req_id = ena_bufs[buf].req_id;
2491
2492         rx_info = &rx_ring->rx_buffer_info[req_id];
2493
2494         mbuf = rx_info->mbuf;
2495         RTE_ASSERT(mbuf != NULL);
2496
2497         ena_init_rx_mbuf(mbuf, len);
2498
2499         /* Fill the mbuf head with the data specific for 1st segment. */
2500         mbuf_head = mbuf;
2501         mbuf_head->nb_segs = descs;
2502         mbuf_head->port = rx_ring->port_id;
2503         mbuf_head->pkt_len = len;
2504         mbuf_head->data_off += offset;
2505
2506         rx_info->mbuf = NULL;
2507         rx_ring->empty_rx_reqs[ntc] = req_id;
2508         ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask);
2509
2510         while (--descs) {
2511                 ++buf;
2512                 len = ena_bufs[buf].len;
2513                 req_id = ena_bufs[buf].req_id;
2514
2515                 rx_info = &rx_ring->rx_buffer_info[req_id];
2516                 RTE_ASSERT(rx_info->mbuf != NULL);
2517
2518                 if (unlikely(len == 0)) {
2519                         /*
2520                          * Some devices can pass descriptor with the length 0.
2521                          * To avoid confusion, the PMD is simply putting the
2522                          * descriptor back, as it was never used. We'll avoid
2523                          * mbuf allocation that way.
2524                          */
2525                         rc = ena_add_single_rx_desc(rx_ring->ena_com_io_sq,
2526                                 rx_info->mbuf, req_id);
2527                         if (unlikely(rc != 0)) {
2528                                 /* Free the mbuf in case of an error. */
2529                                 rte_mbuf_raw_free(rx_info->mbuf);
2530                         } else {
2531                                 /*
2532                                  * If there was no error, just exit the loop as
2533                                  * 0 length descriptor is always the last one.
2534                                  */
2535                                 break;
2536                         }
2537                 } else {
2538                         /* Create an mbuf chain. */
2539                         mbuf->next = rx_info->mbuf;
2540                         mbuf = mbuf->next;
2541
2542                         ena_init_rx_mbuf(mbuf, len);
2543                         mbuf_head->pkt_len += len;
2544                 }
2545
2546                 /*
2547                  * Mark the descriptor as depleted and perform necessary
2548                  * cleanup.
2549                  * This code will execute in two cases:
2550                  *  1. Descriptor len was greater than 0 - normal situation.
2551                  *  2. Descriptor len was 0 and we failed to add the descriptor
2552                  *     to the device. In that situation, we should try to add
2553                  *     the mbuf again in the populate routine and mark the
2554                  *     descriptor as used up by the device.
2555                  */
2556                 rx_info->mbuf = NULL;
2557                 rx_ring->empty_rx_reqs[ntc] = req_id;
2558                 ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask);
2559         }
2560
2561         *next_to_clean = ntc;
2562
2563         return mbuf_head;
2564 }
2565
2566 static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
2567                                   uint16_t nb_pkts)
2568 {
2569         struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue);
2570         unsigned int free_queue_entries;
2571         uint16_t next_to_clean = rx_ring->next_to_clean;
2572         uint16_t descs_in_use;
2573         struct rte_mbuf *mbuf;
2574         uint16_t completed;
2575         struct ena_com_rx_ctx ena_rx_ctx;
2576         int i, rc = 0;
2577         bool fill_hash;
2578
2579 #ifdef RTE_ETHDEV_DEBUG_RX
2580         /* Check adapter state */
2581         if (unlikely(rx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) {
2582                 PMD_RX_LOG(ALERT,
2583                         "Trying to receive pkts while device is NOT running\n");
2584                 return 0;
2585         }
2586 #endif
2587
2588         fill_hash = rx_ring->offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH;
2589
2590         descs_in_use = rx_ring->ring_size -
2591                 ena_com_free_q_entries(rx_ring->ena_com_io_sq) - 1;
2592         nb_pkts = RTE_MIN(descs_in_use, nb_pkts);
2593
2594         for (completed = 0; completed < nb_pkts; completed++) {
2595                 ena_rx_ctx.max_bufs = rx_ring->sgl_size;
2596                 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
2597                 ena_rx_ctx.descs = 0;
2598                 ena_rx_ctx.pkt_offset = 0;
2599                 /* receive packet context */
2600                 rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
2601                                     rx_ring->ena_com_io_sq,
2602                                     &ena_rx_ctx);
2603                 if (unlikely(rc)) {
2604                         PMD_RX_LOG(ERR,
2605                                 "Failed to get the packet from the device, rc: %d\n",
2606                                 rc);
2607                         if (rc == ENA_COM_NO_SPACE) {
2608                                 ++rx_ring->rx_stats.bad_desc_num;
2609                                 rx_ring->adapter->reset_reason =
2610                                         ENA_REGS_RESET_TOO_MANY_RX_DESCS;
2611                         } else {
2612                                 ++rx_ring->rx_stats.bad_req_id;
2613                                 rx_ring->adapter->reset_reason =
2614                                         ENA_REGS_RESET_INV_RX_REQ_ID;
2615                         }
2616                         rx_ring->adapter->trigger_reset = true;
2617                         return 0;
2618                 }
2619
2620                 mbuf = ena_rx_mbuf(rx_ring,
2621                         ena_rx_ctx.ena_bufs,
2622                         ena_rx_ctx.descs,
2623                         &next_to_clean,
2624                         ena_rx_ctx.pkt_offset);
2625                 if (unlikely(mbuf == NULL)) {
2626                         for (i = 0; i < ena_rx_ctx.descs; ++i) {
2627                                 rx_ring->empty_rx_reqs[next_to_clean] =
2628                                         rx_ring->ena_bufs[i].req_id;
2629                                 next_to_clean = ENA_IDX_NEXT_MASKED(
2630                                         next_to_clean, rx_ring->size_mask);
2631                         }
2632                         break;
2633                 }
2634
2635                 /* fill mbuf attributes if any */
2636                 ena_rx_mbuf_prepare(rx_ring, mbuf, &ena_rx_ctx, fill_hash);
2637
2638                 if (unlikely(mbuf->ol_flags &
2639                                 (RTE_MBUF_F_RX_IP_CKSUM_BAD | RTE_MBUF_F_RX_L4_CKSUM_BAD)))
2640                         rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors);
2641
2642                 rx_pkts[completed] = mbuf;
2643                 rx_ring->rx_stats.bytes += mbuf->pkt_len;
2644         }
2645
2646         rx_ring->rx_stats.cnt += completed;
2647         rx_ring->next_to_clean = next_to_clean;
2648
2649         free_queue_entries = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
2650
2651         /* Burst refill to save doorbells, memory barriers, const interval */
2652         if (free_queue_entries >= rx_ring->rx_free_thresh) {
2653                 ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
2654                 ena_populate_rx_queue(rx_ring, free_queue_entries);
2655         }
2656
2657         return completed;
2658 }
2659
2660 static uint16_t
2661 eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
2662                 uint16_t nb_pkts)
2663 {
2664         int32_t ret;
2665         uint32_t i;
2666         struct rte_mbuf *m;
2667         struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue);
2668         struct ena_adapter *adapter = tx_ring->adapter;
2669         struct rte_ipv4_hdr *ip_hdr;
2670         uint64_t ol_flags;
2671         uint64_t l4_csum_flag;
2672         uint64_t dev_offload_capa;
2673         uint16_t frag_field;
2674         bool need_pseudo_csum;
2675
2676         dev_offload_capa = adapter->offloads.tx_offloads;
2677         for (i = 0; i != nb_pkts; i++) {
2678                 m = tx_pkts[i];
2679                 ol_flags = m->ol_flags;
2680
2681                 /* Check if any offload flag was set */
2682                 if (ol_flags == 0)
2683                         continue;
2684
2685                 l4_csum_flag = ol_flags & RTE_MBUF_F_TX_L4_MASK;
2686                 /* SCTP checksum offload is not supported by the ENA. */
2687                 if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) ||
2688                     l4_csum_flag == RTE_MBUF_F_TX_SCTP_CKSUM) {
2689                         PMD_TX_LOG(DEBUG,
2690                                 "mbuf[%" PRIu32 "] has unsupported offloads flags set: 0x%" PRIu64 "\n",
2691                                 i, ol_flags);
2692                         rte_errno = ENOTSUP;
2693                         return i;
2694                 }
2695
2696                 if (unlikely(m->nb_segs >= tx_ring->sgl_size &&
2697                     !(tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV &&
2698                       m->nb_segs == tx_ring->sgl_size &&
2699                       m->data_len < tx_ring->tx_max_header_size))) {
2700                         PMD_TX_LOG(DEBUG,
2701                                 "mbuf[%" PRIu32 "] has too many segments: %" PRIu16 "\n",
2702                                 i, m->nb_segs);
2703                         rte_errno = EINVAL;
2704                         return i;
2705                 }
2706
2707 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
2708                 /* Check if requested offload is also enabled for the queue */
2709                 if ((ol_flags & RTE_MBUF_F_TX_IP_CKSUM &&
2710                      !(tx_ring->offloads & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM)) ||
2711                     (l4_csum_flag == RTE_MBUF_F_TX_TCP_CKSUM &&
2712                      !(tx_ring->offloads & RTE_ETH_TX_OFFLOAD_TCP_CKSUM)) ||
2713                     (l4_csum_flag == RTE_MBUF_F_TX_UDP_CKSUM &&
2714                      !(tx_ring->offloads & RTE_ETH_TX_OFFLOAD_UDP_CKSUM))) {
2715                         PMD_TX_LOG(DEBUG,
2716                                 "mbuf[%" PRIu32 "]: requested offloads: %" PRIu16 " are not enabled for the queue[%u]\n",
2717                                 i, m->nb_segs, tx_ring->id);
2718                         rte_errno = EINVAL;
2719                         return i;
2720                 }
2721
2722                 /* The caller is obligated to set l2 and l3 len if any cksum
2723                  * offload is enabled.
2724                  */
2725                 if (unlikely(ol_flags & (RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_L4_MASK) &&
2726                     (m->l2_len == 0 || m->l3_len == 0))) {
2727                         PMD_TX_LOG(DEBUG,
2728                                 "mbuf[%" PRIu32 "]: l2_len or l3_len values are 0 while the offload was requested\n",
2729                                 i);
2730                         rte_errno = EINVAL;
2731                         return i;
2732                 }
2733                 ret = rte_validate_tx_offload(m);
2734                 if (ret != 0) {
2735                         rte_errno = -ret;
2736                         return i;
2737                 }
2738 #endif
2739
2740                 /* Verify HW support for requested offloads and determine if
2741                  * pseudo header checksum is needed.
2742                  */
2743                 need_pseudo_csum = false;
2744                 if (ol_flags & RTE_MBUF_F_TX_IPV4) {
2745                         if (ol_flags & RTE_MBUF_F_TX_IP_CKSUM &&
2746                             !(dev_offload_capa & ENA_L3_IPV4_CSUM)) {
2747                                 rte_errno = ENOTSUP;
2748                                 return i;
2749                         }
2750
2751                         if (ol_flags & RTE_MBUF_F_TX_TCP_SEG &&
2752                             !(dev_offload_capa & ENA_IPV4_TSO)) {
2753                                 rte_errno = ENOTSUP;
2754                                 return i;
2755                         }
2756
2757                         /* Check HW capabilities and if pseudo csum is needed
2758                          * for L4 offloads.
2759                          */
2760                         if (l4_csum_flag != RTE_MBUF_F_TX_L4_NO_CKSUM &&
2761                             !(dev_offload_capa & ENA_L4_IPV4_CSUM)) {
2762                                 if (dev_offload_capa &
2763                                     ENA_L4_IPV4_CSUM_PARTIAL) {
2764                                         need_pseudo_csum = true;
2765                                 } else {
2766                                         rte_errno = ENOTSUP;
2767                                         return i;
2768                                 }
2769                         }
2770
2771                         /* Parse the DF flag */
2772                         ip_hdr = rte_pktmbuf_mtod_offset(m,
2773                                 struct rte_ipv4_hdr *, m->l2_len);
2774                         frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset);
2775                         if (frag_field & RTE_IPV4_HDR_DF_FLAG) {
2776                                 m->packet_type |= RTE_PTYPE_L4_NONFRAG;
2777                         } else if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
2778                                 /* In case we are supposed to TSO and have DF
2779                                  * not set (DF=0) hardware must be provided with
2780                                  * partial checksum.
2781                                  */
2782                                 need_pseudo_csum = true;
2783                         }
2784                 } else if (ol_flags & RTE_MBUF_F_TX_IPV6) {
2785                         /* There is no support for IPv6 TSO as for now. */
2786                         if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
2787                                 rte_errno = ENOTSUP;
2788                                 return i;
2789                         }
2790
2791                         /* Check HW capabilities and if pseudo csum is needed */
2792                         if (l4_csum_flag != RTE_MBUF_F_TX_L4_NO_CKSUM &&
2793                             !(dev_offload_capa & ENA_L4_IPV6_CSUM)) {
2794                                 if (dev_offload_capa &
2795                                     ENA_L4_IPV6_CSUM_PARTIAL) {
2796                                         need_pseudo_csum = true;
2797                                 } else {
2798                                         rte_errno = ENOTSUP;
2799                                         return i;
2800                                 }
2801                         }
2802                 }
2803
2804                 if (need_pseudo_csum) {
2805                         ret = rte_net_intel_cksum_flags_prepare(m, ol_flags);
2806                         if (ret != 0) {
2807                                 rte_errno = -ret;
2808                                 return i;
2809                         }
2810                 }
2811         }
2812
2813         return i;
2814 }
2815
2816 static void ena_update_hints(struct ena_adapter *adapter,
2817                              struct ena_admin_ena_hw_hints *hints)
2818 {
2819         if (hints->admin_completion_tx_timeout)
2820                 adapter->ena_dev.admin_queue.completion_timeout =
2821                         hints->admin_completion_tx_timeout * 1000;
2822
2823         if (hints->mmio_read_timeout)
2824                 /* convert to usec */
2825                 adapter->ena_dev.mmio_read.reg_read_to =
2826                         hints->mmio_read_timeout * 1000;
2827
2828         if (hints->missing_tx_completion_timeout) {
2829                 if (hints->missing_tx_completion_timeout ==
2830                     ENA_HW_HINTS_NO_TIMEOUT) {
2831                         adapter->missing_tx_completion_to =
2832                                 ENA_HW_HINTS_NO_TIMEOUT;
2833                 } else {
2834                         /* Convert from msecs to ticks */
2835                         adapter->missing_tx_completion_to = rte_get_timer_hz() *
2836                                 hints->missing_tx_completion_timeout / 1000;
2837                         adapter->tx_cleanup_stall_delay =
2838                                 adapter->missing_tx_completion_to / 2;
2839                 }
2840         }
2841
2842         if (hints->driver_watchdog_timeout) {
2843                 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
2844                         adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
2845                 else
2846                         // Convert msecs to ticks
2847                         adapter->keep_alive_timeout =
2848                                 (hints->driver_watchdog_timeout *
2849                                 rte_get_timer_hz()) / 1000;
2850         }
2851 }
2852
2853 static void ena_tx_map_mbuf(struct ena_ring *tx_ring,
2854         struct ena_tx_buffer *tx_info,
2855         struct rte_mbuf *mbuf,
2856         void **push_header,
2857         uint16_t *header_len)
2858 {
2859         struct ena_com_buf *ena_buf;
2860         uint16_t delta, seg_len, push_len;
2861
2862         delta = 0;
2863         seg_len = mbuf->data_len;
2864
2865         tx_info->mbuf = mbuf;
2866         ena_buf = tx_info->bufs;
2867
2868         if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2869                 /*
2870                  * Tx header might be (and will be in most cases) smaller than
2871                  * tx_max_header_size. But it's not an issue to send more data
2872                  * to the device, than actually needed if the mbuf size is
2873                  * greater than tx_max_header_size.
2874                  */
2875                 push_len = RTE_MIN(mbuf->pkt_len, tx_ring->tx_max_header_size);
2876                 *header_len = push_len;
2877
2878                 if (likely(push_len <= seg_len)) {
2879                         /* If the push header is in the single segment, then
2880                          * just point it to the 1st mbuf data.
2881                          */
2882                         *push_header = rte_pktmbuf_mtod(mbuf, uint8_t *);
2883                 } else {
2884                         /* If the push header lays in the several segments, copy
2885                          * it to the intermediate buffer.
2886                          */
2887                         rte_pktmbuf_read(mbuf, 0, push_len,
2888                                 tx_ring->push_buf_intermediate_buf);
2889                         *push_header = tx_ring->push_buf_intermediate_buf;
2890                         delta = push_len - seg_len;
2891                 }
2892         } else {
2893                 *push_header = NULL;
2894                 *header_len = 0;
2895                 push_len = 0;
2896         }
2897
2898         /* Process first segment taking into consideration pushed header */
2899         if (seg_len > push_len) {
2900                 ena_buf->paddr = mbuf->buf_iova +
2901                                 mbuf->data_off +
2902                                 push_len;
2903                 ena_buf->len = seg_len - push_len;
2904                 ena_buf++;
2905                 tx_info->num_of_bufs++;
2906         }
2907
2908         while ((mbuf = mbuf->next) != NULL) {
2909                 seg_len = mbuf->data_len;
2910
2911                 /* Skip mbufs if whole data is pushed as a header */
2912                 if (unlikely(delta > seg_len)) {
2913                         delta -= seg_len;
2914                         continue;
2915                 }
2916
2917                 ena_buf->paddr = mbuf->buf_iova + mbuf->data_off + delta;
2918                 ena_buf->len = seg_len - delta;
2919                 ena_buf++;
2920                 tx_info->num_of_bufs++;
2921
2922                 delta = 0;
2923         }
2924 }
2925
2926 static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf)
2927 {
2928         struct ena_tx_buffer *tx_info;
2929         struct ena_com_tx_ctx ena_tx_ctx = { { 0 } };
2930         uint16_t next_to_use;
2931         uint16_t header_len;
2932         uint16_t req_id;
2933         void *push_header;
2934         int nb_hw_desc;
2935         int rc;
2936
2937         /* Checking for space for 2 additional metadata descriptors due to
2938          * possible header split and metadata descriptor
2939          */
2940         if (!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
2941                                           mbuf->nb_segs + 2)) {
2942                 PMD_DRV_LOG(DEBUG, "Not enough space in the tx queue\n");
2943                 return ENA_COM_NO_MEM;
2944         }
2945
2946         next_to_use = tx_ring->next_to_use;
2947
2948         req_id = tx_ring->empty_tx_reqs[next_to_use];
2949         tx_info = &tx_ring->tx_buffer_info[req_id];
2950         tx_info->num_of_bufs = 0;
2951         RTE_ASSERT(tx_info->mbuf == NULL);
2952
2953         ena_tx_map_mbuf(tx_ring, tx_info, mbuf, &push_header, &header_len);
2954
2955         ena_tx_ctx.ena_bufs = tx_info->bufs;
2956         ena_tx_ctx.push_header = push_header;
2957         ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
2958         ena_tx_ctx.req_id = req_id;
2959         ena_tx_ctx.header_len = header_len;
2960
2961         /* Set Tx offloads flags, if applicable */
2962         ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads,
2963                 tx_ring->disable_meta_caching);
2964
2965         if (unlikely(ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq,
2966                         &ena_tx_ctx))) {
2967                 PMD_TX_LOG(DEBUG,
2968                         "LLQ Tx max burst size of queue %d achieved, writing doorbell to send burst\n",
2969                         tx_ring->id);
2970                 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
2971                 tx_ring->tx_stats.doorbells++;
2972                 tx_ring->pkts_without_db = false;
2973         }
2974
2975         /* prepare the packet's descriptors to dma engine */
2976         rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx,
2977                 &nb_hw_desc);
2978         if (unlikely(rc)) {
2979                 PMD_DRV_LOG(ERR, "Failed to prepare Tx buffers, rc: %d\n", rc);
2980                 ++tx_ring->tx_stats.prepare_ctx_err;
2981                 tx_ring->adapter->reset_reason =
2982                     ENA_REGS_RESET_DRIVER_INVALID_STATE;
2983                 tx_ring->adapter->trigger_reset = true;
2984                 return rc;
2985         }
2986
2987         tx_info->tx_descs = nb_hw_desc;
2988         tx_info->timestamp = rte_get_timer_cycles();
2989
2990         tx_ring->tx_stats.cnt++;
2991         tx_ring->tx_stats.bytes += mbuf->pkt_len;
2992
2993         tx_ring->next_to_use = ENA_IDX_NEXT_MASKED(next_to_use,
2994                 tx_ring->size_mask);
2995
2996         return 0;
2997 }
2998
2999 static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
3000 {
3001         struct ena_ring *tx_ring = (struct ena_ring *)txp;
3002         unsigned int total_tx_descs = 0;
3003         unsigned int total_tx_pkts = 0;
3004         uint16_t cleanup_budget;
3005         uint16_t next_to_clean = tx_ring->next_to_clean;
3006
3007         /*
3008          * If free_pkt_cnt is equal to 0, it means that the user requested
3009          * full cleanup, so attempt to release all Tx descriptors
3010          * (ring_size - 1 -> size_mask)
3011          */
3012         cleanup_budget = (free_pkt_cnt == 0) ? tx_ring->size_mask : free_pkt_cnt;
3013
3014         while (likely(total_tx_pkts < cleanup_budget)) {
3015                 struct rte_mbuf *mbuf;
3016                 struct ena_tx_buffer *tx_info;
3017                 uint16_t req_id;
3018
3019                 if (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) != 0)
3020                         break;
3021
3022                 if (unlikely(validate_tx_req_id(tx_ring, req_id) != 0))
3023                         break;
3024
3025                 /* Get Tx info & store how many descs were processed  */
3026                 tx_info = &tx_ring->tx_buffer_info[req_id];
3027                 tx_info->timestamp = 0;
3028
3029                 mbuf = tx_info->mbuf;
3030                 rte_pktmbuf_free(mbuf);
3031
3032                 tx_info->mbuf = NULL;
3033                 tx_ring->empty_tx_reqs[next_to_clean] = req_id;
3034
3035                 total_tx_descs += tx_info->tx_descs;
3036                 total_tx_pkts++;
3037
3038                 /* Put back descriptor to the ring for reuse */
3039                 next_to_clean = ENA_IDX_NEXT_MASKED(next_to_clean,
3040                         tx_ring->size_mask);
3041         }
3042
3043         if (likely(total_tx_descs > 0)) {
3044                 /* acknowledge completion of sent packets */
3045                 tx_ring->next_to_clean = next_to_clean;
3046                 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs);
3047                 ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
3048         }
3049
3050         /* Notify completion handler that full cleanup was performed */
3051         if (free_pkt_cnt == 0 || total_tx_pkts < cleanup_budget)
3052                 tx_ring->last_cleanup_ticks = rte_get_timer_cycles();
3053
3054         return total_tx_pkts;
3055 }
3056
3057 static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
3058                                   uint16_t nb_pkts)
3059 {
3060         struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue);
3061         int available_desc;
3062         uint16_t sent_idx = 0;
3063
3064 #ifdef RTE_ETHDEV_DEBUG_TX
3065         /* Check adapter state */
3066         if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) {
3067                 PMD_TX_LOG(ALERT,
3068                         "Trying to xmit pkts while device is NOT running\n");
3069                 return 0;
3070         }
3071 #endif
3072
3073         available_desc = ena_com_free_q_entries(tx_ring->ena_com_io_sq);
3074         if (available_desc < tx_ring->tx_free_thresh)
3075                 ena_tx_cleanup((void *)tx_ring, 0);
3076
3077         for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) {
3078                 if (ena_xmit_mbuf(tx_ring, tx_pkts[sent_idx]))
3079                         break;
3080                 tx_ring->pkts_without_db = true;
3081                 rte_prefetch0(tx_pkts[ENA_IDX_ADD_MASKED(sent_idx, 4,
3082                         tx_ring->size_mask)]);
3083         }
3084
3085         /* If there are ready packets to be xmitted... */
3086         if (likely(tx_ring->pkts_without_db)) {
3087                 /* ...let HW do its best :-) */
3088                 ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
3089                 tx_ring->tx_stats.doorbells++;
3090                 tx_ring->pkts_without_db = false;
3091         }
3092
3093         tx_ring->tx_stats.available_desc =
3094                 ena_com_free_q_entries(tx_ring->ena_com_io_sq);
3095         tx_ring->tx_stats.tx_poll++;
3096
3097         return sent_idx;
3098 }
3099
3100 int ena_copy_eni_stats(struct ena_adapter *adapter, struct ena_stats_eni *stats)
3101 {
3102         int rc;
3103
3104         rte_spinlock_lock(&adapter->admin_lock);
3105         /* Retrieve and store the latest statistics from the AQ. This ensures
3106          * that previous value is returned in case of a com error.
3107          */
3108         rc = ENA_PROXY(adapter, ena_com_get_eni_stats, &adapter->ena_dev,
3109                 (struct ena_admin_eni_stats *)stats);
3110         rte_spinlock_unlock(&adapter->admin_lock);
3111         if (rc != 0) {
3112                 if (rc == ENA_COM_UNSUPPORTED) {
3113                         PMD_DRV_LOG(DEBUG,
3114                                 "Retrieving ENI metrics is not supported\n");
3115                 } else {
3116                         PMD_DRV_LOG(WARNING,
3117                                 "Failed to get ENI metrics, rc: %d\n", rc);
3118                 }
3119                 return rc;
3120         }
3121
3122         return 0;
3123 }
3124
3125 /**
3126  * DPDK callback to retrieve names of extended device statistics
3127  *
3128  * @param dev
3129  *   Pointer to Ethernet device structure.
3130  * @param[out] xstats_names
3131  *   Buffer to insert names into.
3132  * @param n
3133  *   Number of names.
3134  *
3135  * @return
3136  *   Number of xstats names.
3137  */
3138 static int ena_xstats_get_names(struct rte_eth_dev *dev,
3139                                 struct rte_eth_xstat_name *xstats_names,
3140                                 unsigned int n)
3141 {
3142         unsigned int xstats_count = ena_xstats_calc_num(dev->data);
3143         unsigned int stat, i, count = 0;
3144
3145         if (n < xstats_count || !xstats_names)
3146                 return xstats_count;
3147
3148         for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++)
3149                 strcpy(xstats_names[count].name,
3150                         ena_stats_global_strings[stat].name);
3151
3152         for (stat = 0; stat < ENA_STATS_ARRAY_ENI; stat++, count++)
3153                 strcpy(xstats_names[count].name,
3154                         ena_stats_eni_strings[stat].name);
3155
3156         for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++)
3157                 for (i = 0; i < dev->data->nb_rx_queues; i++, count++)
3158                         snprintf(xstats_names[count].name,
3159                                 sizeof(xstats_names[count].name),
3160                                 "rx_q%d_%s", i,
3161                                 ena_stats_rx_strings[stat].name);
3162
3163         for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++)
3164                 for (i = 0; i < dev->data->nb_tx_queues; i++, count++)
3165                         snprintf(xstats_names[count].name,
3166                                 sizeof(xstats_names[count].name),
3167                                 "tx_q%d_%s", i,
3168                                 ena_stats_tx_strings[stat].name);
3169
3170         return xstats_count;
3171 }
3172
3173 /**
3174  * DPDK callback to retrieve names of extended device statistics for the given
3175  * ids.
3176  *
3177  * @param dev
3178  *   Pointer to Ethernet device structure.
3179  * @param[out] xstats_names
3180  *   Buffer to insert names into.
3181  * @param ids
3182  *   IDs array for which the names should be retrieved.
3183  * @param size
3184  *   Number of ids.
3185  *
3186  * @return
3187  *   Positive value: number of xstats names. Negative value: error code.
3188  */
3189 static int ena_xstats_get_names_by_id(struct rte_eth_dev *dev,
3190                                       const uint64_t *ids,
3191                                       struct rte_eth_xstat_name *xstats_names,
3192                                       unsigned int size)
3193 {
3194         uint64_t xstats_count = ena_xstats_calc_num(dev->data);
3195         uint64_t id, qid;
3196         unsigned int i;
3197
3198         if (xstats_names == NULL)
3199                 return xstats_count;
3200
3201         for (i = 0; i < size; ++i) {
3202                 id = ids[i];
3203                 if (id > xstats_count) {
3204                         PMD_DRV_LOG(ERR,
3205                                 "ID value out of range: id=%" PRIu64 ", xstats_num=%" PRIu64 "\n",
3206                                  id, xstats_count);
3207                         return -EINVAL;
3208                 }
3209
3210                 if (id < ENA_STATS_ARRAY_GLOBAL) {
3211                         strcpy(xstats_names[i].name,
3212                                ena_stats_global_strings[id].name);
3213                         continue;
3214                 }
3215
3216                 id -= ENA_STATS_ARRAY_GLOBAL;
3217                 if (id < ENA_STATS_ARRAY_ENI) {
3218                         strcpy(xstats_names[i].name,
3219                                ena_stats_eni_strings[id].name);
3220                         continue;
3221                 }
3222
3223                 id -= ENA_STATS_ARRAY_ENI;
3224                 if (id < ENA_STATS_ARRAY_RX) {
3225                         qid = id / dev->data->nb_rx_queues;
3226                         id %= dev->data->nb_rx_queues;
3227                         snprintf(xstats_names[i].name,
3228                                  sizeof(xstats_names[i].name),
3229                                  "rx_q%" PRIu64 "d_%s",
3230                                  qid, ena_stats_rx_strings[id].name);
3231                         continue;
3232                 }
3233
3234                 id -= ENA_STATS_ARRAY_RX;
3235                 /* Although this condition is not needed, it was added for
3236                  * compatibility if new xstat structure would be ever added.
3237                  */
3238                 if (id < ENA_STATS_ARRAY_TX) {
3239                         qid = id / dev->data->nb_tx_queues;
3240                         id %= dev->data->nb_tx_queues;
3241                         snprintf(xstats_names[i].name,
3242                                  sizeof(xstats_names[i].name),
3243                                  "tx_q%" PRIu64 "_%s",
3244                                  qid, ena_stats_tx_strings[id].name);
3245                         continue;
3246                 }
3247         }
3248
3249         return i;
3250 }
3251
3252 /**
3253  * DPDK callback to get extended device statistics.
3254  *
3255  * @param dev
3256  *   Pointer to Ethernet device structure.
3257  * @param[out] stats
3258  *   Stats table output buffer.
3259  * @param n
3260  *   The size of the stats table.
3261  *
3262  * @return
3263  *   Number of xstats on success, negative on failure.
3264  */
3265 static int ena_xstats_get(struct rte_eth_dev *dev,
3266                           struct rte_eth_xstat *xstats,
3267                           unsigned int n)
3268 {
3269         struct ena_adapter *adapter = dev->data->dev_private;
3270         unsigned int xstats_count = ena_xstats_calc_num(dev->data);
3271         struct ena_stats_eni eni_stats;
3272         unsigned int stat, i, count = 0;
3273         int stat_offset;
3274         void *stats_begin;
3275
3276         if (n < xstats_count)
3277                 return xstats_count;
3278
3279         if (!xstats)
3280                 return 0;
3281
3282         for (stat = 0; stat < ENA_STATS_ARRAY_GLOBAL; stat++, count++) {
3283                 stat_offset = ena_stats_global_strings[stat].stat_offset;
3284                 stats_begin = &adapter->dev_stats;
3285
3286                 xstats[count].id = count;
3287                 xstats[count].value = *((uint64_t *)
3288                         ((char *)stats_begin + stat_offset));
3289         }
3290
3291         /* Even if the function below fails, we should copy previous (or initial
3292          * values) to keep structure of rte_eth_xstat consistent.
3293          */
3294         ena_copy_eni_stats(adapter, &eni_stats);
3295         for (stat = 0; stat < ENA_STATS_ARRAY_ENI; stat++, count++) {
3296                 stat_offset = ena_stats_eni_strings[stat].stat_offset;
3297                 stats_begin = &eni_stats;
3298
3299                 xstats[count].id = count;
3300                 xstats[count].value = *((uint64_t *)
3301                     ((char *)stats_begin + stat_offset));
3302         }
3303
3304         for (stat = 0; stat < ENA_STATS_ARRAY_RX; stat++) {
3305                 for (i = 0; i < dev->data->nb_rx_queues; i++, count++) {
3306                         stat_offset = ena_stats_rx_strings[stat].stat_offset;
3307                         stats_begin = &adapter->rx_ring[i].rx_stats;
3308
3309                         xstats[count].id = count;
3310                         xstats[count].value = *((uint64_t *)
3311                                 ((char *)stats_begin + stat_offset));
3312                 }
3313         }
3314
3315         for (stat = 0; stat < ENA_STATS_ARRAY_TX; stat++) {
3316                 for (i = 0; i < dev->data->nb_tx_queues; i++, count++) {
3317                         stat_offset = ena_stats_tx_strings[stat].stat_offset;
3318                         stats_begin = &adapter->tx_ring[i].rx_stats;
3319
3320                         xstats[count].id = count;
3321                         xstats[count].value = *((uint64_t *)
3322                                 ((char *)stats_begin + stat_offset));
3323                 }
3324         }
3325
3326         return count;
3327 }
3328
3329 static int ena_xstats_get_by_id(struct rte_eth_dev *dev,
3330                                 const uint64_t *ids,
3331                                 uint64_t *values,
3332                                 unsigned int n)
3333 {
3334         struct ena_adapter *adapter = dev->data->dev_private;
3335         struct ena_stats_eni eni_stats;
3336         uint64_t id;
3337         uint64_t rx_entries, tx_entries;
3338         unsigned int i;
3339         int qid;
3340         int valid = 0;
3341         bool was_eni_copied = false;
3342
3343         for (i = 0; i < n; ++i) {
3344                 id = ids[i];
3345                 /* Check if id belongs to global statistics */
3346                 if (id < ENA_STATS_ARRAY_GLOBAL) {
3347                         values[i] = *((uint64_t *)&adapter->dev_stats + id);
3348                         ++valid;
3349                         continue;
3350                 }
3351
3352                 /* Check if id belongs to ENI statistics */
3353                 id -= ENA_STATS_ARRAY_GLOBAL;
3354                 if (id < ENA_STATS_ARRAY_ENI) {
3355                         /* Avoid reading ENI stats multiple times in a single
3356                          * function call, as it requires communication with the
3357                          * admin queue.
3358                          */
3359                         if (!was_eni_copied) {
3360                                 was_eni_copied = true;
3361                                 ena_copy_eni_stats(adapter, &eni_stats);
3362                         }
3363                         values[i] = *((uint64_t *)&eni_stats + id);
3364                         ++valid;
3365                         continue;
3366                 }
3367
3368                 /* Check if id belongs to rx queue statistics */
3369                 id -= ENA_STATS_ARRAY_ENI;
3370                 rx_entries = ENA_STATS_ARRAY_RX * dev->data->nb_rx_queues;
3371                 if (id < rx_entries) {
3372                         qid = id % dev->data->nb_rx_queues;
3373                         id /= dev->data->nb_rx_queues;
3374                         values[i] = *((uint64_t *)
3375                                 &adapter->rx_ring[qid].rx_stats + id);
3376                         ++valid;
3377                         continue;
3378                 }
3379                                 /* Check if id belongs to rx queue statistics */
3380                 id -= rx_entries;
3381                 tx_entries = ENA_STATS_ARRAY_TX * dev->data->nb_tx_queues;
3382                 if (id < tx_entries) {
3383                         qid = id % dev->data->nb_tx_queues;
3384                         id /= dev->data->nb_tx_queues;
3385                         values[i] = *((uint64_t *)
3386                                 &adapter->tx_ring[qid].tx_stats + id);
3387                         ++valid;
3388                         continue;
3389                 }
3390         }
3391
3392         return valid;
3393 }
3394
3395 static int ena_process_bool_devarg(const char *key,
3396                                    const char *value,
3397                                    void *opaque)
3398 {
3399         struct ena_adapter *adapter = opaque;
3400         bool bool_value;
3401
3402         /* Parse the value. */
3403         if (strcmp(value, "1") == 0) {
3404                 bool_value = true;
3405         } else if (strcmp(value, "0") == 0) {
3406                 bool_value = false;
3407         } else {
3408                 PMD_INIT_LOG(ERR,
3409                         "Invalid value: '%s' for key '%s'. Accepted: '0' or '1'\n",
3410                         value, key);
3411                 return -EINVAL;
3412         }
3413
3414         /* Now, assign it to the proper adapter field. */
3415         if (strcmp(key, ENA_DEVARG_LARGE_LLQ_HDR) == 0)
3416                 adapter->use_large_llq_hdr = bool_value;
3417
3418         return 0;
3419 }
3420
3421 static int ena_parse_devargs(struct ena_adapter *adapter,
3422                              struct rte_devargs *devargs)
3423 {
3424         static const char * const allowed_args[] = {
3425                 ENA_DEVARG_LARGE_LLQ_HDR,
3426                 NULL,
3427         };
3428         struct rte_kvargs *kvlist;
3429         int rc;
3430
3431         if (devargs == NULL)
3432                 return 0;
3433
3434         kvlist = rte_kvargs_parse(devargs->args, allowed_args);
3435         if (kvlist == NULL) {
3436                 PMD_INIT_LOG(ERR, "Invalid device arguments: %s\n",
3437                         devargs->args);
3438                 return -EINVAL;
3439         }
3440
3441         rc = rte_kvargs_process(kvlist, ENA_DEVARG_LARGE_LLQ_HDR,
3442                 ena_process_bool_devarg, adapter);
3443
3444         rte_kvargs_free(kvlist);
3445
3446         return rc;
3447 }
3448
3449 static int ena_setup_rx_intr(struct rte_eth_dev *dev)
3450 {
3451         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
3452         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
3453         int rc;
3454         uint16_t vectors_nb, i;
3455         bool rx_intr_requested = dev->data->dev_conf.intr_conf.rxq;
3456
3457         if (!rx_intr_requested)
3458                 return 0;
3459
3460         if (!rte_intr_cap_multiple(intr_handle)) {
3461                 PMD_DRV_LOG(ERR,
3462                         "Rx interrupt requested, but it isn't supported by the PCI driver\n");
3463                 return -ENOTSUP;
3464         }
3465
3466         /* Disable interrupt mapping before the configuration starts. */
3467         rte_intr_disable(intr_handle);
3468
3469         /* Verify if there are enough vectors available. */
3470         vectors_nb = dev->data->nb_rx_queues;
3471         if (vectors_nb > RTE_MAX_RXTX_INTR_VEC_ID) {
3472                 PMD_DRV_LOG(ERR,
3473                         "Too many Rx interrupts requested, maximum number: %d\n",
3474                         RTE_MAX_RXTX_INTR_VEC_ID);
3475                 rc = -ENOTSUP;
3476                 goto enable_intr;
3477         }
3478
3479         /* Allocate the vector list */
3480         if (rte_intr_vec_list_alloc(intr_handle, "intr_vec",
3481                                            dev->data->nb_rx_queues)) {
3482                 PMD_DRV_LOG(ERR,
3483                         "Failed to allocate interrupt vector for %d queues\n",
3484                         dev->data->nb_rx_queues);
3485                 rc = -ENOMEM;
3486                 goto enable_intr;
3487         }
3488
3489         rc = rte_intr_efd_enable(intr_handle, vectors_nb);
3490         if (rc != 0)
3491                 goto free_intr_vec;
3492
3493         if (!rte_intr_allow_others(intr_handle)) {
3494                 PMD_DRV_LOG(ERR,
3495                         "Not enough interrupts available to use both ENA Admin and Rx interrupts\n");
3496                 goto disable_intr_efd;
3497         }
3498
3499         for (i = 0; i < vectors_nb; ++i)
3500                 if (rte_intr_vec_list_index_set(intr_handle, i,
3501                                            RTE_INTR_VEC_RXTX_OFFSET + i))
3502                         goto disable_intr_efd;
3503
3504         rte_intr_enable(intr_handle);
3505         return 0;
3506
3507 disable_intr_efd:
3508         rte_intr_efd_disable(intr_handle);
3509 free_intr_vec:
3510         rte_intr_vec_list_free(intr_handle);
3511 enable_intr:
3512         rte_intr_enable(intr_handle);
3513         return rc;
3514 }
3515
3516 static void ena_rx_queue_intr_set(struct rte_eth_dev *dev,
3517                                  uint16_t queue_id,
3518                                  bool unmask)
3519 {
3520         struct ena_adapter *adapter = dev->data->dev_private;
3521         struct ena_ring *rxq = &adapter->rx_ring[queue_id];
3522         struct ena_eth_io_intr_reg intr_reg;
3523
3524         ena_com_update_intr_reg(&intr_reg, 0, 0, unmask);
3525         ena_com_unmask_intr(rxq->ena_com_io_cq, &intr_reg);
3526 }
3527
3528 static int ena_rx_queue_intr_enable(struct rte_eth_dev *dev,
3529                                     uint16_t queue_id)
3530 {
3531         ena_rx_queue_intr_set(dev, queue_id, true);
3532
3533         return 0;
3534 }
3535
3536 static int ena_rx_queue_intr_disable(struct rte_eth_dev *dev,
3537                                      uint16_t queue_id)
3538 {
3539         ena_rx_queue_intr_set(dev, queue_id, false);
3540
3541         return 0;
3542 }
3543
3544 static int ena_configure_aenq(struct ena_adapter *adapter)
3545 {
3546         uint32_t aenq_groups = adapter->all_aenq_groups;
3547         int rc;
3548
3549         /* All_aenq_groups holds all AENQ functions supported by the device and
3550          * the HW, so at first we need to be sure the LSC request is valid.
3551          */
3552         if (adapter->edev_data->dev_conf.intr_conf.lsc != 0) {
3553                 if (!(aenq_groups & BIT(ENA_ADMIN_LINK_CHANGE))) {
3554                         PMD_DRV_LOG(ERR,
3555                                 "LSC requested, but it's not supported by the AENQ\n");
3556                         return -EINVAL;
3557                 }
3558         } else {
3559                 /* If LSC wasn't enabled by the app, let's enable all supported
3560                  * AENQ procedures except the LSC.
3561                  */
3562                 aenq_groups &= ~BIT(ENA_ADMIN_LINK_CHANGE);
3563         }
3564
3565         rc = ena_com_set_aenq_config(&adapter->ena_dev, aenq_groups);
3566         if (rc != 0) {
3567                 PMD_DRV_LOG(ERR, "Cannot configure AENQ groups, rc=%d\n", rc);
3568                 return rc;
3569         }
3570
3571         adapter->active_aenq_groups = aenq_groups;
3572
3573         return 0;
3574 }
3575
3576 int ena_mp_indirect_table_set(struct ena_adapter *adapter)
3577 {
3578         return ENA_PROXY(adapter, ena_com_indirect_table_set, &adapter->ena_dev);
3579 }
3580
3581 int ena_mp_indirect_table_get(struct ena_adapter *adapter,
3582                               uint32_t *indirect_table)
3583 {
3584         return ENA_PROXY(adapter, ena_com_indirect_table_get, &adapter->ena_dev,
3585                 indirect_table);
3586 }
3587
3588 /*********************************************************************
3589  *  ena_plat_dpdk.h functions implementations
3590  *********************************************************************/
3591
3592 const struct rte_memzone *
3593 ena_mem_alloc_coherent(struct rte_eth_dev_data *data, size_t size,
3594                        int socket_id, unsigned int alignment, void **virt_addr,
3595                        dma_addr_t *phys_addr)
3596 {
3597         char z_name[RTE_MEMZONE_NAMESIZE];
3598         struct ena_adapter *adapter = data->dev_private;
3599         const struct rte_memzone *memzone;
3600         int rc;
3601
3602         rc = snprintf(z_name, RTE_MEMZONE_NAMESIZE, "ena_p%d_mz%" PRIu64 "",
3603                 data->port_id, adapter->memzone_cnt);
3604         if (rc >= RTE_MEMZONE_NAMESIZE) {
3605                 PMD_DRV_LOG(ERR,
3606                         "Name for the ena_com memzone is too long. Port: %d, mz_num: %" PRIu64 "\n",
3607                         data->port_id, adapter->memzone_cnt);
3608                 goto error;
3609         }
3610         adapter->memzone_cnt++;
3611
3612         memzone = rte_memzone_reserve_aligned(z_name, size, socket_id,
3613                 RTE_MEMZONE_IOVA_CONTIG, alignment);
3614         if (memzone == NULL) {
3615                 PMD_DRV_LOG(ERR, "Failed to allocate ena_com memzone: %s\n",
3616                         z_name);
3617                 goto error;
3618         }
3619
3620         memset(memzone->addr, 0, size);
3621         *virt_addr = memzone->addr;
3622         *phys_addr = memzone->iova;
3623
3624         return memzone;
3625
3626 error:
3627         *virt_addr = NULL;
3628         *phys_addr = 0;
3629
3630         return NULL;
3631 }
3632
3633
3634 /*********************************************************************
3635  *  PMD configuration
3636  *********************************************************************/
3637 static int eth_ena_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3638         struct rte_pci_device *pci_dev)
3639 {
3640         return rte_eth_dev_pci_generic_probe(pci_dev,
3641                 sizeof(struct ena_adapter), eth_ena_dev_init);
3642 }
3643
3644 static int eth_ena_pci_remove(struct rte_pci_device *pci_dev)
3645 {
3646         return rte_eth_dev_pci_generic_remove(pci_dev, eth_ena_dev_uninit);
3647 }
3648
3649 static struct rte_pci_driver rte_ena_pmd = {
3650         .id_table = pci_id_ena_map,
3651         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
3652                      RTE_PCI_DRV_WC_ACTIVATE,
3653         .probe = eth_ena_pci_probe,
3654         .remove = eth_ena_pci_remove,
3655 };
3656
3657 RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd);
3658 RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map);
3659 RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci");
3660 RTE_PMD_REGISTER_PARAM_STRING(net_ena, ENA_DEVARG_LARGE_LLQ_HDR "=<0|1>");
3661 RTE_LOG_REGISTER_SUFFIX(ena_logtype_init, init, NOTICE);
3662 RTE_LOG_REGISTER_SUFFIX(ena_logtype_driver, driver, NOTICE);
3663 #ifdef RTE_ETHDEV_DEBUG_RX
3664 RTE_LOG_REGISTER_SUFFIX(ena_logtype_rx, rx, DEBUG);
3665 #endif
3666 #ifdef RTE_ETHDEV_DEBUG_TX
3667 RTE_LOG_REGISTER_SUFFIX(ena_logtype_tx, tx, DEBUG);
3668 #endif
3669 RTE_LOG_REGISTER_SUFFIX(ena_logtype_com, com, WARNING);
3670
3671 /******************************************************************************
3672  ******************************** AENQ Handlers *******************************
3673  *****************************************************************************/
3674 static void ena_update_on_link_change(void *adapter_data,
3675                                       struct ena_admin_aenq_entry *aenq_e)
3676 {
3677         struct rte_eth_dev *eth_dev = adapter_data;
3678         struct ena_adapter *adapter = eth_dev->data->dev_private;
3679         struct ena_admin_aenq_link_change_desc *aenq_link_desc;
3680         uint32_t status;
3681
3682         aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e;
3683
3684         status = get_ena_admin_aenq_link_change_desc_link_status(aenq_link_desc);
3685         adapter->link_status = status;
3686
3687         ena_link_update(eth_dev, 0);
3688         rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL);
3689 }
3690
3691 static void ena_notification(void *adapter_data,
3692                              struct ena_admin_aenq_entry *aenq_e)
3693 {
3694         struct rte_eth_dev *eth_dev = adapter_data;
3695         struct ena_adapter *adapter = eth_dev->data->dev_private;
3696         struct ena_admin_ena_hw_hints *hints;
3697
3698         if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION)
3699                 PMD_DRV_LOG(WARNING, "Invalid AENQ group: %x. Expected: %x\n",
3700                         aenq_e->aenq_common_desc.group,
3701                         ENA_ADMIN_NOTIFICATION);
3702
3703         switch (aenq_e->aenq_common_desc.syndrome) {
3704         case ENA_ADMIN_UPDATE_HINTS:
3705                 hints = (struct ena_admin_ena_hw_hints *)
3706                         (&aenq_e->inline_data_w4);
3707                 ena_update_hints(adapter, hints);
3708                 break;
3709         default:
3710                 PMD_DRV_LOG(ERR, "Invalid AENQ notification link state: %d\n",
3711                         aenq_e->aenq_common_desc.syndrome);
3712         }
3713 }
3714
3715 static void ena_keep_alive(void *adapter_data,
3716                            __rte_unused struct ena_admin_aenq_entry *aenq_e)
3717 {
3718         struct rte_eth_dev *eth_dev = adapter_data;
3719         struct ena_adapter *adapter = eth_dev->data->dev_private;
3720         struct ena_admin_aenq_keep_alive_desc *desc;
3721         uint64_t rx_drops;
3722         uint64_t tx_drops;
3723
3724         adapter->timestamp_wd = rte_get_timer_cycles();
3725
3726         desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
3727         rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low;
3728         tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low;
3729
3730         adapter->drv_stats->rx_drops = rx_drops;
3731         adapter->dev_stats.tx_drops = tx_drops;
3732 }
3733
3734 /**
3735  * This handler will called for unknown event group or unimplemented handlers
3736  **/
3737 static void unimplemented_aenq_handler(__rte_unused void *data,
3738                                        __rte_unused struct ena_admin_aenq_entry *aenq_e)
3739 {
3740         PMD_DRV_LOG(ERR,
3741                 "Unknown event was received or event with unimplemented handler\n");
3742 }
3743
3744 static struct ena_aenq_handlers aenq_handlers = {
3745         .handlers = {
3746                 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
3747                 [ENA_ADMIN_NOTIFICATION] = ena_notification,
3748                 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive
3749         },
3750         .unimplemented_handler = unimplemented_aenq_handler
3751 };
3752
3753 /*********************************************************************
3754  *  Multi-Process communication request handling (in primary)
3755  *********************************************************************/
3756 static int
3757 ena_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer)
3758 {
3759         const struct ena_mp_body *req =
3760                 (const struct ena_mp_body *)mp_msg->param;
3761         struct ena_adapter *adapter;
3762         struct ena_com_dev *ena_dev;
3763         struct ena_mp_body *rsp;
3764         struct rte_mp_msg mp_rsp;
3765         struct rte_eth_dev *dev;
3766         int res = 0;
3767
3768         rsp = (struct ena_mp_body *)&mp_rsp.param;
3769         mp_msg_init(&mp_rsp, req->type, req->port_id);
3770
3771         if (!rte_eth_dev_is_valid_port(req->port_id)) {
3772                 rte_errno = ENODEV;
3773                 res = -rte_errno;
3774                 PMD_DRV_LOG(ERR, "Unknown port %d in request %d\n",
3775                             req->port_id, req->type);
3776                 goto end;
3777         }
3778         dev = &rte_eth_devices[req->port_id];
3779         adapter = dev->data->dev_private;
3780         ena_dev = &adapter->ena_dev;
3781
3782         switch (req->type) {
3783         case ENA_MP_DEV_STATS_GET:
3784                 res = ena_com_get_dev_basic_stats(ena_dev,
3785                                                   &adapter->basic_stats);
3786                 break;
3787         case ENA_MP_ENI_STATS_GET:
3788                 res = ena_com_get_eni_stats(ena_dev,
3789                         (struct ena_admin_eni_stats *)&adapter->eni_stats);
3790                 break;
3791         case ENA_MP_MTU_SET:
3792                 res = ena_com_set_dev_mtu(ena_dev, req->args.mtu);
3793                 break;
3794         case ENA_MP_IND_TBL_GET:
3795                 res = ena_com_indirect_table_get(ena_dev,
3796                                                  adapter->indirect_table);
3797                 break;
3798         case ENA_MP_IND_TBL_SET:
3799                 res = ena_com_indirect_table_set(ena_dev);
3800                 break;
3801         default:
3802                 PMD_DRV_LOG(ERR, "Unknown request type %d\n", req->type);
3803                 res = -EINVAL;
3804                 break;
3805         }
3806
3807 end:
3808         /* Save processing result in the reply */
3809         rsp->result = res;
3810         /* Return just IPC processing status */
3811         return rte_mp_reply(&mp_rsp, peer);
3812 }