X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4.c;h=7caa5e44ef2e6e220400f63d599aead2ad06d6d8;hb=ada5b88f6337bea0ab1839102ecfa7cd370c49cf;hp=ce518cf21bd13d526726f9d4b43e465bff00632b;hpb=d86046f0cf50fb3fdd48ba9f6730c7c9fc2ad831;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index ce518cf21b..7caa5e44ef 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -1,8 +1,8 @@ /*- * BSD LICENSE * - * Copyright 2012-2015 6WIND S.A. - * Copyright 2012 Mellanox. + * Copyright 2012-2017 6WIND S.A. + * Copyright 2012-2017 Mellanox. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -54,25 +54,10 @@ #include #include #include -#include #include #include #include -/* Verbs header. */ -/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" -#endif -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" -#endif - -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" -#endif #include #include #include @@ -87,30 +72,14 @@ #include #include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" -#endif +#include /* Generated configuration header. */ #include "mlx4_autoconf.h" -/* PMD header. */ +/* PMD headers. */ #include "mlx4.h" - -/* Runtime logging through RTE_LOG() is enabled when not in debugging mode. - * Intermediate LOG_*() macros add the required end-of-line characters. */ -#ifndef NDEBUG -#define INFO(...) DEBUG(__VA_ARGS__) -#define WARN(...) DEBUG(__VA_ARGS__) -#define ERROR(...) DEBUG(__VA_ARGS__) -#else -#define LOG__(level, m, ...) \ - RTE_LOG(level, PMD, MLX4_DRIVER_NAME ": " m "%c", __VA_ARGS__) -#define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n') -#define INFO(...) LOG_(INFO, __VA_ARGS__) -#define WARN(...) LOG_(WARNING, __VA_ARGS__) -#define ERROR(...) LOG_(ERR, __VA_ARGS__) -#endif +#include "mlx4_flow.h" /* Convenience macros for accessing mbuf fields. */ #define NEXT(m) ((m)->next) @@ -138,158 +107,6 @@ typedef union { (((val) & (from)) / ((from) / (to))) : \ (((val) & (from)) * ((to) / (from)))) -struct mlx4_rxq_stats { - unsigned int idx; /**< Mapping index. */ -#ifdef MLX4_PMD_SOFT_COUNTERS - uint64_t ipackets; /**< Total of successfully received packets. */ - uint64_t ibytes; /**< Total of successfully received bytes. */ -#endif - uint64_t idropped; /**< Total of packets dropped when RX ring full. */ - uint64_t rx_nombuf; /**< Total of RX mbuf allocation failures. */ -}; - -struct mlx4_txq_stats { - unsigned int idx; /**< Mapping index. */ -#ifdef MLX4_PMD_SOFT_COUNTERS - uint64_t opackets; /**< Total of successfully sent packets. */ - uint64_t obytes; /**< Total of successfully sent bytes. */ -#endif - uint64_t odropped; /**< Total of packets not sent when TX ring full. */ -}; - -/* RX element (scattered packets). */ -struct rxq_elt_sp { - struct ibv_recv_wr wr; /* Work Request. */ - struct ibv_sge sges[MLX4_PMD_SGE_WR_N]; /* Scatter/Gather Elements. */ - struct rte_mbuf *bufs[MLX4_PMD_SGE_WR_N]; /* SGEs buffers. */ -}; - -/* RX element. */ -struct rxq_elt { - struct ibv_recv_wr wr; /* Work Request. */ - struct ibv_sge sge; /* Scatter/Gather Element. */ - /* mbuf pointer is derived from WR_ID(wr.wr_id).offset. */ -}; - -/* RX queue descriptor. */ -struct rxq { - struct priv *priv; /* Back pointer to private data. */ - struct rte_mempool *mp; /* Memory Pool for allocations. */ - struct ibv_mr *mr; /* Memory Region (for mp). */ - struct ibv_cq *cq; /* Completion Queue. */ - struct ibv_qp *qp; /* Queue Pair. */ - struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */ - struct ibv_exp_cq_family *if_cq; /* CQ interface. */ - /* - * Each VLAN ID requires a separate flow steering rule. - */ - BITFIELD_DECLARE(mac_configured, uint32_t, MLX4_MAX_MAC_ADDRESSES); - struct ibv_flow *mac_flow[MLX4_MAX_MAC_ADDRESSES][MLX4_MAX_VLAN_IDS]; - struct ibv_flow *promisc_flow; /* Promiscuous flow. */ - struct ibv_flow *allmulti_flow; /* Multicast flow. */ - unsigned int port_id; /* Port ID for incoming packets. */ - unsigned int elts_n; /* (*elts)[] length. */ - unsigned int elts_head; /* Current index in (*elts)[]. */ - union { - struct rxq_elt_sp (*sp)[]; /* Scattered RX elements. */ - struct rxq_elt (*no_sp)[]; /* RX elements. */ - } elts; - unsigned int sp:1; /* Use scattered RX elements. */ - unsigned int csum:1; /* Enable checksum offloading. */ - unsigned int csum_l2tun:1; /* Same for L2 tunnels. */ - uint32_t mb_len; /* Length of a mp-issued mbuf. */ - struct mlx4_rxq_stats stats; /* RX queue counters. */ - unsigned int socket; /* CPU socket ID for allocations. */ - struct ibv_exp_res_domain *rd; /* Resource Domain. */ -}; - -/* TX element. */ -struct txq_elt { - struct rte_mbuf *buf; -}; - -/* Linear buffer type. It is used when transmitting buffers with too many - * segments that do not fit the hardware queue (see max_send_sge). - * Extra segments are copied (linearized) in such buffers, replacing the - * last SGE during TX. - * The size is arbitrary but large enough to hold a jumbo frame with - * 8 segments considering mbuf.buf_len is about 2048 bytes. */ -typedef uint8_t linear_t[16384]; - -/* TX queue descriptor. */ -struct txq { - struct priv *priv; /* Back pointer to private data. */ - struct { - const struct rte_mempool *mp; /* Cached Memory Pool. */ - struct ibv_mr *mr; /* Memory Region (for mp). */ - uint32_t lkey; /* mr->lkey */ - } mp2mr[MLX4_PMD_TX_MP_CACHE]; /* MP to MR translation table. */ - struct ibv_cq *cq; /* Completion Queue. */ - struct ibv_qp *qp; /* Queue Pair. */ - struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */ - struct ibv_exp_cq_family *if_cq; /* CQ interface. */ -#if MLX4_PMD_MAX_INLINE > 0 - uint32_t max_inline; /* Max inline send size <= MLX4_PMD_MAX_INLINE. */ -#endif - unsigned int elts_n; /* (*elts)[] length. */ - struct txq_elt (*elts)[]; /* TX elements. */ - unsigned int elts_head; /* Current index in (*elts)[]. */ - unsigned int elts_tail; /* First element awaiting completion. */ - unsigned int elts_comp; /* Number of completion requests. */ - unsigned int elts_comp_cd; /* Countdown for next completion request. */ - unsigned int elts_comp_cd_init; /* Initial value for countdown. */ - struct mlx4_txq_stats stats; /* TX queue counters. */ - linear_t (*elts_linear)[]; /* Linearized buffers. */ - struct ibv_mr *mr_linear; /* Memory Region for linearized buffers. */ - unsigned int socket; /* CPU socket ID for allocations. */ - struct ibv_exp_res_domain *rd; /* Resource Domain. */ -}; - -struct priv { - struct rte_eth_dev *dev; /* Ethernet device. */ - struct ibv_context *ctx; /* Verbs context. */ - struct ibv_device_attr device_attr; /* Device properties. */ - struct ibv_pd *pd; /* Protection Domain. */ - /* - * MAC addresses array and configuration bit-field. - * An extra entry that cannot be modified by the DPDK is reserved - * for broadcast frames (destination MAC address ff:ff:ff:ff:ff:ff). - */ - struct ether_addr mac[MLX4_MAX_MAC_ADDRESSES]; - BITFIELD_DECLARE(mac_configured, uint32_t, MLX4_MAX_MAC_ADDRESSES); - /* VLAN filters. */ - struct { - unsigned int enabled:1; /* If enabled. */ - unsigned int id:12; /* VLAN ID (0-4095). */ - } vlan_filter[MLX4_MAX_VLAN_IDS]; /* VLAN filters table. */ - /* Device properties. */ - uint16_t mtu; /* Configured MTU. */ - uint8_t port; /* Physical port number. */ - unsigned int started:1; /* Device started, flows enabled. */ - unsigned int promisc:1; /* Device in promiscuous mode. */ - unsigned int allmulti:1; /* Device receives all multicast packets. */ - unsigned int hw_qpg:1; /* QP groups are supported. */ - unsigned int hw_tss:1; /* TSS is supported. */ - unsigned int hw_rss:1; /* RSS is supported. */ - unsigned int hw_csum:1; /* Checksum offload is supported. */ - unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */ - unsigned int rss:1; /* RSS is enabled. */ - unsigned int vf:1; /* This is a VF device. */ - unsigned int pending_alarm:1; /* An alarm is pending. */ -#ifdef INLINE_RECV - unsigned int inl_recv_size; /* Inline recv size */ -#endif - unsigned int max_rss_tbl_sz; /* Maximum number of RSS queues. */ - /* RX/TX queues. */ - struct rxq rxq_parent; /* Parent queue when RSS is enabled. */ - unsigned int rxqs_n; /* RX queues array size. */ - unsigned int txqs_n; /* TX queues array size. */ - struct rxq *(*rxqs)[]; /* RX queues. */ - struct txq *(*txqs)[]; /* TX queues. */ - struct rte_intr_handle intr_handle; /* Interrupt handler. */ - rte_spinlock_t lock; /* Lock for control functions. */ -}; - /* Local storage for secondary process data. */ struct mlx4_secondary_data { struct rte_eth_dev_data data; /* Local device data. */ @@ -337,8 +154,7 @@ mlx4_get_priv(struct rte_eth_dev *dev) * @param priv * Pointer to private structure. */ -static void -priv_lock(struct priv *priv) +void priv_lock(struct priv *priv) { rte_spinlock_lock(&priv->lock); } @@ -349,8 +165,7 @@ priv_lock(struct priv *priv) * @param priv * Pointer to private structure. */ -static void -priv_unlock(struct priv *priv) +void priv_unlock(struct priv *priv) { rte_spinlock_unlock(&priv->lock); } @@ -659,7 +474,15 @@ priv_get_mtu(struct priv *priv, uint16_t *mtu) static int priv_set_mtu(struct priv *priv, uint16_t mtu) { - return priv_set_sysfs_ulong(priv, "mtu", mtu); + uint16_t new_mtu; + + if (priv_set_sysfs_ulong(priv, "mtu", mtu) || + priv_get_mtu(priv, &new_mtu)) + return -1; + if (new_mtu == mtu) + return 0; + errno = EINVAL; + return -1; } /** @@ -683,7 +506,7 @@ priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) return -1; tmp &= keep; - tmp |= flags; + tmp |= (flags & (~keep)); return priv_set_sysfs_ulong(priv, "flags", tmp); } @@ -998,7 +821,7 @@ txq_alloc_elts(struct txq *txq, unsigned int elts_n) } mr_linear = ibv_reg_mr(txq->priv->pd, elts_linear, sizeof(*elts_linear), - (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE)); + IBV_ACCESS_LOCAL_WRITE); if (mr_linear == NULL) { ERROR("%p: unable to configure MR, ibv_reg_mr() failed", (void *)txq); @@ -1198,8 +1021,71 @@ txq_complete(struct txq *txq) return 0; } +struct mlx4_check_mempool_data { + int ret; + char *start; + char *end; +}; + +/* Called by mlx4_check_mempool() when iterating the memory chunks. */ +static void mlx4_check_mempool_cb(struct rte_mempool *mp, + void *opaque, struct rte_mempool_memhdr *memhdr, + unsigned mem_idx) +{ + struct mlx4_check_mempool_data *data = opaque; + + (void)mp; + (void)mem_idx; + + /* It already failed, skip the next chunks. */ + if (data->ret != 0) + return; + /* It is the first chunk. */ + if (data->start == NULL && data->end == NULL) { + data->start = memhdr->addr; + data->end = data->start + memhdr->len; + return; + } + if (data->end == memhdr->addr) { + data->end += memhdr->len; + return; + } + if (data->start == (char *)memhdr->addr + memhdr->len) { + data->start -= memhdr->len; + return; + } + /* Error, mempool is not virtually contigous. */ + data->ret = -1; +} + +/** + * Check if a mempool can be used: it must be virtually contiguous. + * + * @param[in] mp + * Pointer to memory pool. + * @param[out] start + * Pointer to the start address of the mempool virtual memory area + * @param[out] end + * Pointer to the end address of the mempool virtual memory area + * + * @return + * 0 on success (mempool is virtually contiguous), -1 on error. + */ +static int mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, + uintptr_t *end) +{ + struct mlx4_check_mempool_data data; + + memset(&data, 0, sizeof(data)); + rte_mempool_mem_iter(mp, mlx4_check_mempool_cb, &data); + *start = (uintptr_t)data.start; + *end = (uintptr_t)data.end; + + return data.ret; +} + /* For best performance, this function should not be inlined. */ -static struct ibv_mr *mlx4_mp2mr(struct ibv_pd *, const struct rte_mempool *) +static struct ibv_mr *mlx4_mp2mr(struct ibv_pd *, struct rte_mempool *) __attribute__((noinline)); /** @@ -1214,15 +1100,21 @@ static struct ibv_mr *mlx4_mp2mr(struct ibv_pd *, const struct rte_mempool *) * Memory region pointer, NULL in case of error. */ static struct ibv_mr * -mlx4_mp2mr(struct ibv_pd *pd, const struct rte_mempool *mp) +mlx4_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp) { const struct rte_memseg *ms = rte_eal_get_physmem_layout(); - uintptr_t start = mp->elt_va_start; - uintptr_t end = mp->elt_va_end; + uintptr_t start; + uintptr_t end; unsigned int i; + if (mlx4_check_mempool(mp, &start, &end) != 0) { + ERROR("mempool %p: not virtually contiguous", + (void *)mp); + return NULL; + } + DEBUG("mempool %p area start=%p end=%p size=%zu", - (const void *)mp, (void *)start, (void *)end, + (void *)mp, (void *)start, (void *)end, (size_t)(end - start)); /* Round start and end to page boundary if found in memory segments. */ for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) { @@ -1236,12 +1128,12 @@ mlx4_mp2mr(struct ibv_pd *pd, const struct rte_mempool *mp) end = RTE_ALIGN_CEIL(end, align); } DEBUG("mempool %p using start=%p end=%p size=%zu for MR", - (const void *)mp, (void *)start, (void *)end, + (void *)mp, (void *)start, (void *)end, (size_t)(end - start)); return ibv_reg_mr(pd, (void *)start, end - start, - IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + IBV_ACCESS_LOCAL_WRITE); } /** @@ -1276,7 +1168,7 @@ txq_mb2mp(struct rte_mbuf *buf) * mr->lkey on success, (uint32_t)-1 on failure. */ static uint32_t -txq_mp2mr(struct txq *txq, const struct rte_mempool *mp) +txq_mp2mr(struct txq *txq, struct rte_mempool *mp) { unsigned int i; struct ibv_mr *mr; @@ -1294,7 +1186,7 @@ txq_mp2mr(struct txq *txq, const struct rte_mempool *mp) } /* Add a new entry, register MR first. */ DEBUG("%p: discovered new memory pool \"%s\" (%p)", - (void *)txq, mp->name, (const void *)mp); + (void *)txq, mp->name, (void *)mp); mr = mlx4_mp2mr(txq->priv->pd, mp); if (unlikely(mr == NULL)) { DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.", @@ -1315,7 +1207,7 @@ txq_mp2mr(struct txq *txq, const struct rte_mempool *mp) txq->mp2mr[i].mr = mr; txq->mp2mr[i].lkey = mr->lkey; DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, - (void *)txq, mp->name, (const void *)mp, txq->mp2mr[i].lkey); + (void *)txq, mp->name, (void *)mp, txq->mp2mr[i].lkey); return txq->mp2mr[i].lkey; } @@ -2451,6 +2343,7 @@ rxq_add_flow(struct rxq *rxq, unsigned int mac_index, unsigned int vlan_index) assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec); *attr = (struct ibv_flow_attr){ .type = IBV_FLOW_ATTR_NORMAL, + .priority = 3, .num_of_specs = 1, .port = priv->port, .flags = 0 @@ -2886,19 +2779,25 @@ rxq_cq_to_pkt_type(uint32_t flags) if (flags & IBV_EXP_CQ_RX_TUNNEL_PACKET) pkt_type = TRANSPOSE(flags, - IBV_EXP_CQ_RX_OUTER_IPV4_PACKET, RTE_PTYPE_L3_IPV4) | + IBV_EXP_CQ_RX_OUTER_IPV4_PACKET, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_OUTER_IPV6_PACKET, RTE_PTYPE_L3_IPV6) | + IBV_EXP_CQ_RX_OUTER_IPV6_PACKET, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV4_PACKET, RTE_PTYPE_INNER_L3_IPV4) | + IBV_EXP_CQ_RX_IPV4_PACKET, + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV6_PACKET, RTE_PTYPE_INNER_L3_IPV6); + IBV_EXP_CQ_RX_IPV6_PACKET, + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN); else pkt_type = TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV4_PACKET, RTE_PTYPE_L3_IPV4) | + IBV_EXP_CQ_RX_IPV4_PACKET, + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN) | TRANSPOSE(flags, - IBV_EXP_CQ_RX_IPV6_PACKET, RTE_PTYPE_L3_IPV6); + IBV_EXP_CQ_RX_IPV6_PACKET, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN); return pkt_type; } @@ -2920,25 +2819,20 @@ rxq_cq_to_ol_flags(const struct rxq *rxq, uint32_t flags) if (rxq->csum) ol_flags |= - TRANSPOSE(~flags, + TRANSPOSE(flags, IBV_EXP_CQ_RX_IP_CSUM_OK, - PKT_RX_IP_CKSUM_BAD) | - TRANSPOSE(~flags, + PKT_RX_IP_CKSUM_GOOD) | + TRANSPOSE(flags, IBV_EXP_CQ_RX_TCP_UDP_CSUM_OK, - PKT_RX_L4_CKSUM_BAD); - /* - * PKT_RX_IP_CKSUM_BAD and PKT_RX_L4_CKSUM_BAD are used in place - * of PKT_RX_EIP_CKSUM_BAD because the latter is not functional - * (its value is 0). - */ + PKT_RX_L4_CKSUM_GOOD); if ((flags & IBV_EXP_CQ_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) ol_flags |= - TRANSPOSE(~flags, + TRANSPOSE(flags, IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK, - PKT_RX_IP_CKSUM_BAD) | - TRANSPOSE(~flags, + PKT_RX_IP_CKSUM_GOOD) | + TRANSPOSE(flags, IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK, - PKT_RX_L4_CKSUM_BAD); + PKT_RX_L4_CKSUM_GOOD); return ol_flags; } @@ -3083,7 +2977,6 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) rep->ol_flags = -1; #endif assert(rep->buf_len == seg->buf_len); - assert(rep->buf_len == rxq->mb_len); /* Reconfigure sge to use rep instead of seg. */ assert(sge->lkey == rxq->mr->lkey); sge->addr = ((uintptr_t)rep->buf_addr + seg_headroom); @@ -3214,8 +3107,8 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) * Fetch initial bytes of packet descriptor into a * cacheline while allocating rep. */ - rte_prefetch0(seg); - rte_prefetch0(&seg->cacheline1); + rte_mbuf_prefetch_part1(seg); + rte_mbuf_prefetch_part2(seg); ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL, &flags); if (unlikely(ret < 0)) { @@ -3504,6 +3397,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) unsigned int i, k; struct ibv_exp_qp_attr mod; struct ibv_recv_wr *bad_wr; + unsigned int mb_len; int err; int parent = (rxq == &priv->rxq_parent); @@ -3512,6 +3406,7 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) (void *)dev, (void *)rxq); return EINVAL; } + mb_len = rte_pktmbuf_data_room_size(rxq->mp); DEBUG("%p: rehashing queue %p", (void *)dev, (void *)rxq); /* Number of descriptors and mbufs currently allocated. */ desc_n = (tmpl.elts_n * (tmpl.sp ? MLX4_PMD_SGE_WR_N : 1)); @@ -3526,9 +3421,10 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) rxq->csum_l2tun = tmpl.csum_l2tun; } /* Enable scattered packets support for this queue if necessary. */ + assert(mb_len >= RTE_PKTMBUF_HEADROOM); if ((dev->data->dev_conf.rxmode.jumbo_frame) && (dev->data->dev_conf.rxmode.max_rx_pkt_len > - (tmpl.mb_len - RTE_PKTMBUF_HEADROOM))) { + (mb_len - RTE_PKTMBUF_HEADROOM))) { tmpl.sp = 1; desc_n /= MLX4_PMD_SGE_WR_N; } else @@ -3719,7 +3615,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, } attr; enum ibv_exp_query_intf_status status; struct ibv_recv_wr *bad_wr; - struct rte_mbuf *buf; + unsigned int mb_len; int ret = 0; int parent = (rxq == &priv->rxq_parent); @@ -3735,31 +3631,22 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, desc = 1; goto skip_mr; } + mb_len = rte_pktmbuf_data_room_size(mp); if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) { ERROR("%p: invalid number of RX descriptors (must be a" " multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N); return EINVAL; } - /* Get mbuf length. */ - buf = rte_pktmbuf_alloc(mp); - if (buf == NULL) { - ERROR("%p: unable to allocate mbuf", (void *)dev); - return ENOMEM; - } - tmpl.mb_len = buf->buf_len; - assert((rte_pktmbuf_headroom(buf) + - rte_pktmbuf_tailroom(buf)) == tmpl.mb_len); - assert(rte_pktmbuf_headroom(buf) == RTE_PKTMBUF_HEADROOM); - rte_pktmbuf_free(buf); /* Toggle RX checksum offload if hardware supports it. */ if (priv->hw_csum) tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; if (priv->hw_csum_l2tun) tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum; /* Enable scattered packets support for this queue if necessary. */ + assert(mb_len >= RTE_PKTMBUF_HEADROOM); if ((dev->data->dev_conf.rxmode.jumbo_frame) && (dev->data->dev_conf.rxmode.max_rx_pkt_len > - (tmpl.mb_len - RTE_PKTMBUF_HEADROOM))) { + (mb_len - RTE_PKTMBUF_HEADROOM))) { tmpl.sp = 1; desc /= MLX4_PMD_SGE_WR_N; } @@ -4044,6 +3931,7 @@ mlx4_dev_start(struct rte_eth_dev *dev) unsigned int i = 0; unsigned int r; struct rxq *rxq; + int ret; if (mlx4_is_secondary()) return -E_RTE_SECONDARY; @@ -4063,8 +3951,6 @@ mlx4_dev_start(struct rte_eth_dev *dev) } /* Iterate only once when RSS is enabled. */ do { - int ret; - /* Ignore nonexistent RX queues. */ if (rxq == NULL) continue; @@ -4077,22 +3963,30 @@ mlx4_dev_start(struct rte_eth_dev *dev) continue; WARN("%p: QP flow attachment failed: %s", (void *)dev, strerror(ret)); - /* Rollback. */ - while (i != 0) { - rxq = (*priv->rxqs)[--i]; - if (rxq != NULL) { - rxq_allmulticast_disable(rxq); - rxq_promiscuous_disable(rxq); - rxq_mac_addrs_del(rxq); - } - } - priv->started = 0; - priv_unlock(priv); - return -ret; + goto err; } while ((--r) && ((rxq = (*priv->rxqs)[++i]), i)); priv_dev_interrupt_handler_install(priv, dev); + ret = mlx4_priv_flow_start(priv); + if (ret) { + ERROR("%p: flow start failed: %s", + (void *)dev, strerror(ret)); + goto err; + } priv_unlock(priv); return 0; +err: + /* Rollback. */ + while (i != 0) { + rxq = (*priv->rxqs)[i--]; + if (rxq != NULL) { + rxq_allmulticast_disable(rxq); + rxq_promiscuous_disable(rxq); + rxq_mac_addrs_del(rxq); + } + } + priv->started = 0; + priv_unlock(priv); + return -ret; } /** @@ -4127,6 +4021,7 @@ mlx4_dev_stop(struct rte_eth_dev *dev) rxq = (*priv->rxqs)[0]; r = priv->rxqs_n; } + mlx4_priv_flow_stop(priv); /* Iterate only once when RSS is enabled. */ do { /* Ignore nonexistent RX queues. */ @@ -4259,6 +4154,90 @@ mlx4_dev_close(struct rte_eth_dev *dev) memset(priv, 0, sizeof(*priv)); } +/** + * Change the link state (UP / DOWN). + * + * @param priv + * Pointer to Ethernet device private data. + * @param up + * Nonzero for link up, otherwise link down. + * + * @return + * 0 on success, errno value on failure. + */ +static int +priv_set_link(struct priv *priv, int up) +{ + struct rte_eth_dev *dev = priv->dev; + int err; + unsigned int i; + + if (up) { + err = priv_set_flags(priv, ~IFF_UP, IFF_UP); + if (err) + return err; + for (i = 0; i < priv->rxqs_n; i++) + if ((*priv->rxqs)[i]->sp) + break; + /* Check if an sp queue exists. + * Note: Some old frames might be received. + */ + if (i == priv->rxqs_n) + dev->rx_pkt_burst = mlx4_rx_burst; + else + dev->rx_pkt_burst = mlx4_rx_burst_sp; + dev->tx_pkt_burst = mlx4_tx_burst; + } else { + err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); + if (err) + return err; + dev->rx_pkt_burst = removed_rx_burst; + dev->tx_pkt_burst = removed_tx_burst; + } + return 0; +} + +/** + * DPDK callback to bring the link DOWN. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, errno value on failure. + */ +static int +mlx4_set_link_down(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + int err; + + priv_lock(priv); + err = priv_set_link(priv, 0); + priv_unlock(priv); + return err; +} + +/** + * DPDK callback to bring the link UP. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, errno value on failure. + */ +static int +mlx4_set_link_up(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + int err; + + priv_lock(priv); + err = priv_set_link(priv, 1); + priv_unlock(priv); + return err; +} /** * DPDK callback to get information about the device. * @@ -4274,6 +4253,8 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) unsigned int max; char ifname[IF_NAMESIZE]; + info->pci_dev = RTE_DEV_TO_PCI(dev->device); + if (priv == NULL) return; priv_lock(priv); @@ -4674,7 +4655,7 @@ end: } /** - * DPDK callback to retrieve physical link information (unlocked version). + * DPDK callback to retrieve physical link information. * * @param dev * Pointer to Ethernet device structure. @@ -4682,9 +4663,9 @@ end: * Wait for request completion (ignored). */ static int -mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) +mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = mlx4_get_priv(dev); + const struct priv *priv = mlx4_get_priv(dev); struct ethtool_cmd edata = { .cmd = ETHTOOL_GSET }; @@ -4692,6 +4673,8 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) struct rte_eth_link dev_link; int link_speed = 0; + /* priv_lock() is not taken to allow concurrent calls. */ + if (priv == NULL) return -EINVAL; (void)wait_to_complete; @@ -4702,7 +4685,7 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) memset(&dev_link, 0, sizeof(dev_link)); dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); - ifr.ifr_data = &edata; + ifr.ifr_data = (void *)&edata; if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", strerror(errno)); @@ -4726,28 +4709,6 @@ mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) return -1; } -/** - * DPDK callback to retrieve physical link information. - * - * @param dev - * Pointer to Ethernet device structure. - * @param wait_to_complete - * Wait for request completion (ignored). - */ -static int -mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete) -{ - struct priv *priv = mlx4_get_priv(dev); - int ret; - - if (priv == NULL) - return -EINVAL; - priv_lock(priv); - ret = mlx4_link_update_unlocked(dev, wait_to_complete); - priv_unlock(priv); - return ret; -} - /** * DPDK callback to change the MTU. * @@ -4796,6 +4757,7 @@ mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) /* Reconfigure each RX queue. */ for (i = 0; (i != priv->rxqs_n); ++i) { struct rxq *rxq = (*priv->rxqs)[i]; + unsigned int mb_len; unsigned int max_frame_len; int sp; @@ -4805,7 +4767,9 @@ mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) * toggle scattered support (sp) if necessary. */ max_frame_len = (priv->mtu + ETHER_HDR_LEN + (ETHER_MAX_VLAN_FRAME_LEN - ETHER_MAX_LEN)); - sp = (max_frame_len > (rxq->mb_len - RTE_PKTMBUF_HEADROOM)); + mb_len = rte_pktmbuf_data_room_size(rxq->mp); + assert(mb_len >= RTE_PKTMBUF_HEADROOM); + sp = (max_frame_len > (mb_len - RTE_PKTMBUF_HEADROOM)); /* Provide new values to rxq_setup(). */ dev->data->dev_conf.rxmode.jumbo_frame = sp; dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame_len; @@ -4861,7 +4825,7 @@ mlx4_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) if (mlx4_is_secondary()) return -E_RTE_SECONDARY; - ifr.ifr_data = ðpause; + ifr.ifr_data = (void *)ðpause; priv_lock(priv); if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { ret = errno; @@ -4911,7 +4875,7 @@ mlx4_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) if (mlx4_is_secondary()) return -E_RTE_SECONDARY; - ifr.ifr_data = ðpause; + ifr.ifr_data = (void *)ðpause; ethpause.autoneg = fc_conf->autoneg; if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || (fc_conf->mode & RTE_FC_RX_PAUSE)) @@ -5059,10 +5023,61 @@ mlx4_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) return -ret; } +const struct rte_flow_ops mlx4_flow_ops = { + .validate = mlx4_flow_validate, + .create = mlx4_flow_create, + .destroy = mlx4_flow_destroy, + .flush = mlx4_flow_flush, + .query = NULL, +}; + +/** + * Manage filter operations. + * + * @param dev + * Pointer to Ethernet device structure. + * @param filter_type + * Filter type. + * @param filter_op + * Operation to perform. + * @param arg + * Pointer to operation-specific structure. + * + * @return + * 0 on success, negative errno value on failure. + */ +static int +mlx4_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg) +{ + int ret = EINVAL; + + switch (filter_type) { + case RTE_ETH_FILTER_GENERIC: + if (filter_op != RTE_ETH_FILTER_GET) + return -EINVAL; + *(const void **)arg = &mlx4_flow_ops; + return 0; + case RTE_ETH_FILTER_FDIR: + DEBUG("%p: filter type FDIR is not supported by this PMD", + (void *)dev); + break; + default: + ERROR("%p: filter type (%d) not supported", + (void *)dev, filter_type); + break; + } + return -ret; +} + static const struct eth_dev_ops mlx4_dev_ops = { .dev_configure = mlx4_dev_configure, .dev_start = mlx4_dev_start, .dev_stop = mlx4_dev_stop, + .dev_set_link_down = mlx4_set_link_down, + .dev_set_link_up = mlx4_set_link_up, .dev_close = mlx4_dev_close, .promiscuous_enable = mlx4_promiscuous_enable, .promiscuous_disable = mlx4_promiscuous_disable, @@ -5091,6 +5106,7 @@ static const struct eth_dev_ops mlx4_dev_ops = { .mac_addr_add = mlx4_mac_addr_add, .mac_addr_set = mlx4_mac_addr_set, .mtu_set = mlx4_dev_set_mtu, + .filter_ctrl = mlx4_dev_filter_ctrl, }; /** @@ -5259,7 +5275,7 @@ priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev) struct rte_eth_link *link = &dev->data->dev_link; priv->pending_alarm = 0; - mlx4_link_update_unlocked(dev, 0); + mlx4_link_update(dev, 0); if (((link->link_speed == 0) && link->link_status) || ((link->link_speed != 0) && !link->link_status)) { /* Inconsistent status, check again later. */ @@ -5291,7 +5307,7 @@ mlx4_dev_link_status_handler(void *arg) ret = priv_dev_link_status_handler(priv, dev); priv_unlock(priv); if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); } /** @@ -5314,7 +5330,7 @@ mlx4_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg) ret = priv_dev_link_status_handler(priv, dev); priv_unlock(priv); if (ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); } /** @@ -5337,7 +5353,7 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) rte_eal_alarm_cancel(mlx4_dev_link_status_handler, dev); priv->pending_alarm = 0; priv->intr_handle.fd = 0; - priv->intr_handle.type = 0; + priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; } /** @@ -5387,7 +5403,7 @@ static struct eth_driver mlx4_driver; * 0 on success, negative errno value on failure. */ static int -mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) { struct ibv_device **list; struct ibv_device *ibv_dev; @@ -5646,7 +5662,7 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) snprintf(name, sizeof(name), "%s port %u", ibv_get_device_name(ibv_dev), port); - eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_PCI); + eth_dev = rte_eth_dev_allocate(name); } if (eth_dev == NULL) { ERROR("can not allocate rte ethdev"); @@ -5682,11 +5698,9 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) eth_dev->rx_pkt_burst = mlx4_rx_burst_secondary_setup; } else { eth_dev->data->dev_private = priv; - eth_dev->data->rx_mbuf_alloc_failed = 0; - eth_dev->data->mtu = ETHER_MTU; eth_dev->data->mac_addrs = priv->mac; } - eth_dev->pci_dev = pci_dev; + eth_dev->device = &pci_dev->device; rte_eth_copy_pci_info(eth_dev, pci_dev); @@ -5694,11 +5708,13 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) priv->dev = eth_dev; eth_dev->dev_ops = &mlx4_dev_ops; - TAILQ_INIT(ð_dev->link_intr_cbs); /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); priv_set_flags(priv, ~IFF_UP, IFF_UP); + /* Update link status once if waiting for LSC. */ + if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) + mlx4_link_update(eth_dev, 0); continue; port_error: @@ -5736,22 +5752,16 @@ error: static const struct rte_pci_id mlx4_pci_id_map[] = { { - .vendor_id = PCI_VENDOR_ID_MELLANOX, - .device_id = PCI_DEVICE_ID_MELLANOX_CONNECTX3, - .subsystem_vendor_id = PCI_ANY_ID, - .subsystem_device_id = PCI_ANY_ID + RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, + PCI_DEVICE_ID_MELLANOX_CONNECTX3) }, { - .vendor_id = PCI_VENDOR_ID_MELLANOX, - .device_id = PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO, - .subsystem_vendor_id = PCI_ANY_ID, - .subsystem_device_id = PCI_ANY_ID + RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, + PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO) }, { - .vendor_id = PCI_VENDOR_ID_MELLANOX, - .device_id = PCI_DEVICE_ID_MELLANOX_CONNECTX3VF, - .subsystem_vendor_id = PCI_ANY_ID, - .subsystem_device_id = PCI_ANY_ID + RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, + PCI_DEVICE_ID_MELLANOX_CONNECTX3VF) }, { .vendor_id = 0 @@ -5760,9 +5770,11 @@ static const struct rte_pci_id mlx4_pci_id_map[] = { static struct eth_driver mlx4_driver = { .pci_drv = { - .name = MLX4_DRIVER_NAME, + .driver = { + .name = MLX4_DRIVER_NAME + }, .id_table = mlx4_pci_id_map, - .devinit = mlx4_pci_devinit, + .probe = mlx4_pci_probe, .drv_flags = RTE_PCI_DRV_INTR_LSC, }, .dev_private_size = sizeof(struct priv) @@ -5771,12 +5783,10 @@ static struct eth_driver mlx4_driver = { /** * Driver initialization routine. */ -static int -rte_mlx4_pmd_init(const char *name, const char *args) +RTE_INIT(rte_mlx4_pmd_init); +static void +rte_mlx4_pmd_init(void) { - (void)name; - (void)args; - RTE_BUILD_BUG_ON(sizeof(wr_id_t) != sizeof(uint64_t)); /* * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use @@ -5787,13 +5797,9 @@ rte_mlx4_pmd_init(const char *name, const char *args) setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); ibv_fork_init(); rte_eal_pci_register(&mlx4_driver.pci_drv); - return 0; } -static struct rte_driver rte_mlx4_driver = { - .type = PMD_PDEV, - .name = MLX4_DRIVER_NAME, - .init = rte_mlx4_pmd_init, -}; - -PMD_REGISTER_DRIVER(rte_mlx4_driver) +RTE_PMD_EXPORT_NAME(net_mlx4, __COUNTER__); +RTE_PMD_REGISTER_PCI_TABLE(net_mlx4, mlx4_pci_id_map); +RTE_PMD_REGISTER_KMOD_DEP(net_mlx4, + "* ib_uverbs & mlx4_en & mlx4_core & mlx4_ib");