net/bnxt: allow dynamic creation of VNIC
[dpdk.git] / drivers / net / bnxt / bnxt_rxq.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2014-2018 Broadcom
3  * All rights reserved.
4  */
5
6 #include <inttypes.h>
7
8 #include <rte_malloc.h>
9
10 #include "bnxt.h"
11 #include "bnxt_cpr.h"
12 #include "bnxt_filter.h"
13 #include "bnxt_hwrm.h"
14 #include "bnxt_ring.h"
15 #include "bnxt_rxq.h"
16 #include "bnxt_rxr.h"
17 #include "bnxt_vnic.h"
18 #include "hsi_struct_def_dpdk.h"
19
20 /*
21  * RX Queues
22  */
23
24 void bnxt_free_rxq_stats(struct bnxt_rx_queue *rxq)
25 {
26         if (rxq && rxq->cp_ring && rxq->cp_ring->hw_stats)
27                 rxq->cp_ring->hw_stats = NULL;
28 }
29
30 int bnxt_mq_rx_configure(struct bnxt *bp)
31 {
32         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
33         const struct rte_eth_vmdq_rx_conf *conf =
34                     &dev_conf->rx_adv_conf.vmdq_rx_conf;
35         unsigned int i, j, nb_q_per_grp = 1, ring_idx = 0;
36         int start_grp_id, end_grp_id = 1, rc = 0;
37         struct bnxt_vnic_info *vnic;
38         struct bnxt_filter_info *filter;
39         enum rte_eth_nb_pools pools = bp->rx_cp_nr_rings, max_pools = 0;
40         struct bnxt_rx_queue *rxq;
41
42         bp->nr_vnics = 0;
43
44         /* Single queue mode */
45         if (bp->rx_cp_nr_rings < 2) {
46                 vnic = &bp->vnic_info[0];
47                 if (!vnic) {
48                         PMD_DRV_LOG(ERR, "VNIC alloc failed\n");
49                         rc = -ENOMEM;
50                         goto err_out;
51                 }
52                 vnic->flags |= BNXT_VNIC_INFO_BCAST;
53                 bp->nr_vnics++;
54
55                 rxq = bp->eth_dev->data->rx_queues[0];
56                 rxq->vnic = vnic;
57
58                 vnic->func_default = true;
59                 vnic->start_grp_id = 0;
60                 vnic->end_grp_id = vnic->start_grp_id;
61                 filter = bnxt_alloc_filter(bp);
62                 if (!filter) {
63                         PMD_DRV_LOG(ERR, "L2 filter alloc failed\n");
64                         rc = -ENOMEM;
65                         goto err_out;
66                 }
67                 STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
68                 goto out;
69         }
70
71         /* Multi-queue mode */
72         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB_RSS) {
73                 /* VMDq ONLY, VMDq+RSS, VMDq+DCB, VMDq+DCB+RSS */
74
75                 switch (dev_conf->rxmode.mq_mode) {
76                 case ETH_MQ_RX_VMDQ_RSS:
77                 case ETH_MQ_RX_VMDQ_ONLY:
78                         /* FALLTHROUGH */
79                         /* ETH_8/64_POOLs */
80                         pools = conf->nb_queue_pools;
81                         /* For each pool, allocate MACVLAN CFA rule & VNIC */
82                         max_pools = RTE_MIN(bp->max_vnics,
83                                             RTE_MIN(bp->max_l2_ctx,
84                                             RTE_MIN(bp->max_rsscos_ctx,
85                                                     ETH_64_POOLS)));
86                         PMD_DRV_LOG(DEBUG,
87                                     "pools = %u max_pools = %u\n",
88                                     pools, max_pools);
89                         if (pools > max_pools)
90                                 pools = max_pools;
91                         break;
92                 case ETH_MQ_RX_RSS:
93                         pools = 1;
94                         break;
95                 default:
96                         PMD_DRV_LOG(ERR, "Unsupported mq_mod %d\n",
97                                 dev_conf->rxmode.mq_mode);
98                         rc = -EINVAL;
99                         goto err_out;
100                 }
101         }
102         nb_q_per_grp = bp->rx_cp_nr_rings / pools;
103         bp->rx_num_qs_per_vnic = nb_q_per_grp;
104         PMD_DRV_LOG(DEBUG, "pools = %u nb_q_per_grp = %u\n",
105                     pools, nb_q_per_grp);
106         start_grp_id = 0;
107         end_grp_id = nb_q_per_grp;
108
109         for (i = 0; i < pools; i++) {
110                 vnic = &bp->vnic_info[i];
111                 if (!vnic) {
112                         PMD_DRV_LOG(ERR, "VNIC alloc failed\n");
113                         rc = -ENOMEM;
114                         goto err_out;
115                 }
116                 vnic->flags |= BNXT_VNIC_INFO_BCAST;
117                 bp->nr_vnics++;
118
119                 for (j = 0; j < nb_q_per_grp; j++, ring_idx++) {
120                         rxq = bp->eth_dev->data->rx_queues[ring_idx];
121                         rxq->vnic = vnic;
122                         PMD_DRV_LOG(DEBUG,
123                                     "rxq[%d] = %p vnic[%d] = %p\n",
124                                     ring_idx, rxq, i, vnic);
125                 }
126                 if (i == 0) {
127                         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB) {
128                                 bp->eth_dev->data->promiscuous = 1;
129                                 vnic->flags |= BNXT_VNIC_INFO_PROMISC;
130                         }
131                         vnic->func_default = true;
132                 }
133                 vnic->start_grp_id = start_grp_id;
134                 vnic->end_grp_id = end_grp_id;
135
136                 if (i) {
137                         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_VMDQ_DCB ||
138                             !(dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS))
139                                 vnic->rss_dflt_cr = true;
140                         goto skip_filter_allocation;
141                 }
142                 filter = bnxt_alloc_filter(bp);
143                 if (!filter) {
144                         PMD_DRV_LOG(ERR, "L2 filter alloc failed\n");
145                         rc = -ENOMEM;
146                         goto err_out;
147                 }
148                 /*
149                  * TODO: Configure & associate CFA rule for
150                  * each VNIC for each VMDq with MACVLAN, MACVLAN+TC
151                  */
152                 STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
153
154 skip_filter_allocation:
155                 start_grp_id = end_grp_id;
156                 end_grp_id += nb_q_per_grp;
157         }
158
159 out:
160         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
161                 struct rte_eth_rss_conf *rss = &dev_conf->rx_adv_conf.rss_conf;
162
163                 if (bp->flags & BNXT_FLAG_UPDATE_HASH) {
164                         rss = &bp->rss_conf;
165                         bp->flags &= ~BNXT_FLAG_UPDATE_HASH;
166                 }
167
168                 for (i = 0; i < bp->nr_vnics; i++) {
169                         vnic = &bp->vnic_info[i];
170                         vnic->hash_type =
171                                 bnxt_rte_to_hwrm_hash_types(rss->rss_hf);
172
173                         /*
174                          * Use the supplied key if the key length is
175                          * acceptable and the rss_key is not NULL
176                          */
177                         if (rss->rss_key &&
178                             rss->rss_key_len <= HW_HASH_KEY_SIZE)
179                                 memcpy(vnic->rss_hash_key,
180                                        rss->rss_key, rss->rss_key_len);
181                 }
182         }
183
184         return rc;
185
186 err_out:
187         /* Free allocated vnic/filters */
188
189         return rc;
190 }
191
192 void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq)
193 {
194         struct bnxt_sw_rx_bd *sw_ring;
195         struct bnxt_tpa_info *tpa_info;
196         uint16_t i;
197
198         if (!rxq)
199                 return;
200
201         rte_spinlock_lock(&rxq->lock);
202
203         sw_ring = rxq->rx_ring->rx_buf_ring;
204         if (sw_ring) {
205                 for (i = 0;
206                      i < rxq->rx_ring->rx_ring_struct->ring_size; i++) {
207                         if (sw_ring[i].mbuf) {
208                                 rte_pktmbuf_free_seg(sw_ring[i].mbuf);
209                                 sw_ring[i].mbuf = NULL;
210                         }
211                 }
212         }
213         /* Free up mbufs in Agg ring */
214         sw_ring = rxq->rx_ring->ag_buf_ring;
215         if (sw_ring) {
216                 for (i = 0;
217                      i < rxq->rx_ring->ag_ring_struct->ring_size; i++) {
218                         if (sw_ring[i].mbuf) {
219                                 rte_pktmbuf_free_seg(sw_ring[i].mbuf);
220                                 sw_ring[i].mbuf = NULL;
221                         }
222                 }
223         }
224
225         /* Free up mbufs in TPA */
226         tpa_info = rxq->rx_ring->tpa_info;
227         if (tpa_info) {
228                 for (i = 0; i < BNXT_TPA_MAX; i++) {
229                         if (tpa_info[i].mbuf) {
230                                 rte_pktmbuf_free_seg(tpa_info[i].mbuf);
231                                 tpa_info[i].mbuf = NULL;
232                         }
233                 }
234         }
235
236         rte_spinlock_unlock(&rxq->lock);
237 }
238
239 void bnxt_free_rx_mbufs(struct bnxt *bp)
240 {
241         struct bnxt_rx_queue *rxq;
242         int i;
243
244         for (i = 0; i < (int)bp->rx_nr_rings; i++) {
245                 rxq = bp->rx_queues[i];
246                 bnxt_rx_queue_release_mbufs(rxq);
247         }
248 }
249
250 void bnxt_rx_queue_release_op(void *rx_queue)
251 {
252         struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue;
253
254         if (rxq) {
255                 if (is_bnxt_in_error(rxq->bp))
256                         return;
257
258                 bnxt_rx_queue_release_mbufs(rxq);
259
260                 /* Free RX ring hardware descriptors */
261                 bnxt_free_ring(rxq->rx_ring->rx_ring_struct);
262                 /* Free RX Agg ring hardware descriptors */
263                 bnxt_free_ring(rxq->rx_ring->ag_ring_struct);
264
265                 /* Free RX completion ring hardware descriptors */
266                 bnxt_free_ring(rxq->cp_ring->cp_ring_struct);
267
268                 bnxt_free_rxq_stats(rxq);
269                 rte_memzone_free(rxq->mz);
270                 rxq->mz = NULL;
271
272                 rte_free(rxq);
273         }
274 }
275
276 int bnxt_rx_queue_setup_op(struct rte_eth_dev *eth_dev,
277                                uint16_t queue_idx,
278                                uint16_t nb_desc,
279                                unsigned int socket_id,
280                                const struct rte_eth_rxconf *rx_conf,
281                                struct rte_mempool *mp)
282 {
283         struct bnxt *bp = eth_dev->data->dev_private;
284         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
285         struct bnxt_rx_queue *rxq;
286         int rc = 0;
287         uint8_t queue_state;
288
289         rc = is_bnxt_in_error(bp);
290         if (rc)
291                 return rc;
292
293         if (queue_idx >= bp->max_rx_rings) {
294                 PMD_DRV_LOG(ERR,
295                         "Cannot create Rx ring %d. Only %d rings available\n",
296                         queue_idx, bp->max_rx_rings);
297                 return -EINVAL;
298         }
299
300         if (!nb_desc || nb_desc > MAX_RX_DESC_CNT) {
301                 PMD_DRV_LOG(ERR, "nb_desc %d is invalid\n", nb_desc);
302                 rc = -EINVAL;
303                 goto out;
304         }
305
306         if (eth_dev->data->rx_queues) {
307                 rxq = eth_dev->data->rx_queues[queue_idx];
308                 if (rxq)
309                         bnxt_rx_queue_release_op(rxq);
310         }
311         rxq = rte_zmalloc_socket("bnxt_rx_queue", sizeof(struct bnxt_rx_queue),
312                                  RTE_CACHE_LINE_SIZE, socket_id);
313         if (!rxq) {
314                 PMD_DRV_LOG(ERR, "bnxt_rx_queue allocation failed!\n");
315                 rc = -ENOMEM;
316                 goto out;
317         }
318         rxq->bp = bp;
319         rxq->mb_pool = mp;
320         rxq->nb_rx_desc = nb_desc;
321         rxq->rx_free_thresh = rx_conf->rx_free_thresh;
322
323         PMD_DRV_LOG(DEBUG, "RX Buf MTU %d\n", eth_dev->data->mtu);
324
325         rc = bnxt_init_rx_ring_struct(rxq, socket_id);
326         if (rc)
327                 goto out;
328
329         PMD_DRV_LOG(DEBUG, "RX Buf size is %d\n", rxq->rx_buf_size);
330         rxq->queue_id = queue_idx;
331         rxq->port_id = eth_dev->data->port_id;
332         if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC)
333                 rxq->crc_len = RTE_ETHER_CRC_LEN;
334         else
335                 rxq->crc_len = 0;
336
337         eth_dev->data->rx_queues[queue_idx] = rxq;
338         /* Allocate RX ring hardware descriptors */
339         if (bnxt_alloc_rings(bp, queue_idx, NULL, rxq, rxq->cp_ring,
340                         rxq->nq_ring, "rxr")) {
341                 PMD_DRV_LOG(ERR,
342                         "ring_dma_zone_reserve for rx_ring failed!\n");
343                 bnxt_rx_queue_release_op(rxq);
344                 rc = -ENOMEM;
345                 goto out;
346         }
347         rte_atomic64_init(&rxq->rx_mbuf_alloc_fail);
348
349         /* rxq 0 must not be stopped when used as async CPR */
350         if (!BNXT_NUM_ASYNC_CPR(bp) && queue_idx == 0)
351                 rxq->rx_deferred_start = false;
352         else
353                 rxq->rx_deferred_start = rx_conf->rx_deferred_start;
354
355         if (rxq->rx_deferred_start) {
356                 queue_state = RTE_ETH_QUEUE_STATE_STOPPED;
357                 rxq->rx_started = false;
358         } else {
359                 queue_state = RTE_ETH_QUEUE_STATE_STARTED;
360                 rxq->rx_started = true;
361         }
362         eth_dev->data->rx_queue_state[queue_idx] = queue_state;
363         rte_spinlock_init(&rxq->lock);
364
365 out:
366         return rc;
367 }
368
369 int
370 bnxt_rx_queue_intr_enable_op(struct rte_eth_dev *eth_dev, uint16_t queue_id)
371 {
372         struct bnxt *bp = eth_dev->data->dev_private;
373         struct bnxt_rx_queue *rxq;
374         struct bnxt_cp_ring_info *cpr;
375         int rc = 0;
376
377         rc = is_bnxt_in_error(bp);
378         if (rc)
379                 return rc;
380
381         if (eth_dev->data->rx_queues) {
382                 rxq = eth_dev->data->rx_queues[queue_id];
383                 if (!rxq) {
384                         rc = -EINVAL;
385                         return rc;
386                 }
387                 cpr = rxq->cp_ring;
388                 B_CP_DB_REARM(cpr, cpr->cp_raw_cons);
389         }
390         return rc;
391 }
392
393 int
394 bnxt_rx_queue_intr_disable_op(struct rte_eth_dev *eth_dev, uint16_t queue_id)
395 {
396         struct bnxt *bp = eth_dev->data->dev_private;
397         struct bnxt_rx_queue *rxq;
398         struct bnxt_cp_ring_info *cpr;
399         int rc = 0;
400
401         rc = is_bnxt_in_error(bp);
402         if (rc)
403                 return rc;
404
405         if (eth_dev->data->rx_queues) {
406                 rxq = eth_dev->data->rx_queues[queue_id];
407                 if (!rxq) {
408                         rc = -EINVAL;
409                         return rc;
410                 }
411                 cpr = rxq->cp_ring;
412                 B_CP_DB_DISARM(cpr);
413         }
414         return rc;
415 }
416
417 int bnxt_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
418 {
419         struct bnxt *bp = dev->data->dev_private;
420         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
421         struct bnxt_rx_queue *rxq = bp->rx_queues[rx_queue_id];
422         struct bnxt_vnic_info *vnic = NULL;
423         int rc = 0;
424
425         rc = is_bnxt_in_error(bp);
426         if (rc)
427                 return rc;
428
429         if (rxq == NULL) {
430                 PMD_DRV_LOG(ERR, "Invalid Rx queue %d\n", rx_queue_id);
431                 return -EINVAL;
432         }
433
434         /* Set the queue state to started here.
435          * We check the status of the queue while posting buffer.
436          * If queue is it started, we do not post buffers for Rx.
437          */
438         rxq->rx_started = true;
439         bnxt_free_hwrm_rx_ring(bp, rx_queue_id);
440         rc = bnxt_alloc_hwrm_rx_ring(bp, rx_queue_id);
441         if (rc)
442                 return rc;
443
444         PMD_DRV_LOG(INFO, "Rx queue started %d\n", rx_queue_id);
445
446         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
447                 vnic = rxq->vnic;
448
449                 if (BNXT_HAS_RING_GRPS(bp)) {
450                         if (vnic->fw_grp_ids[rx_queue_id] != INVALID_HW_RING_ID)
451                                 return 0;
452
453                         vnic->fw_grp_ids[rx_queue_id] =
454                                         bp->grp_info[rx_queue_id].fw_grp_id;
455                         PMD_DRV_LOG(DEBUG,
456                                     "vnic = %p fw_grp_id = %d\n",
457                                     vnic, bp->grp_info[rx_queue_id].fw_grp_id);
458                 }
459
460                 PMD_DRV_LOG(DEBUG, "Rx Queue Count %d\n", vnic->rx_queue_cnt);
461                 if (vnic->rx_queue_cnt > 1)
462                         rc = bnxt_vnic_rss_configure(bp, vnic);
463         }
464
465         if (rc == 0)
466                 dev->data->rx_queue_state[rx_queue_id] =
467                                 RTE_ETH_QUEUE_STATE_STARTED;
468         else
469                 rxq->rx_started = false;
470
471         PMD_DRV_LOG(INFO,
472                     "queue %d, rx_deferred_start %d, state %d!\n",
473                     rx_queue_id, rxq->rx_deferred_start,
474                     bp->eth_dev->data->rx_queue_state[rx_queue_id]);
475
476         return rc;
477 }
478
479 int bnxt_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
480 {
481         struct bnxt *bp = dev->data->dev_private;
482         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
483         struct bnxt_vnic_info *vnic = NULL;
484         struct bnxt_rx_queue *rxq = NULL;
485         int rc = 0;
486
487         rc = is_bnxt_in_error(bp);
488         if (rc)
489                 return rc;
490
491         /* For the stingray platform and other platforms needing tighter
492          * control of resource utilization, Rx CQ 0 also works as
493          * Default CQ for async notifications
494          */
495         if (!BNXT_NUM_ASYNC_CPR(bp) && !rx_queue_id) {
496                 PMD_DRV_LOG(ERR, "Cannot stop Rx queue id %d\n", rx_queue_id);
497                 return -EINVAL;
498         }
499
500         rxq = bp->rx_queues[rx_queue_id];
501
502         if (rxq == NULL) {
503                 PMD_DRV_LOG(ERR, "Invalid Rx queue %d\n", rx_queue_id);
504                 return -EINVAL;
505         }
506
507         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
508         rxq->rx_started = false;
509         PMD_DRV_LOG(DEBUG, "Rx queue stopped\n");
510
511         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
512                 vnic = rxq->vnic;
513                 if (BNXT_HAS_RING_GRPS(bp))
514                         vnic->fw_grp_ids[rx_queue_id] = INVALID_HW_RING_ID;
515
516                 PMD_DRV_LOG(DEBUG, "Rx Queue Count %d\n", vnic->rx_queue_cnt);
517                 if (vnic->rx_queue_cnt > 1)
518                         rc = bnxt_vnic_rss_configure(bp, vnic);
519         }
520
521         if (rc == 0)
522                 bnxt_rx_queue_release_mbufs(rxq);
523
524         return rc;
525 }