vdpa/mlx5: support queue update
[dpdk.git] / drivers / net / ice / ice_dcf.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2020 Intel Corporation
3  */
4
5 #include <sys/queue.h>
6 #include <stdio.h>
7 #include <errno.h>
8 #include <stdint.h>
9 #include <string.h>
10 #include <unistd.h>
11 #include <stdarg.h>
12 #include <inttypes.h>
13 #include <rte_byteorder.h>
14 #include <rte_common.h>
15
16 #include <rte_pci.h>
17 #include <rte_atomic.h>
18 #include <rte_eal.h>
19 #include <rte_ether.h>
20 #include <rte_ethdev_driver.h>
21 #include <rte_ethdev_pci.h>
22 #include <rte_malloc.h>
23 #include <rte_memzone.h>
24 #include <rte_dev.h>
25
26 #include "ice_dcf.h"
27 #include "ice_rxtx.h"
28
29 #define ICE_DCF_AQ_LEN     32
30 #define ICE_DCF_AQ_BUF_SZ  4096
31
32 #define ICE_DCF_ARQ_MAX_RETRIES 200
33 #define ICE_DCF_ARQ_CHECK_TIME  2   /* msecs */
34
35 #define ICE_DCF_VF_RES_BUF_SZ   \
36         (sizeof(struct virtchnl_vf_resource) +  \
37                 IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource))
38
39 static __rte_always_inline int
40 ice_dcf_send_cmd_req_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op,
41                             uint8_t *req_msg, uint16_t req_msglen)
42 {
43         return iavf_aq_send_msg_to_pf(&hw->avf, op, IAVF_SUCCESS,
44                                       req_msg, req_msglen, NULL);
45 }
46
47 static int
48 ice_dcf_recv_cmd_rsp_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op,
49                             uint8_t *rsp_msgbuf, uint16_t rsp_buflen,
50                             uint16_t *rsp_msglen)
51 {
52         struct iavf_arq_event_info event;
53         enum virtchnl_ops v_op;
54         int i = 0;
55         int err;
56
57         event.buf_len = rsp_buflen;
58         event.msg_buf = rsp_msgbuf;
59
60         do {
61                 err = iavf_clean_arq_element(&hw->avf, &event, NULL);
62                 if (err != IAVF_SUCCESS)
63                         goto again;
64
65                 v_op = rte_le_to_cpu_32(event.desc.cookie_high);
66                 if (v_op != op)
67                         goto again;
68
69                 if (rsp_msglen != NULL)
70                         *rsp_msglen = event.msg_len;
71                 return rte_le_to_cpu_32(event.desc.cookie_low);
72
73 again:
74                 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME);
75         } while (i++ < ICE_DCF_ARQ_MAX_RETRIES);
76
77         return -EIO;
78 }
79
80 static __rte_always_inline void
81 ice_dcf_aq_cmd_clear(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd)
82 {
83         rte_spinlock_lock(&hw->vc_cmd_queue_lock);
84
85         TAILQ_REMOVE(&hw->vc_cmd_queue, cmd, next);
86
87         rte_spinlock_unlock(&hw->vc_cmd_queue_lock);
88 }
89
90 static __rte_always_inline void
91 ice_dcf_vc_cmd_set(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd)
92 {
93         cmd->v_ret = IAVF_ERR_NOT_READY;
94         cmd->rsp_msglen = 0;
95         cmd->pending = 1;
96
97         rte_spinlock_lock(&hw->vc_cmd_queue_lock);
98
99         TAILQ_INSERT_TAIL(&hw->vc_cmd_queue, cmd, next);
100
101         rte_spinlock_unlock(&hw->vc_cmd_queue_lock);
102 }
103
104 static __rte_always_inline int
105 ice_dcf_vc_cmd_send(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd)
106 {
107         return iavf_aq_send_msg_to_pf(&hw->avf,
108                                       cmd->v_op, IAVF_SUCCESS,
109                                       cmd->req_msg, cmd->req_msglen, NULL);
110 }
111
112 static __rte_always_inline void
113 ice_dcf_aq_cmd_handle(struct ice_dcf_hw *hw, struct iavf_arq_event_info *info)
114 {
115         struct dcf_virtchnl_cmd *cmd;
116         enum virtchnl_ops v_op;
117         enum iavf_status v_ret;
118         uint16_t aq_op;
119
120         aq_op = rte_le_to_cpu_16(info->desc.opcode);
121         if (unlikely(aq_op != iavf_aqc_opc_send_msg_to_vf)) {
122                 PMD_DRV_LOG(ERR,
123                             "Request %u is not supported yet", aq_op);
124                 return;
125         }
126
127         v_op = rte_le_to_cpu_32(info->desc.cookie_high);
128         if (v_op == VIRTCHNL_OP_EVENT) {
129                 if (hw->vc_event_msg_cb != NULL)
130                         hw->vc_event_msg_cb(hw,
131                                             info->msg_buf,
132                                             info->msg_len);
133                 return;
134         }
135
136         v_ret = rte_le_to_cpu_32(info->desc.cookie_low);
137
138         rte_spinlock_lock(&hw->vc_cmd_queue_lock);
139
140         TAILQ_FOREACH(cmd, &hw->vc_cmd_queue, next) {
141                 if (cmd->v_op == v_op && cmd->pending) {
142                         cmd->v_ret = v_ret;
143                         cmd->rsp_msglen = RTE_MIN(info->msg_len,
144                                                   cmd->rsp_buflen);
145                         if (likely(cmd->rsp_msglen != 0))
146                                 rte_memcpy(cmd->rsp_msgbuf, info->msg_buf,
147                                            cmd->rsp_msglen);
148
149                         /* prevent compiler reordering */
150                         rte_compiler_barrier();
151                         cmd->pending = 0;
152                         break;
153                 }
154         }
155
156         rte_spinlock_unlock(&hw->vc_cmd_queue_lock);
157 }
158
159 static void
160 ice_dcf_handle_virtchnl_msg(struct ice_dcf_hw *hw)
161 {
162         struct iavf_arq_event_info info;
163         uint16_t pending = 1;
164         int ret;
165
166         info.buf_len = ICE_DCF_AQ_BUF_SZ;
167         info.msg_buf = hw->arq_buf;
168
169         while (pending) {
170                 ret = iavf_clean_arq_element(&hw->avf, &info, &pending);
171                 if (ret != IAVF_SUCCESS)
172                         break;
173
174                 ice_dcf_aq_cmd_handle(hw, &info);
175         }
176 }
177
178 static int
179 ice_dcf_init_check_api_version(struct ice_dcf_hw *hw)
180 {
181 #define ICE_CPF_VIRTCHNL_VERSION_MAJOR_START    1
182 #define ICE_CPF_VIRTCHNL_VERSION_MINOR_START    1
183         struct virtchnl_version_info version, *pver;
184         int err;
185
186         version.major = VIRTCHNL_VERSION_MAJOR;
187         version.minor = VIRTCHNL_VERSION_MINOR;
188         err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_VERSION,
189                                           (uint8_t *)&version, sizeof(version));
190         if (err) {
191                 PMD_INIT_LOG(ERR, "Failed to send OP_VERSION");
192                 return err;
193         }
194
195         pver = &hw->virtchnl_version;
196         err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_VERSION,
197                                           (uint8_t *)pver, sizeof(*pver), NULL);
198         if (err) {
199                 PMD_INIT_LOG(ERR, "Failed to get response of OP_VERSION");
200                 return -1;
201         }
202
203         PMD_INIT_LOG(DEBUG,
204                      "Peer PF API version: %u.%u", pver->major, pver->minor);
205
206         if (pver->major < ICE_CPF_VIRTCHNL_VERSION_MAJOR_START ||
207             (pver->major == ICE_CPF_VIRTCHNL_VERSION_MAJOR_START &&
208              pver->minor < ICE_CPF_VIRTCHNL_VERSION_MINOR_START)) {
209                 PMD_INIT_LOG(ERR,
210                              "VIRTCHNL API version should not be lower than (%u.%u)",
211                              ICE_CPF_VIRTCHNL_VERSION_MAJOR_START,
212                              ICE_CPF_VIRTCHNL_VERSION_MAJOR_START);
213                 return -1;
214         } else if (pver->major > VIRTCHNL_VERSION_MAJOR ||
215                    (pver->major == VIRTCHNL_VERSION_MAJOR &&
216                     pver->minor > VIRTCHNL_VERSION_MINOR)) {
217                 PMD_INIT_LOG(ERR,
218                              "PF/VF API version mismatch:(%u.%u)-(%u.%u)",
219                              pver->major, pver->minor,
220                              VIRTCHNL_VERSION_MAJOR, VIRTCHNL_VERSION_MINOR);
221                 return -1;
222         }
223
224         PMD_INIT_LOG(DEBUG, "Peer is supported PF host");
225
226         return 0;
227 }
228
229 static int
230 ice_dcf_get_vf_resource(struct ice_dcf_hw *hw)
231 {
232         uint32_t caps;
233         int err, i;
234
235         caps = VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_RX_POLLING |
236                VIRTCHNL_VF_CAP_ADV_LINK_SPEED | VIRTCHNL_VF_CAP_DCF |
237                VF_BASE_MODE_OFFLOADS | VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC;
238
239         err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES,
240                                           (uint8_t *)&caps, sizeof(caps));
241         if (err) {
242                 PMD_DRV_LOG(ERR, "Failed to send msg OP_GET_VF_RESOURCE");
243                 return err;
244         }
245
246         err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES,
247                                           (uint8_t *)hw->vf_res,
248                                           ICE_DCF_VF_RES_BUF_SZ, NULL);
249         if (err) {
250                 PMD_DRV_LOG(ERR, "Failed to get response of OP_GET_VF_RESOURCE");
251                 return -1;
252         }
253
254         iavf_vf_parse_hw_config(&hw->avf, hw->vf_res);
255
256         hw->vsi_res = NULL;
257         for (i = 0; i < hw->vf_res->num_vsis; i++) {
258                 if (hw->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
259                         hw->vsi_res = &hw->vf_res->vsi_res[i];
260         }
261
262         if (!hw->vsi_res) {
263                 PMD_DRV_LOG(ERR, "no LAN VSI found");
264                 return -1;
265         }
266
267         hw->vsi_id = hw->vsi_res->vsi_id;
268         PMD_DRV_LOG(DEBUG, "VSI ID is %u", hw->vsi_id);
269
270         return 0;
271 }
272
273 static int
274 ice_dcf_get_vf_vsi_map(struct ice_dcf_hw *hw)
275 {
276         struct virtchnl_dcf_vsi_map *vsi_map;
277         uint32_t valid_msg_len;
278         uint16_t len;
279         int err;
280
281         err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP,
282                                           NULL, 0);
283         if (err) {
284                 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_GET_VSI_MAP");
285                 return err;
286         }
287
288         err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP,
289                                           hw->arq_buf, ICE_DCF_AQ_BUF_SZ,
290                                           &len);
291         if (err) {
292                 PMD_DRV_LOG(ERR, "Failed to get response of OP_DCF_GET_VSI_MAP");
293                 return err;
294         }
295
296         vsi_map = (struct virtchnl_dcf_vsi_map *)hw->arq_buf;
297         valid_msg_len = (vsi_map->num_vfs - 1) * sizeof(vsi_map->vf_vsi[0]) +
298                         sizeof(*vsi_map);
299         if (len != valid_msg_len) {
300                 PMD_DRV_LOG(ERR, "invalid vf vsi map response with length %u",
301                             len);
302                 return -EINVAL;
303         }
304
305         if (hw->num_vfs != 0 && hw->num_vfs != vsi_map->num_vfs) {
306                 PMD_DRV_LOG(ERR, "The number VSI map (%u) doesn't match the number of VFs (%u)",
307                             vsi_map->num_vfs, hw->num_vfs);
308                 return -EINVAL;
309         }
310
311         len = vsi_map->num_vfs * sizeof(vsi_map->vf_vsi[0]);
312
313         if (!hw->vf_vsi_map) {
314                 hw->vf_vsi_map = rte_zmalloc("vf_vsi_ctx", len, 0);
315                 if (!hw->vf_vsi_map) {
316                         PMD_DRV_LOG(ERR, "Failed to alloc memory for VSI context");
317                         return -ENOMEM;
318                 }
319
320                 hw->num_vfs = vsi_map->num_vfs;
321         }
322
323         if (!memcmp(hw->vf_vsi_map, vsi_map->vf_vsi, len)) {
324                 PMD_DRV_LOG(DEBUG, "VF VSI map doesn't change");
325                 return 1;
326         }
327
328         rte_memcpy(hw->vf_vsi_map, vsi_map->vf_vsi, len);
329         return 0;
330 }
331
332 static int
333 ice_dcf_mode_disable(struct ice_dcf_hw *hw)
334 {
335         int err;
336
337         err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE,
338                                           NULL, 0);
339         if (err) {
340                 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_DISABLE");
341                 return err;
342         }
343
344         err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE,
345                                           hw->arq_buf, ICE_DCF_AQ_BUF_SZ, NULL);
346         if (err) {
347                 PMD_DRV_LOG(ERR,
348                             "Failed to get response of OP_DCF_DISABLE %d",
349                             err);
350                 return -1;
351         }
352
353         return 0;
354 }
355
356 static int
357 ice_dcf_check_reset_done(struct ice_dcf_hw *hw)
358 {
359 #define ICE_DCF_RESET_WAIT_CNT       50
360         struct iavf_hw *avf = &hw->avf;
361         int i, reset;
362
363         for (i = 0; i < ICE_DCF_RESET_WAIT_CNT; i++) {
364                 reset = IAVF_READ_REG(avf, IAVF_VFGEN_RSTAT) &
365                                         IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
366                 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT;
367
368                 if (reset == VIRTCHNL_VFR_VFACTIVE ||
369                     reset == VIRTCHNL_VFR_COMPLETED)
370                         break;
371
372                 rte_delay_ms(20);
373         }
374
375         if (i >= ICE_DCF_RESET_WAIT_CNT)
376                 return -1;
377
378         return 0;
379 }
380
381 static inline void
382 ice_dcf_enable_irq0(struct ice_dcf_hw *hw)
383 {
384         struct iavf_hw *avf = &hw->avf;
385
386         /* Enable admin queue interrupt trigger */
387         IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1,
388                        IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
389         IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01,
390                        IAVF_VFINT_DYN_CTL01_INTENA_MASK |
391                        IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK |
392                        IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
393
394         IAVF_WRITE_FLUSH(avf);
395 }
396
397 static inline void
398 ice_dcf_disable_irq0(struct ice_dcf_hw *hw)
399 {
400         struct iavf_hw *avf = &hw->avf;
401
402         /* Disable all interrupt types */
403         IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 0);
404         IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01,
405                        IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
406
407         IAVF_WRITE_FLUSH(avf);
408 }
409
410 static void
411 ice_dcf_dev_interrupt_handler(void *param)
412 {
413         struct ice_dcf_hw *hw = param;
414
415         ice_dcf_disable_irq0(hw);
416
417         ice_dcf_handle_virtchnl_msg(hw);
418
419         ice_dcf_enable_irq0(hw);
420 }
421
422 int
423 ice_dcf_execute_virtchnl_cmd(struct ice_dcf_hw *hw,
424                              struct dcf_virtchnl_cmd *cmd)
425 {
426         int i = 0;
427         int err;
428
429         if ((cmd->req_msg && !cmd->req_msglen) ||
430             (!cmd->req_msg && cmd->req_msglen) ||
431             (cmd->rsp_msgbuf && !cmd->rsp_buflen) ||
432             (!cmd->rsp_msgbuf && cmd->rsp_buflen))
433                 return -EINVAL;
434
435         rte_spinlock_lock(&hw->vc_cmd_send_lock);
436         ice_dcf_vc_cmd_set(hw, cmd);
437
438         err = ice_dcf_vc_cmd_send(hw, cmd);
439         if (err) {
440                 PMD_DRV_LOG(ERR, "fail to send cmd %d", cmd->v_op);
441                 goto ret;
442         }
443
444         do {
445                 if (!cmd->pending)
446                         break;
447
448                 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME);
449         } while (i++ < ICE_DCF_ARQ_MAX_RETRIES);
450
451         if (cmd->v_ret != IAVF_SUCCESS) {
452                 err = -1;
453                 PMD_DRV_LOG(ERR,
454                             "No response (%d times) or return failure (%d) for cmd %d",
455                             i, cmd->v_ret, cmd->v_op);
456         }
457
458 ret:
459         ice_dcf_aq_cmd_clear(hw, cmd);
460         rte_spinlock_unlock(&hw->vc_cmd_send_lock);
461         return err;
462 }
463
464 int
465 ice_dcf_send_aq_cmd(void *dcf_hw, struct ice_aq_desc *desc,
466                     void *buf, uint16_t buf_size)
467 {
468         struct dcf_virtchnl_cmd desc_cmd, buff_cmd;
469         struct ice_dcf_hw *hw = dcf_hw;
470         int err = 0;
471         int i = 0;
472
473         if ((buf && !buf_size) || (!buf && buf_size) ||
474             buf_size > ICE_DCF_AQ_BUF_SZ)
475                 return -EINVAL;
476
477         desc_cmd.v_op = VIRTCHNL_OP_DCF_CMD_DESC;
478         desc_cmd.req_msglen = sizeof(*desc);
479         desc_cmd.req_msg = (uint8_t *)desc;
480         desc_cmd.rsp_buflen = sizeof(*desc);
481         desc_cmd.rsp_msgbuf = (uint8_t *)desc;
482
483         if (buf == NULL)
484                 return ice_dcf_execute_virtchnl_cmd(hw, &desc_cmd);
485
486         desc->flags |= rte_cpu_to_le_16(ICE_AQ_FLAG_BUF);
487
488         buff_cmd.v_op = VIRTCHNL_OP_DCF_CMD_BUFF;
489         buff_cmd.req_msglen = buf_size;
490         buff_cmd.req_msg = buf;
491         buff_cmd.rsp_buflen = buf_size;
492         buff_cmd.rsp_msgbuf = buf;
493
494         rte_spinlock_lock(&hw->vc_cmd_send_lock);
495         ice_dcf_vc_cmd_set(hw, &desc_cmd);
496         ice_dcf_vc_cmd_set(hw, &buff_cmd);
497
498         if (ice_dcf_vc_cmd_send(hw, &desc_cmd) ||
499             ice_dcf_vc_cmd_send(hw, &buff_cmd)) {
500                 err = -1;
501                 PMD_DRV_LOG(ERR, "fail to send OP_DCF_CMD_DESC/BUFF");
502                 goto ret;
503         }
504
505         do {
506                 if ((!desc_cmd.pending && !buff_cmd.pending) ||
507                     (!desc_cmd.pending && desc_cmd.v_ret != IAVF_SUCCESS) ||
508                     (!buff_cmd.pending && buff_cmd.v_ret != IAVF_SUCCESS))
509                         break;
510
511                 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME);
512         } while (i++ < ICE_DCF_ARQ_MAX_RETRIES);
513
514         if (desc_cmd.v_ret != IAVF_SUCCESS || buff_cmd.v_ret != IAVF_SUCCESS) {
515                 err = -1;
516                 PMD_DRV_LOG(ERR,
517                             "No response (%d times) or return failure (desc: %d / buff: %d)",
518                             i, desc_cmd.v_ret, buff_cmd.v_ret);
519         }
520
521 ret:
522         ice_dcf_aq_cmd_clear(hw, &desc_cmd);
523         ice_dcf_aq_cmd_clear(hw, &buff_cmd);
524         rte_spinlock_unlock(&hw->vc_cmd_send_lock);
525
526         return err;
527 }
528
529 int
530 ice_dcf_handle_vsi_update_event(struct ice_dcf_hw *hw)
531 {
532         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(hw->eth_dev);
533         int err = 0;
534
535         rte_spinlock_lock(&hw->vc_cmd_send_lock);
536
537         rte_intr_disable(&pci_dev->intr_handle);
538         ice_dcf_disable_irq0(hw);
539
540         if (ice_dcf_get_vf_resource(hw) || ice_dcf_get_vf_vsi_map(hw) < 0)
541                 err = -1;
542
543         rte_intr_enable(&pci_dev->intr_handle);
544         ice_dcf_enable_irq0(hw);
545
546         rte_spinlock_unlock(&hw->vc_cmd_send_lock);
547
548         return err;
549 }
550
551 static int
552 ice_dcf_get_supported_rxdid(struct ice_dcf_hw *hw)
553 {
554         int err;
555
556         err = ice_dcf_send_cmd_req_no_irq(hw,
557                                           VIRTCHNL_OP_GET_SUPPORTED_RXDIDS,
558                                           NULL, 0);
559         if (err) {
560                 PMD_INIT_LOG(ERR, "Failed to send OP_GET_SUPPORTED_RXDIDS");
561                 return -1;
562         }
563
564         err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_SUPPORTED_RXDIDS,
565                                           (uint8_t *)&hw->supported_rxdid,
566                                           sizeof(uint64_t), NULL);
567         if (err) {
568                 PMD_INIT_LOG(ERR, "Failed to get response of OP_GET_SUPPORTED_RXDIDS");
569                 return -1;
570         }
571
572         return 0;
573 }
574
575 int
576 ice_dcf_init_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw)
577 {
578         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
579         int ret;
580
581         hw->avf.hw_addr = pci_dev->mem_resource[0].addr;
582         hw->avf.back = hw;
583
584         hw->avf.bus.bus_id = pci_dev->addr.bus;
585         hw->avf.bus.device = pci_dev->addr.devid;
586         hw->avf.bus.func = pci_dev->addr.function;
587
588         hw->avf.device_id = pci_dev->id.device_id;
589         hw->avf.vendor_id = pci_dev->id.vendor_id;
590         hw->avf.subsystem_device_id = pci_dev->id.subsystem_device_id;
591         hw->avf.subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
592
593         hw->avf.aq.num_arq_entries = ICE_DCF_AQ_LEN;
594         hw->avf.aq.num_asq_entries = ICE_DCF_AQ_LEN;
595         hw->avf.aq.arq_buf_size = ICE_DCF_AQ_BUF_SZ;
596         hw->avf.aq.asq_buf_size = ICE_DCF_AQ_BUF_SZ;
597
598         rte_spinlock_init(&hw->vc_cmd_send_lock);
599         rte_spinlock_init(&hw->vc_cmd_queue_lock);
600         TAILQ_INIT(&hw->vc_cmd_queue);
601
602         hw->arq_buf = rte_zmalloc("arq_buf", ICE_DCF_AQ_BUF_SZ, 0);
603         if (hw->arq_buf == NULL) {
604                 PMD_INIT_LOG(ERR, "unable to allocate AdminQ buffer memory");
605                 goto err;
606         }
607
608         ret = iavf_set_mac_type(&hw->avf);
609         if (ret) {
610                 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", ret);
611                 goto err;
612         }
613
614         ret = ice_dcf_check_reset_done(hw);
615         if (ret) {
616                 PMD_INIT_LOG(ERR, "VF is still resetting");
617                 goto err;
618         }
619
620         ret = iavf_init_adminq(&hw->avf);
621         if (ret) {
622                 PMD_INIT_LOG(ERR, "init_adminq failed: %d", ret);
623                 goto err;
624         }
625
626         if (ice_dcf_init_check_api_version(hw)) {
627                 PMD_INIT_LOG(ERR, "check_api version failed");
628                 goto err_api;
629         }
630
631         hw->vf_res = rte_zmalloc("vf_res", ICE_DCF_VF_RES_BUF_SZ, 0);
632         if (hw->vf_res == NULL) {
633                 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory");
634                 goto err_api;
635         }
636
637         if (ice_dcf_get_vf_resource(hw)) {
638                 PMD_INIT_LOG(ERR, "Failed to get VF resource");
639                 goto err_alloc;
640         }
641
642         if (ice_dcf_get_vf_vsi_map(hw) < 0) {
643                 PMD_INIT_LOG(ERR, "Failed to get VF VSI map");
644                 ice_dcf_mode_disable(hw);
645                 goto err_alloc;
646         }
647
648         /* Allocate memory for RSS info */
649         if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
650                 hw->rss_key = rte_zmalloc(NULL,
651                                           hw->vf_res->rss_key_size, 0);
652                 if (!hw->rss_key) {
653                         PMD_INIT_LOG(ERR, "unable to allocate rss_key memory");
654                         goto err_alloc;
655                 }
656                 hw->rss_lut = rte_zmalloc("rss_lut",
657                                           hw->vf_res->rss_lut_size, 0);
658                 if (!hw->rss_lut) {
659                         PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory");
660                         goto err_rss;
661                 }
662         }
663
664         if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) {
665                 if (ice_dcf_get_supported_rxdid(hw) != 0) {
666                         PMD_INIT_LOG(ERR, "failed to do get supported rxdid");
667                         goto err_rss;
668                 }
669         }
670
671         hw->eth_dev = eth_dev;
672         rte_intr_callback_register(&pci_dev->intr_handle,
673                                    ice_dcf_dev_interrupt_handler, hw);
674         rte_intr_enable(&pci_dev->intr_handle);
675         ice_dcf_enable_irq0(hw);
676
677         return 0;
678
679 err_rss:
680         rte_free(hw->rss_key);
681         rte_free(hw->rss_lut);
682 err_alloc:
683         rte_free(hw->vf_res);
684 err_api:
685         iavf_shutdown_adminq(&hw->avf);
686 err:
687         rte_free(hw->arq_buf);
688
689         return -1;
690 }
691
692 void
693 ice_dcf_uninit_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw)
694 {
695         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
696         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
697
698         ice_dcf_disable_irq0(hw);
699         rte_intr_disable(intr_handle);
700         rte_intr_callback_unregister(intr_handle,
701                                      ice_dcf_dev_interrupt_handler, hw);
702
703         ice_dcf_mode_disable(hw);
704         iavf_shutdown_adminq(&hw->avf);
705
706         rte_free(hw->arq_buf);
707         rte_free(hw->vf_vsi_map);
708         rte_free(hw->vf_res);
709         rte_free(hw->rss_lut);
710         rte_free(hw->rss_key);
711 }
712
713 static int
714 ice_dcf_configure_rss_key(struct ice_dcf_hw *hw)
715 {
716         struct virtchnl_rss_key *rss_key;
717         struct dcf_virtchnl_cmd args;
718         int len, err;
719
720         len = sizeof(*rss_key) + hw->vf_res->rss_key_size - 1;
721         rss_key = rte_zmalloc("rss_key", len, 0);
722         if (!rss_key)
723                 return -ENOMEM;
724
725         rss_key->vsi_id = hw->vsi_res->vsi_id;
726         rss_key->key_len = hw->vf_res->rss_key_size;
727         rte_memcpy(rss_key->key, hw->rss_key, hw->vf_res->rss_key_size);
728
729         args.v_op = VIRTCHNL_OP_CONFIG_RSS_KEY;
730         args.req_msglen = len;
731         args.req_msg = (uint8_t *)rss_key;
732         args.rsp_msglen = 0;
733         args.rsp_buflen = 0;
734         args.rsp_msgbuf = NULL;
735         args.pending = 0;
736
737         err = ice_dcf_execute_virtchnl_cmd(hw, &args);
738         if (err)
739                 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_KEY");
740
741         rte_free(rss_key);
742         return err;
743 }
744
745 static int
746 ice_dcf_configure_rss_lut(struct ice_dcf_hw *hw)
747 {
748         struct virtchnl_rss_lut *rss_lut;
749         struct dcf_virtchnl_cmd args;
750         int len, err;
751
752         len = sizeof(*rss_lut) + hw->vf_res->rss_lut_size - 1;
753         rss_lut = rte_zmalloc("rss_lut", len, 0);
754         if (!rss_lut)
755                 return -ENOMEM;
756
757         rss_lut->vsi_id = hw->vsi_res->vsi_id;
758         rss_lut->lut_entries = hw->vf_res->rss_lut_size;
759         rte_memcpy(rss_lut->lut, hw->rss_lut, hw->vf_res->rss_lut_size);
760
761         args.v_op = VIRTCHNL_OP_CONFIG_RSS_LUT;
762         args.req_msglen = len;
763         args.req_msg = (uint8_t *)rss_lut;
764         args.rsp_msglen = 0;
765         args.rsp_buflen = 0;
766         args.rsp_msgbuf = NULL;
767         args.pending = 0;
768
769         err = ice_dcf_execute_virtchnl_cmd(hw, &args);
770         if (err)
771                 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_LUT");
772
773         rte_free(rss_lut);
774         return err;
775 }
776
777 int
778 ice_dcf_init_rss(struct ice_dcf_hw *hw)
779 {
780         struct rte_eth_dev *dev = hw->eth_dev;
781         struct rte_eth_rss_conf *rss_conf;
782         uint8_t i, j, nb_q;
783         int ret;
784
785         rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
786         nb_q = dev->data->nb_rx_queues;
787
788         if (!(hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
789                 PMD_DRV_LOG(DEBUG, "RSS is not supported");
790                 return -ENOTSUP;
791         }
792         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
793                 PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default");
794                 /* set all lut items to default queue */
795                 memset(hw->rss_lut, 0, hw->vf_res->rss_lut_size);
796                 return ice_dcf_configure_rss_lut(hw);
797         }
798
799         /* In IAVF, RSS enablement is set by PF driver. It is not supported
800          * to set based on rss_conf->rss_hf.
801          */
802
803         /* configure RSS key */
804         if (!rss_conf->rss_key)
805                 /* Calculate the default hash key */
806                 for (i = 0; i < hw->vf_res->rss_key_size; i++)
807                         hw->rss_key[i] = (uint8_t)rte_rand();
808         else
809                 rte_memcpy(hw->rss_key, rss_conf->rss_key,
810                            RTE_MIN(rss_conf->rss_key_len,
811                                    hw->vf_res->rss_key_size));
812
813         /* init RSS LUT table */
814         for (i = 0, j = 0; i < hw->vf_res->rss_lut_size; i++, j++) {
815                 if (j >= nb_q)
816                         j = 0;
817                 hw->rss_lut[i] = j;
818         }
819         /* send virtchnnl ops to configure rss*/
820         ret = ice_dcf_configure_rss_lut(hw);
821         if (ret)
822                 return ret;
823         ret = ice_dcf_configure_rss_key(hw);
824         if (ret)
825                 return ret;
826
827         return 0;
828 }
829
830 #define IAVF_RXDID_LEGACY_1 1
831 #define IAVF_RXDID_COMMS_GENERIC 16
832
833 int
834 ice_dcf_configure_queues(struct ice_dcf_hw *hw)
835 {
836         struct ice_rx_queue **rxq =
837                 (struct ice_rx_queue **)hw->eth_dev->data->rx_queues;
838         struct ice_tx_queue **txq =
839                 (struct ice_tx_queue **)hw->eth_dev->data->tx_queues;
840         struct virtchnl_vsi_queue_config_info *vc_config;
841         struct virtchnl_queue_pair_info *vc_qp;
842         struct dcf_virtchnl_cmd args;
843         uint16_t i, size;
844         int err;
845
846         size = sizeof(*vc_config) +
847                sizeof(vc_config->qpair[0]) * hw->num_queue_pairs;
848         vc_config = rte_zmalloc("cfg_queue", size, 0);
849         if (!vc_config)
850                 return -ENOMEM;
851
852         vc_config->vsi_id = hw->vsi_res->vsi_id;
853         vc_config->num_queue_pairs = hw->num_queue_pairs;
854
855         for (i = 0, vc_qp = vc_config->qpair;
856              i < hw->num_queue_pairs;
857              i++, vc_qp++) {
858                 vc_qp->txq.vsi_id = hw->vsi_res->vsi_id;
859                 vc_qp->txq.queue_id = i;
860                 if (i < hw->eth_dev->data->nb_tx_queues) {
861                         vc_qp->txq.ring_len = txq[i]->nb_tx_desc;
862                         vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_dma;
863                 }
864                 vc_qp->rxq.vsi_id = hw->vsi_res->vsi_id;
865                 vc_qp->rxq.queue_id = i;
866                 vc_qp->rxq.max_pkt_size = rxq[i]->max_pkt_len;
867
868                 if (i >= hw->eth_dev->data->nb_rx_queues)
869                         continue;
870
871                 vc_qp->rxq.ring_len = rxq[i]->nb_rx_desc;
872                 vc_qp->rxq.dma_ring_addr = rxq[i]->rx_ring_dma;
873                 vc_qp->rxq.databuffer_size = rxq[i]->rx_buf_len;
874
875                 if (hw->vf_res->vf_cap_flags &
876                     VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC &&
877                     hw->supported_rxdid &
878                     BIT(IAVF_RXDID_COMMS_GENERIC)) {
879                         vc_qp->rxq.rxdid = IAVF_RXDID_COMMS_GENERIC;
880                         PMD_DRV_LOG(NOTICE, "request RXDID == %d in "
881                                     "Queue[%d]", vc_qp->rxq.rxdid, i);
882                 } else {
883                         PMD_DRV_LOG(ERR, "RXDID 16 is not supported");
884                         return -EINVAL;
885                 }
886         }
887
888         memset(&args, 0, sizeof(args));
889         args.v_op = VIRTCHNL_OP_CONFIG_VSI_QUEUES;
890         args.req_msg = (uint8_t *)vc_config;
891         args.req_msglen = size;
892
893         err = ice_dcf_execute_virtchnl_cmd(hw, &args);
894         if (err)
895                 PMD_DRV_LOG(ERR, "Failed to execute command of"
896                             " VIRTCHNL_OP_CONFIG_VSI_QUEUES");
897
898         rte_free(vc_config);
899         return err;
900 }
901
902 int
903 ice_dcf_config_irq_map(struct ice_dcf_hw *hw)
904 {
905         struct virtchnl_irq_map_info *map_info;
906         struct virtchnl_vector_map *vecmap;
907         struct dcf_virtchnl_cmd args;
908         int len, i, err;
909
910         len = sizeof(struct virtchnl_irq_map_info) +
911               sizeof(struct virtchnl_vector_map) * hw->nb_msix;
912
913         map_info = rte_zmalloc("map_info", len, 0);
914         if (!map_info)
915                 return -ENOMEM;
916
917         map_info->num_vectors = hw->nb_msix;
918         for (i = 0; i < hw->nb_msix; i++) {
919                 vecmap = &map_info->vecmap[i];
920                 vecmap->vsi_id = hw->vsi_res->vsi_id;
921                 vecmap->rxitr_idx = 0;
922                 vecmap->vector_id = hw->msix_base + i;
923                 vecmap->txq_map = 0;
924                 vecmap->rxq_map = hw->rxq_map[hw->msix_base + i];
925         }
926
927         memset(&args, 0, sizeof(args));
928         args.v_op = VIRTCHNL_OP_CONFIG_IRQ_MAP;
929         args.req_msg = (u8 *)map_info;
930         args.req_msglen = len;
931
932         err = ice_dcf_execute_virtchnl_cmd(hw, &args);
933         if (err)
934                 PMD_DRV_LOG(ERR, "fail to execute command OP_CONFIG_IRQ_MAP");
935
936         rte_free(map_info);
937         return err;
938 }
939
940 int
941 ice_dcf_switch_queue(struct ice_dcf_hw *hw, uint16_t qid, bool rx, bool on)
942 {
943         struct virtchnl_queue_select queue_select;
944         struct dcf_virtchnl_cmd args;
945         int err;
946
947         memset(&queue_select, 0, sizeof(queue_select));
948         queue_select.vsi_id = hw->vsi_res->vsi_id;
949         if (rx)
950                 queue_select.rx_queues |= 1 << qid;
951         else
952                 queue_select.tx_queues |= 1 << qid;
953
954         memset(&args, 0, sizeof(args));
955         if (on)
956                 args.v_op = VIRTCHNL_OP_ENABLE_QUEUES;
957         else
958                 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES;
959
960         args.req_msg = (u8 *)&queue_select;
961         args.req_msglen = sizeof(queue_select);
962
963         err = ice_dcf_execute_virtchnl_cmd(hw, &args);
964         if (err)
965                 PMD_DRV_LOG(ERR, "Failed to execute command of %s",
966                             on ? "OP_ENABLE_QUEUES" : "OP_DISABLE_QUEUES");
967
968         return err;
969 }
970
971 int
972 ice_dcf_disable_queues(struct ice_dcf_hw *hw)
973 {
974         struct virtchnl_queue_select queue_select;
975         struct dcf_virtchnl_cmd args;
976         int err;
977
978         memset(&queue_select, 0, sizeof(queue_select));
979         queue_select.vsi_id = hw->vsi_res->vsi_id;
980
981         queue_select.rx_queues = BIT(hw->eth_dev->data->nb_rx_queues) - 1;
982         queue_select.tx_queues = BIT(hw->eth_dev->data->nb_tx_queues) - 1;
983
984         memset(&args, 0, sizeof(args));
985         args.v_op = VIRTCHNL_OP_DISABLE_QUEUES;
986         args.req_msg = (u8 *)&queue_select;
987         args.req_msglen = sizeof(queue_select);
988
989         err = ice_dcf_execute_virtchnl_cmd(hw, &args);
990         if (err)
991                 PMD_DRV_LOG(ERR,
992                             "Failed to execute command of OP_DISABLE_QUEUES");
993
994         return err;
995 }
996
997 int
998 ice_dcf_query_stats(struct ice_dcf_hw *hw,
999                                    struct virtchnl_eth_stats *pstats)
1000 {
1001         struct virtchnl_queue_select q_stats;
1002         struct dcf_virtchnl_cmd args;
1003         int err;
1004
1005         memset(&q_stats, 0, sizeof(q_stats));
1006         q_stats.vsi_id = hw->vsi_res->vsi_id;
1007
1008         args.v_op = VIRTCHNL_OP_GET_STATS;
1009         args.req_msg = (uint8_t *)&q_stats;
1010         args.req_msglen = sizeof(q_stats);
1011         args.rsp_msglen = sizeof(*pstats);
1012         args.rsp_msgbuf = (uint8_t *)pstats;
1013         args.rsp_buflen = sizeof(*pstats);
1014
1015         err = ice_dcf_execute_virtchnl_cmd(hw, &args);
1016         if (err) {
1017                 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS");
1018                 return err;
1019         }
1020
1021         return 0;
1022 }
1023
1024 int
1025 ice_dcf_add_del_all_mac_addr(struct ice_dcf_hw *hw, bool add)
1026 {
1027         struct virtchnl_ether_addr_list *list;
1028         struct rte_ether_addr *addr;
1029         struct dcf_virtchnl_cmd args;
1030         int len, err = 0;
1031
1032         len = sizeof(struct virtchnl_ether_addr_list);
1033         addr = hw->eth_dev->data->mac_addrs;
1034         len += sizeof(struct virtchnl_ether_addr);
1035
1036         list = rte_zmalloc(NULL, len, 0);
1037         if (!list) {
1038                 PMD_DRV_LOG(ERR, "fail to allocate memory");
1039                 return -ENOMEM;
1040         }
1041
1042         rte_memcpy(list->list[0].addr, addr->addr_bytes,
1043                         sizeof(addr->addr_bytes));
1044         PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x",
1045                             addr->addr_bytes[0], addr->addr_bytes[1],
1046                             addr->addr_bytes[2], addr->addr_bytes[3],
1047                             addr->addr_bytes[4], addr->addr_bytes[5]);
1048
1049         list->vsi_id = hw->vsi_res->vsi_id;
1050         list->num_elements = 1;
1051
1052         memset(&args, 0, sizeof(args));
1053         args.v_op = add ? VIRTCHNL_OP_ADD_ETH_ADDR :
1054                         VIRTCHNL_OP_DEL_ETH_ADDR;
1055         args.req_msg = (uint8_t *)list;
1056         args.req_msglen  = len;
1057         err = ice_dcf_execute_virtchnl_cmd(hw, &args);
1058         if (err)
1059                 PMD_DRV_LOG(ERR, "fail to execute command %s",
1060                             add ? "OP_ADD_ETHER_ADDRESS" :
1061                             "OP_DEL_ETHER_ADDRESS");
1062         rte_free(list);
1063         return err;
1064 }