net/i40e: fix Rx packet statistics
[dpdk.git] / drivers / net / ionic / ionic_main.c
1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
3  */
4
5 #include <stdbool.h>
6
7 #include <rte_memzone.h>
8
9 #include "ionic.h"
10 #include "ionic_ethdev.h"
11 #include "ionic_lif.h"
12
13 static const char *
14 ionic_error_to_str(enum ionic_status_code code)
15 {
16         switch (code) {
17         case IONIC_RC_SUCCESS:
18                 return "IONIC_RC_SUCCESS";
19         case IONIC_RC_EVERSION:
20                 return "IONIC_RC_EVERSION";
21         case IONIC_RC_EOPCODE:
22                 return "IONIC_RC_EOPCODE";
23         case IONIC_RC_EIO:
24                 return "IONIC_RC_EIO";
25         case IONIC_RC_EPERM:
26                 return "IONIC_RC_EPERM";
27         case IONIC_RC_EQID:
28                 return "IONIC_RC_EQID";
29         case IONIC_RC_EQTYPE:
30                 return "IONIC_RC_EQTYPE";
31         case IONIC_RC_ENOENT:
32                 return "IONIC_RC_ENOENT";
33         case IONIC_RC_EINTR:
34                 return "IONIC_RC_EINTR";
35         case IONIC_RC_EAGAIN:
36                 return "IONIC_RC_EAGAIN";
37         case IONIC_RC_ENOMEM:
38                 return "IONIC_RC_ENOMEM";
39         case IONIC_RC_EFAULT:
40                 return "IONIC_RC_EFAULT";
41         case IONIC_RC_EBUSY:
42                 return "IONIC_RC_EBUSY";
43         case IONIC_RC_EEXIST:
44                 return "IONIC_RC_EEXIST";
45         case IONIC_RC_EINVAL:
46                 return "IONIC_RC_EINVAL";
47         case IONIC_RC_ENOSPC:
48                 return "IONIC_RC_ENOSPC";
49         case IONIC_RC_ERANGE:
50                 return "IONIC_RC_ERANGE";
51         case IONIC_RC_BAD_ADDR:
52                 return "IONIC_RC_BAD_ADDR";
53         case IONIC_RC_DEV_CMD:
54                 return "IONIC_RC_DEV_CMD";
55         case IONIC_RC_ERROR:
56                 return "IONIC_RC_ERROR";
57         case IONIC_RC_ERDMA:
58                 return "IONIC_RC_ERDMA";
59         default:
60                 return "IONIC_RC_UNKNOWN";
61         }
62 }
63
64 const char *
65 ionic_opcode_to_str(enum ionic_cmd_opcode opcode)
66 {
67         switch (opcode) {
68         case IONIC_CMD_NOP:
69                 return "IONIC_CMD_NOP";
70         case IONIC_CMD_INIT:
71                 return "IONIC_CMD_INIT";
72         case IONIC_CMD_RESET:
73                 return "IONIC_CMD_RESET";
74         case IONIC_CMD_IDENTIFY:
75                 return "IONIC_CMD_IDENTIFY";
76         case IONIC_CMD_GETATTR:
77                 return "IONIC_CMD_GETATTR";
78         case IONIC_CMD_SETATTR:
79                 return "IONIC_CMD_SETATTR";
80         case IONIC_CMD_PORT_IDENTIFY:
81                 return "IONIC_CMD_PORT_IDENTIFY";
82         case IONIC_CMD_PORT_INIT:
83                 return "IONIC_CMD_PORT_INIT";
84         case IONIC_CMD_PORT_RESET:
85                 return "IONIC_CMD_PORT_RESET";
86         case IONIC_CMD_PORT_GETATTR:
87                 return "IONIC_CMD_PORT_GETATTR";
88         case IONIC_CMD_PORT_SETATTR:
89                 return "IONIC_CMD_PORT_SETATTR";
90         case IONIC_CMD_LIF_INIT:
91                 return "IONIC_CMD_LIF_INIT";
92         case IONIC_CMD_LIF_RESET:
93                 return "IONIC_CMD_LIF_RESET";
94         case IONIC_CMD_LIF_IDENTIFY:
95                 return "IONIC_CMD_LIF_IDENTIFY";
96         case IONIC_CMD_LIF_SETATTR:
97                 return "IONIC_CMD_LIF_SETATTR";
98         case IONIC_CMD_LIF_GETATTR:
99                 return "IONIC_CMD_LIF_GETATTR";
100         case IONIC_CMD_RX_MODE_SET:
101                 return "IONIC_CMD_RX_MODE_SET";
102         case IONIC_CMD_RX_FILTER_ADD:
103                 return "IONIC_CMD_RX_FILTER_ADD";
104         case IONIC_CMD_RX_FILTER_DEL:
105                 return "IONIC_CMD_RX_FILTER_DEL";
106         case IONIC_CMD_Q_INIT:
107                 return "IONIC_CMD_Q_INIT";
108         case IONIC_CMD_Q_CONTROL:
109                 return "IONIC_CMD_Q_CONTROL";
110         case IONIC_CMD_Q_IDENTIFY:
111                 return "IONIC_CMD_Q_IDENTIFY";
112         case IONIC_CMD_RDMA_RESET_LIF:
113                 return "IONIC_CMD_RDMA_RESET_LIF";
114         case IONIC_CMD_RDMA_CREATE_EQ:
115                 return "IONIC_CMD_RDMA_CREATE_EQ";
116         case IONIC_CMD_RDMA_CREATE_CQ:
117                 return "IONIC_CMD_RDMA_CREATE_CQ";
118         case IONIC_CMD_RDMA_CREATE_ADMINQ:
119                 return "IONIC_CMD_RDMA_CREATE_ADMINQ";
120         default:
121                 return "DEVCMD_UNKNOWN";
122         }
123 }
124
125 static int
126 ionic_adminq_check_err(struct ionic_admin_ctx *ctx, bool timeout)
127 {
128         const char *name;
129         const char *status;
130
131         name = ionic_opcode_to_str(ctx->cmd.cmd.opcode);
132
133         if (ctx->comp.comp.status || timeout) {
134                 status = ionic_error_to_str(ctx->comp.comp.status);
135                 IONIC_PRINT(ERR, "%s (%d) failed: %s (%d)",
136                         name,
137                         ctx->cmd.cmd.opcode,
138                         timeout ? "TIMEOUT" : status,
139                         timeout ? -1 : ctx->comp.comp.status);
140                 return -EIO;
141         }
142
143         IONIC_PRINT(DEBUG, "%s (%d) succeeded", name, ctx->cmd.cmd.opcode);
144
145         return 0;
146 }
147
148 static bool
149 ionic_adminq_service(struct ionic_cq *cq, uint16_t cq_desc_index,
150                 void *cb_arg __rte_unused)
151 {
152         struct ionic_admin_comp *cq_desc_base = cq->base;
153         struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
154         struct ionic_qcq *qcq = IONIC_CQ_TO_QCQ(cq);
155         struct ionic_queue *q = &qcq->q;
156         struct ionic_admin_ctx *ctx;
157         uint16_t curr_q_tail_idx;
158         uint16_t stop_index;
159         void **info;
160
161         if (!color_match(cq_desc->color, cq->done_color))
162                 return false;
163
164         stop_index = rte_le_to_cpu_16(cq_desc->comp_index);
165
166         do {
167                 info = IONIC_INFO_PTR(q, q->tail_idx);
168
169                 ctx = info[0];
170                 if (ctx) {
171                         memcpy(&ctx->comp, cq_desc, sizeof(*cq_desc));
172
173                         ctx->pending_work = false; /* done */
174                 }
175
176                 curr_q_tail_idx = q->tail_idx;
177                 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
178         } while (curr_q_tail_idx != stop_index);
179
180         return true;
181 }
182
183 /** ionic_adminq_post - Post an admin command.
184  * @lif:                Handle to lif.
185  * @cmd_ctx:            Api admin command context.
186  *
187  * Post the command to an admin queue in the ethernet driver.  If this command
188  * succeeds, then the command has been posted, but that does not indicate a
189  * completion.  If this command returns success, then the completion callback
190  * will eventually be called.
191  *
192  * Return: zero or negative error status.
193  */
194 static int
195 ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
196 {
197         struct ionic_queue *q = &lif->adminqcq->qcq.q;
198         struct ionic_admin_cmd *q_desc_base = q->base;
199         struct ionic_admin_cmd *q_desc;
200         void **info;
201         int err = 0;
202
203         rte_spinlock_lock(&lif->adminq_lock);
204
205         if (ionic_q_space_avail(q) < 1) {
206                 err = -ENOSPC;
207                 goto err_out;
208         }
209
210         q_desc = &q_desc_base[q->head_idx];
211
212         memcpy(q_desc, &ctx->cmd, sizeof(ctx->cmd));
213
214         info = IONIC_INFO_PTR(q, q->head_idx);
215         info[0] = ctx;
216
217         q->head_idx = Q_NEXT_TO_POST(q, 1);
218
219         /* Ring doorbell */
220         rte_wmb();
221         ionic_q_flush(q);
222
223 err_out:
224         rte_spinlock_unlock(&lif->adminq_lock);
225
226         return err;
227 }
228
229 static int
230 ionic_adminq_wait_for_completion(struct ionic_lif *lif,
231                 struct ionic_admin_ctx *ctx, unsigned long max_wait)
232 {
233         unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
234         unsigned long max_wait_usec = max_wait * 1000000L;
235         unsigned long elapsed_usec = 0;
236         int budget = 8;
237
238         while (ctx->pending_work && elapsed_usec < max_wait_usec) {
239                 /*
240                  * Locking here as adminq is served inline and could be
241                  * called from multiple places
242                  */
243                 rte_spinlock_lock(&lif->adminq_service_lock);
244
245                 ionic_qcq_service(&lif->adminqcq->qcq, budget,
246                                 ionic_adminq_service, NULL);
247
248                 rte_spinlock_unlock(&lif->adminq_service_lock);
249
250                 rte_delay_us_block(step_usec);
251                 elapsed_usec += step_usec;
252         }
253
254         return (!ctx->pending_work);
255 }
256
257 int
258 ionic_adminq_post_wait(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
259 {
260         bool done;
261         int err;
262
263         IONIC_PRINT(DEBUG, "Sending %s (%d) via the admin queue",
264                 ionic_opcode_to_str(ctx->cmd.cmd.opcode), ctx->cmd.cmd.opcode);
265
266         err = ionic_adminq_post(lif, ctx);
267         if (err) {
268                 IONIC_PRINT(ERR, "Failure posting %d to the admin queue (%d)",
269                         ctx->cmd.cmd.opcode, err);
270                 return err;
271         }
272
273         done = ionic_adminq_wait_for_completion(lif, ctx,
274                 IONIC_DEVCMD_TIMEOUT);
275
276         return ionic_adminq_check_err(ctx, !done /* timed out */);
277 }
278
279 static int
280 ionic_dev_cmd_wait(struct ionic_dev *idev, unsigned long max_wait)
281 {
282         unsigned long step_usec = IONIC_DEVCMD_CHECK_PERIOD_US;
283         unsigned long max_wait_usec = max_wait * 1000000L;
284         unsigned long elapsed_usec = 0;
285         int done;
286
287         /* Wait for dev cmd to complete.. but no more than max_wait sec */
288
289         do {
290                 done = ionic_dev_cmd_done(idev);
291                 if (done) {
292                         IONIC_PRINT(DEBUG, "DEVCMD %d done took %ld usecs",
293                                 ioread8(&idev->dev_cmd->cmd.cmd.opcode),
294                                 elapsed_usec);
295                         return 0;
296                 }
297
298                 rte_delay_us_block(step_usec);
299
300                 elapsed_usec += step_usec;
301         } while (elapsed_usec < max_wait_usec);
302
303         IONIC_PRINT(ERR, "DEVCMD %d timeout after %ld usecs",
304                 ioread8(&idev->dev_cmd->cmd.cmd.opcode),
305                 elapsed_usec);
306
307         return -ETIMEDOUT;
308 }
309
310 static int
311 ionic_dev_cmd_check_error(struct ionic_dev *idev)
312 {
313         uint8_t status;
314
315         status = ionic_dev_cmd_status(idev);
316         if (status == 0)
317                 return 0;
318
319         return -EIO;
320 }
321
322 int
323 ionic_dev_cmd_wait_check(struct ionic_dev *idev, unsigned long max_wait)
324 {
325         int err;
326
327         err = ionic_dev_cmd_wait(idev, max_wait);
328
329         if (!err)
330                 err = ionic_dev_cmd_check_error(idev);
331
332         IONIC_PRINT(DEBUG, "dev_cmd returned %d", err);
333         return err;
334 }
335
336 int
337 ionic_setup(struct ionic_adapter *adapter)
338 {
339         return ionic_dev_setup(adapter);
340 }
341
342 int
343 ionic_identify(struct ionic_adapter *adapter)
344 {
345         struct ionic_dev *idev = &adapter->idev;
346         struct ionic_identity *ident = &adapter->ident;
347         uint32_t drv_size = RTE_DIM(ident->drv.words);
348         uint32_t cmd_size = RTE_DIM(idev->dev_cmd->data);
349         uint32_t dev_size = RTE_DIM(ident->dev.words);
350         uint32_t i, nwords;
351         int err;
352
353         memset(ident, 0, sizeof(*ident));
354
355         ident->drv.os_type = IONIC_OS_TYPE_LINUX;
356         ident->drv.os_dist = 0;
357         snprintf(ident->drv.os_dist_str,
358                 sizeof(ident->drv.os_dist_str), "Unknown");
359         ident->drv.kernel_ver = 0;
360         snprintf(ident->drv.kernel_ver_str,
361                 sizeof(ident->drv.kernel_ver_str), "DPDK");
362         strncpy(ident->drv.driver_ver_str, IONIC_DRV_VERSION,
363                 sizeof(ident->drv.driver_ver_str) - 1);
364
365         nwords = RTE_MIN(drv_size, cmd_size);
366         for (i = 0; i < nwords; i++)
367                 iowrite32(ident->drv.words[i], &idev->dev_cmd->data[i]);
368
369         ionic_dev_cmd_identify(idev, IONIC_IDENTITY_VERSION_1);
370         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
371         if (!err) {
372                 nwords = RTE_MIN(dev_size, cmd_size);
373                 for (i = 0; i < nwords; i++)
374                         ident->dev.words[i] = ioread32(&idev->dev_cmd->data[i]);
375         }
376
377         return err;
378 }
379
380 int
381 ionic_init(struct ionic_adapter *adapter)
382 {
383         struct ionic_dev *idev = &adapter->idev;
384
385         ionic_dev_cmd_init(idev);
386         return ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
387 }
388
389 int
390 ionic_reset(struct ionic_adapter *adapter)
391 {
392         struct ionic_dev *idev = &adapter->idev;
393
394         ionic_dev_cmd_reset(idev);
395         return ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
396 }
397
398 int
399 ionic_port_identify(struct ionic_adapter *adapter)
400 {
401         struct ionic_dev *idev = &adapter->idev;
402         struct ionic_identity *ident = &adapter->ident;
403         uint32_t port_words = RTE_DIM(ident->port.words);
404         uint32_t cmd_words = RTE_DIM(idev->dev_cmd->data);
405         uint32_t i, nwords;
406         int err;
407
408         ionic_dev_cmd_port_identify(idev);
409         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
410         if (!err) {
411                 nwords = RTE_MIN(port_words, cmd_words);
412                 for (i = 0; i < nwords; i++)
413                         ident->port.words[i] =
414                                 ioread32(&idev->dev_cmd->data[i]);
415         }
416
417         IONIC_PRINT(INFO, "speed %d",
418                 rte_le_to_cpu_32(ident->port.config.speed));
419         IONIC_PRINT(INFO, "mtu %d",
420                 rte_le_to_cpu_32(ident->port.config.mtu));
421         IONIC_PRINT(INFO, "state %d", ident->port.config.state);
422         IONIC_PRINT(INFO, "an_enable %d", ident->port.config.an_enable);
423         IONIC_PRINT(INFO, "fec_type %d", ident->port.config.fec_type);
424         IONIC_PRINT(INFO, "pause_type %d", ident->port.config.pause_type);
425         IONIC_PRINT(INFO, "loopback_mode %d",
426                 ident->port.config.loopback_mode);
427
428         return err;
429 }
430
431 static const struct rte_memzone *
432 ionic_memzone_reserve(const char *name, uint32_t len, int socket_id)
433 {
434         const struct rte_memzone *mz;
435
436         mz = rte_memzone_lookup(name);
437         if (mz)
438                 return mz;
439
440         mz = rte_memzone_reserve_aligned(name, len, socket_id,
441                 RTE_MEMZONE_IOVA_CONTIG, IONIC_ALIGN);
442         return mz;
443 }
444
445 int
446 ionic_port_init(struct ionic_adapter *adapter)
447 {
448         struct ionic_dev *idev = &adapter->idev;
449         struct ionic_identity *ident = &adapter->ident;
450         char z_name[RTE_MEMZONE_NAMESIZE];
451         uint32_t config_words = RTE_DIM(ident->port.config.words);
452         uint32_t cmd_words = RTE_DIM(idev->dev_cmd->data);
453         uint32_t i, nwords;
454         int err;
455
456         if (idev->port_info)
457                 return 0;
458
459         idev->port_info_sz = RTE_ALIGN(sizeof(*idev->port_info),
460                         rte_mem_page_size());
461
462         snprintf(z_name, sizeof(z_name), "%s_port_%s_info",
463                 IONIC_DRV_NAME, adapter->name);
464
465         idev->port_info_z = ionic_memzone_reserve(z_name, idev->port_info_sz,
466                 SOCKET_ID_ANY);
467         if (!idev->port_info_z) {
468                 IONIC_PRINT(ERR, "Cannot reserve port info DMA memory");
469                 return -ENOMEM;
470         }
471
472         idev->port_info = idev->port_info_z->addr;
473         idev->port_info_pa = idev->port_info_z->iova;
474
475         nwords = RTE_MIN(config_words, cmd_words);
476
477         for (i = 0; i < nwords; i++)
478                 iowrite32(ident->port.config.words[i], &idev->dev_cmd->data[i]);
479
480         idev->port_info->config.state = IONIC_PORT_ADMIN_STATE_UP;
481         ionic_dev_cmd_port_init(idev);
482         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
483         if (err)
484                 IONIC_PRINT(ERR, "Failed to init port");
485
486         return err;
487 }
488
489 int
490 ionic_port_reset(struct ionic_adapter *adapter)
491 {
492         struct ionic_dev *idev = &adapter->idev;
493         int err;
494
495         if (!idev->port_info)
496                 return 0;
497
498         ionic_dev_cmd_port_reset(idev);
499         err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
500         if (err) {
501                 IONIC_PRINT(ERR, "Failed to reset port");
502                 return err;
503         }
504
505         idev->port_info = NULL;
506         idev->port_info_pa = 0;
507
508         return 0;
509 }