net/sfc/base: fix warnings from VS2015 C compiler (C4100)
[dpdk.git] / drivers / net / sfc / base / ef10_rx.c
1 /*
2  * Copyright (c) 2012-2016 Solarflare Communications Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *
8  * 1. Redistributions of source code must retain the above copyright notice,
9  *    this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright notice,
11  *    this list of conditions and the following disclaimer in the documentation
12  *    and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are
27  * those of the authors and should not be interpreted as representing official
28  * policies, either expressed or implied, of the FreeBSD Project.
29  */
30
31 #include "efx.h"
32 #include "efx_impl.h"
33
34
35 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
36
37
38 static  __checkReturn   efx_rc_t
39 efx_mcdi_init_rxq(
40         __in            efx_nic_t *enp,
41         __in            uint32_t ndescs,
42         __in            uint32_t target_evq,
43         __in            uint32_t label,
44         __in            uint32_t instance,
45         __in            efsys_mem_t *esmp,
46         __in            boolean_t disable_scatter,
47         __in            uint32_t ps_bufsize)
48 {
49         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
50         efx_mcdi_req_t req;
51         uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN,
52                             MC_CMD_INIT_RXQ_EXT_OUT_LEN)];
53         int npages = EFX_RXQ_NBUFS(ndescs);
54         int i;
55         efx_qword_t *dma_addr;
56         uint64_t addr;
57         efx_rc_t rc;
58         uint32_t dma_mode;
59         boolean_t want_outer_classes;
60
61         EFSYS_ASSERT3U(ndescs, <=, EFX_RXQ_MAXNDESCS);
62
63         if (ps_bufsize > 0)
64                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
65         else
66                 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
67
68         if (encp->enc_tunnel_encapsulations_supported != 0) {
69                 /*
70                  * WANT_OUTER_CLASSES can only be specified on hardware which
71                  * supports tunnel encapsulation offloads, even though it is
72                  * effectively the behaviour the hardware gives.
73                  *
74                  * Also, on hardware which does support such offloads, older
75                  * firmware rejects the flag if the offloads are not supported
76                  * by the current firmware variant, which means this may fail if
77                  * the capabilities are not updated when the firmware variant
78                  * changes. This is not an issue on newer firmware, as it was
79                  * changed in bug 69842 (v6.4.2.1007) to permit this flag to be
80                  * specified on all firmware variants.
81                  */
82                 want_outer_classes = B_TRUE;
83         } else {
84                 want_outer_classes = B_FALSE;
85         }
86
87         (void) memset(payload, 0, sizeof (payload));
88         req.emr_cmd = MC_CMD_INIT_RXQ;
89         req.emr_in_buf = payload;
90         req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN;
91         req.emr_out_buf = payload;
92         req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN;
93
94         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs);
95         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
96         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
97         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
98         MCDI_IN_POPULATE_DWORD_9(req, INIT_RXQ_EXT_IN_FLAGS,
99             INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
100             INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
101             INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
102             INIT_RXQ_EXT_IN_CRC_MODE, 0,
103             INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
104             INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
105             INIT_RXQ_EXT_IN_DMA_MODE,
106             dma_mode,
107             INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize,
108             INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes);
109         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
110         MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
111
112         dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
113         addr = EFSYS_MEM_ADDR(esmp);
114
115         for (i = 0; i < npages; i++) {
116                 EFX_POPULATE_QWORD_2(*dma_addr,
117                     EFX_DWORD_1, (uint32_t)(addr >> 32),
118                     EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
119
120                 dma_addr++;
121                 addr += EFX_BUF_SIZE;
122         }
123
124         efx_mcdi_execute(enp, &req);
125
126         if (req.emr_rc != 0) {
127                 rc = req.emr_rc;
128                 goto fail1;
129         }
130
131         return (0);
132
133 fail1:
134         EFSYS_PROBE1(fail1, efx_rc_t, rc);
135
136         return (rc);
137 }
138
139 static  __checkReturn   efx_rc_t
140 efx_mcdi_fini_rxq(
141         __in            efx_nic_t *enp,
142         __in            uint32_t instance)
143 {
144         efx_mcdi_req_t req;
145         uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
146                             MC_CMD_FINI_RXQ_OUT_LEN)];
147         efx_rc_t rc;
148
149         (void) memset(payload, 0, sizeof (payload));
150         req.emr_cmd = MC_CMD_FINI_RXQ;
151         req.emr_in_buf = payload;
152         req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
153         req.emr_out_buf = payload;
154         req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
155
156         MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
157
158         efx_mcdi_execute_quiet(enp, &req);
159
160         if (req.emr_rc != 0) {
161                 rc = req.emr_rc;
162                 goto fail1;
163         }
164
165         return (0);
166
167 fail1:
168         /*
169          * EALREADY is not an error, but indicates that the MC has rebooted and
170          * that the RXQ has already been destroyed.
171          */
172         if (rc != EALREADY)
173                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
174
175         return (rc);
176 }
177
178 #if EFSYS_OPT_RX_SCALE
179 static  __checkReturn   efx_rc_t
180 efx_mcdi_rss_context_alloc(
181         __in            efx_nic_t *enp,
182         __in            efx_rx_scale_context_type_t type,
183         __in            uint32_t num_queues,
184         __out           uint32_t *rss_contextp)
185 {
186         efx_mcdi_req_t req;
187         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
188                             MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
189         uint32_t rss_context;
190         uint32_t context_type;
191         efx_rc_t rc;
192
193         if (num_queues > EFX_MAXRSS) {
194                 rc = EINVAL;
195                 goto fail1;
196         }
197
198         switch (type) {
199         case EFX_RX_SCALE_EXCLUSIVE:
200                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
201                 break;
202         case EFX_RX_SCALE_SHARED:
203                 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
204                 break;
205         default:
206                 rc = EINVAL;
207                 goto fail2;
208         }
209
210         (void) memset(payload, 0, sizeof (payload));
211         req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
212         req.emr_in_buf = payload;
213         req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
214         req.emr_out_buf = payload;
215         req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
216
217         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
218             EVB_PORT_ID_ASSIGNED);
219         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
220         /* NUM_QUEUES is only used to validate indirection table offsets */
221         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
222
223         efx_mcdi_execute(enp, &req);
224
225         if (req.emr_rc != 0) {
226                 rc = req.emr_rc;
227                 goto fail3;
228         }
229
230         if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
231                 rc = EMSGSIZE;
232                 goto fail4;
233         }
234
235         rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
236         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
237                 rc = ENOENT;
238                 goto fail5;
239         }
240
241         *rss_contextp = rss_context;
242
243         return (0);
244
245 fail5:
246         EFSYS_PROBE(fail5);
247 fail4:
248         EFSYS_PROBE(fail4);
249 fail3:
250         EFSYS_PROBE(fail3);
251 fail2:
252         EFSYS_PROBE(fail2);
253 fail1:
254         EFSYS_PROBE1(fail1, efx_rc_t, rc);
255
256         return (rc);
257 }
258 #endif /* EFSYS_OPT_RX_SCALE */
259
260 #if EFSYS_OPT_RX_SCALE
261 static                  efx_rc_t
262 efx_mcdi_rss_context_free(
263         __in            efx_nic_t *enp,
264         __in            uint32_t rss_context)
265 {
266         efx_mcdi_req_t req;
267         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
268                             MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
269         efx_rc_t rc;
270
271         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
272                 rc = EINVAL;
273                 goto fail1;
274         }
275
276         (void) memset(payload, 0, sizeof (payload));
277         req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
278         req.emr_in_buf = payload;
279         req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
280         req.emr_out_buf = payload;
281         req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
282
283         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
284
285         efx_mcdi_execute_quiet(enp, &req);
286
287         if (req.emr_rc != 0) {
288                 rc = req.emr_rc;
289                 goto fail2;
290         }
291
292         return (0);
293
294 fail2:
295         EFSYS_PROBE(fail2);
296 fail1:
297         EFSYS_PROBE1(fail1, efx_rc_t, rc);
298
299         return (rc);
300 }
301 #endif /* EFSYS_OPT_RX_SCALE */
302
303 #if EFSYS_OPT_RX_SCALE
304 static                  efx_rc_t
305 efx_mcdi_rss_context_set_flags(
306         __in            efx_nic_t *enp,
307         __in            uint32_t rss_context,
308         __in            efx_rx_hash_type_t type)
309 {
310         efx_mcdi_req_t req;
311         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
312                             MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
313         efx_rc_t rc;
314
315         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
316                 rc = EINVAL;
317                 goto fail1;
318         }
319
320         (void) memset(payload, 0, sizeof (payload));
321         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
322         req.emr_in_buf = payload;
323         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
324         req.emr_out_buf = payload;
325         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
326
327         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
328             rss_context);
329
330         MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
331             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
332             (type & EFX_RX_HASH_IPV4) ? 1 : 0,
333             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
334             (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0,
335             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
336             (type & EFX_RX_HASH_IPV6) ? 1 : 0,
337             RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
338             (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0);
339
340         efx_mcdi_execute(enp, &req);
341
342         if (req.emr_rc != 0) {
343                 rc = req.emr_rc;
344                 goto fail2;
345         }
346
347         return (0);
348
349 fail2:
350         EFSYS_PROBE(fail2);
351 fail1:
352         EFSYS_PROBE1(fail1, efx_rc_t, rc);
353
354         return (rc);
355 }
356 #endif /* EFSYS_OPT_RX_SCALE */
357
358 #if EFSYS_OPT_RX_SCALE
359 static                  efx_rc_t
360 efx_mcdi_rss_context_set_key(
361         __in            efx_nic_t *enp,
362         __in            uint32_t rss_context,
363         __in_ecount(n)  uint8_t *key,
364         __in            size_t n)
365 {
366         efx_mcdi_req_t req;
367         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
368                             MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
369         efx_rc_t rc;
370
371         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
372                 rc = EINVAL;
373                 goto fail1;
374         }
375
376         (void) memset(payload, 0, sizeof (payload));
377         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
378         req.emr_in_buf = payload;
379         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
380         req.emr_out_buf = payload;
381         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
382
383         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
384             rss_context);
385
386         EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
387         if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
388                 rc = EINVAL;
389                 goto fail2;
390         }
391
392         memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
393             key, n);
394
395         efx_mcdi_execute(enp, &req);
396
397         if (req.emr_rc != 0) {
398                 rc = req.emr_rc;
399                 goto fail3;
400         }
401
402         return (0);
403
404 fail3:
405         EFSYS_PROBE(fail3);
406 fail2:
407         EFSYS_PROBE(fail2);
408 fail1:
409         EFSYS_PROBE1(fail1, efx_rc_t, rc);
410
411         return (rc);
412 }
413 #endif /* EFSYS_OPT_RX_SCALE */
414
415 #if EFSYS_OPT_RX_SCALE
416 static                  efx_rc_t
417 efx_mcdi_rss_context_set_table(
418         __in            efx_nic_t *enp,
419         __in            uint32_t rss_context,
420         __in_ecount(n)  unsigned int *table,
421         __in            size_t n)
422 {
423         efx_mcdi_req_t req;
424         uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
425                             MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
426         uint8_t *req_table;
427         int i, rc;
428
429         if (rss_context == EF10_RSS_CONTEXT_INVALID) {
430                 rc = EINVAL;
431                 goto fail1;
432         }
433
434         (void) memset(payload, 0, sizeof (payload));
435         req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
436         req.emr_in_buf = payload;
437         req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
438         req.emr_out_buf = payload;
439         req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
440
441         MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
442             rss_context);
443
444         req_table =
445             MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
446
447         for (i = 0;
448             i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
449             i++) {
450                 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
451         }
452
453         efx_mcdi_execute(enp, &req);
454
455         if (req.emr_rc != 0) {
456                 rc = req.emr_rc;
457                 goto fail2;
458         }
459
460         return (0);
461
462 fail2:
463         EFSYS_PROBE(fail2);
464 fail1:
465         EFSYS_PROBE1(fail1, efx_rc_t, rc);
466
467         return (rc);
468 }
469 #endif /* EFSYS_OPT_RX_SCALE */
470
471
472         __checkReturn   efx_rc_t
473 ef10_rx_init(
474         __in            efx_nic_t *enp)
475 {
476 #if EFSYS_OPT_RX_SCALE
477
478         if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
479                 &enp->en_rss_context) == 0) {
480                 /*
481                  * Allocated an exclusive RSS context, which allows both the
482                  * indirection table and key to be modified.
483                  */
484                 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
485                 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
486         } else {
487                 /*
488                  * Failed to allocate an exclusive RSS context. Continue
489                  * operation without support for RSS. The pseudo-header in
490                  * received packets will not contain a Toeplitz hash value.
491                  */
492                 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
493                 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
494         }
495
496 #endif /* EFSYS_OPT_RX_SCALE */
497
498         return (0);
499 }
500
501 #if EFSYS_OPT_RX_SCATTER
502         __checkReturn   efx_rc_t
503 ef10_rx_scatter_enable(
504         __in            efx_nic_t *enp,
505         __in            unsigned int buf_size)
506 {
507         _NOTE(ARGUNUSED(enp, buf_size))
508         return (0);
509 }
510 #endif  /* EFSYS_OPT_RX_SCATTER */
511
512 #if EFSYS_OPT_RX_SCALE
513         __checkReturn   efx_rc_t
514 ef10_rx_scale_context_alloc(
515         __in            efx_nic_t *enp,
516         __in            efx_rx_scale_context_type_t type,
517         __in            uint32_t num_queues,
518         __out           uint32_t *rss_contextp)
519 {
520         efx_rc_t rc;
521
522         rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp);
523         if (rc != 0)
524                 goto fail1;
525
526         return (0);
527
528 fail1:
529         EFSYS_PROBE1(fail1, efx_rc_t, rc);
530         return (rc);
531 }
532 #endif /* EFSYS_OPT_RX_SCALE */
533
534 #if EFSYS_OPT_RX_SCALE
535         __checkReturn   efx_rc_t
536 ef10_rx_scale_context_free(
537         __in            efx_nic_t *enp,
538         __in            uint32_t rss_context)
539 {
540         efx_rc_t rc;
541
542         rc = efx_mcdi_rss_context_free(enp, rss_context);
543         if (rc != 0)
544                 goto fail1;
545
546         return (0);
547
548 fail1:
549         EFSYS_PROBE1(fail1, efx_rc_t, rc);
550         return (rc);
551 }
552 #endif /* EFSYS_OPT_RX_SCALE */
553
554 #if EFSYS_OPT_RX_SCALE
555         __checkReturn   efx_rc_t
556 ef10_rx_scale_mode_set(
557         __in            efx_nic_t *enp,
558         __in            uint32_t rss_context,
559         __in            efx_rx_hash_alg_t alg,
560         __in            efx_rx_hash_type_t type,
561         __in            boolean_t insert)
562 {
563         efx_rc_t rc;
564
565         EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
566         EFSYS_ASSERT3U(insert, ==, B_TRUE);
567
568         if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
569                 rc = EINVAL;
570                 goto fail1;
571         }
572
573         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
574                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
575                         rc = ENOTSUP;
576                         goto fail2;
577                 }
578                 rss_context = enp->en_rss_context;
579         }
580
581         if ((rc = efx_mcdi_rss_context_set_flags(enp,
582                     rss_context, type)) != 0)
583                 goto fail3;
584
585         return (0);
586
587 fail3:
588         EFSYS_PROBE(fail3);
589 fail2:
590         EFSYS_PROBE(fail2);
591 fail1:
592         EFSYS_PROBE1(fail1, efx_rc_t, rc);
593
594         return (rc);
595 }
596 #endif /* EFSYS_OPT_RX_SCALE */
597
598 #if EFSYS_OPT_RX_SCALE
599         __checkReturn   efx_rc_t
600 ef10_rx_scale_key_set(
601         __in            efx_nic_t *enp,
602         __in            uint32_t rss_context,
603         __in_ecount(n)  uint8_t *key,
604         __in            size_t n)
605 {
606         efx_rc_t rc;
607
608         EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE ==
609             MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
610
611         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
612                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
613                         rc = ENOTSUP;
614                         goto fail1;
615                 }
616                 rss_context = enp->en_rss_context;
617         }
618
619         if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0)
620                 goto fail2;
621
622         return (0);
623
624 fail2:
625         EFSYS_PROBE(fail2);
626 fail1:
627         EFSYS_PROBE1(fail1, efx_rc_t, rc);
628
629         return (rc);
630 }
631 #endif /* EFSYS_OPT_RX_SCALE */
632
633 #if EFSYS_OPT_RX_SCALE
634         __checkReturn   efx_rc_t
635 ef10_rx_scale_tbl_set(
636         __in            efx_nic_t *enp,
637         __in            uint32_t rss_context,
638         __in_ecount(n)  unsigned int *table,
639         __in            size_t n)
640 {
641         efx_rc_t rc;
642
643
644         if (rss_context == EFX_RSS_CONTEXT_DEFAULT) {
645                 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) {
646                         rc = ENOTSUP;
647                         goto fail1;
648                 }
649                 rss_context = enp->en_rss_context;
650         }
651
652         if ((rc = efx_mcdi_rss_context_set_table(enp,
653                     rss_context, table, n)) != 0)
654                 goto fail2;
655
656         return (0);
657
658 fail2:
659         EFSYS_PROBE(fail2);
660 fail1:
661         EFSYS_PROBE1(fail1, efx_rc_t, rc);
662
663         return (rc);
664 }
665 #endif /* EFSYS_OPT_RX_SCALE */
666
667
668 /*
669  * EF10 RX pseudo-header
670  * ---------------------
671  *
672  * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
673  *
674  *  +00: Toeplitz hash value.
675  *       (32bit little-endian)
676  *  +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
677  *       (16bit big-endian)
678  *  +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
679  *       (16bit big-endian)
680  *  +08: Packet Length. Zero if the RX datapath was in cut-through mode.
681  *       (16bit little-endian)
682  *  +10: MAC timestamp. Zero if timestamping is not enabled.
683  *       (32bit little-endian)
684  *
685  * See "The RX Pseudo-header" in SF-109306-TC.
686  */
687
688         __checkReturn   efx_rc_t
689 ef10_rx_prefix_pktlen(
690         __in            efx_nic_t *enp,
691         __in            uint8_t *buffer,
692         __out           uint16_t *lengthp)
693 {
694         _NOTE(ARGUNUSED(enp))
695
696         /*
697          * The RX pseudo-header contains the packet length, excluding the
698          * pseudo-header. If the hardware receive datapath was operating in
699          * cut-through mode then the length in the RX pseudo-header will be
700          * zero, and the packet length must be obtained from the DMA length
701          * reported in the RX event.
702          */
703         *lengthp = buffer[8] | (buffer[9] << 8);
704         return (0);
705 }
706
707 #if EFSYS_OPT_RX_SCALE
708         __checkReturn   uint32_t
709 ef10_rx_prefix_hash(
710         __in            efx_nic_t *enp,
711         __in            efx_rx_hash_alg_t func,
712         __in            uint8_t *buffer)
713 {
714         _NOTE(ARGUNUSED(enp))
715
716         switch (func) {
717         case EFX_RX_HASHALG_TOEPLITZ:
718                 return (buffer[0] |
719                     (buffer[1] << 8) |
720                     (buffer[2] << 16) |
721                     (buffer[3] << 24));
722
723         default:
724                 EFSYS_ASSERT(0);
725                 return (0);
726         }
727 }
728 #endif /* EFSYS_OPT_RX_SCALE */
729
730 #if EFSYS_OPT_RX_PACKED_STREAM
731 /*
732  * Fake length for RXQ descriptors in packed stream mode
733  * to make hardware happy
734  */
735 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
736 #endif
737
738                                 void
739 ef10_rx_qpost(
740         __in                    efx_rxq_t *erp,
741         __in_ecount(ndescs)     efsys_dma_addr_t *addrp,
742         __in                    size_t size,
743         __in                    unsigned int ndescs,
744         __in                    unsigned int completed,
745         __in                    unsigned int added)
746 {
747         efx_qword_t qword;
748         unsigned int i;
749         unsigned int offset;
750         unsigned int id;
751
752         _NOTE(ARGUNUSED(completed))
753
754 #if EFSYS_OPT_RX_PACKED_STREAM
755         /*
756          * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT
757          * and equal to 0 after applying mask. Hardware does not like it.
758          */
759         if (erp->er_ev_qstate->eers_rx_packed_stream)
760                 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE;
761 #endif
762
763         /* The client driver must not overfill the queue */
764         EFSYS_ASSERT3U(added - completed + ndescs, <=,
765             EFX_RXQ_LIMIT(erp->er_mask + 1));
766
767         id = added & (erp->er_mask);
768         for (i = 0; i < ndescs; i++) {
769                 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
770                     unsigned int, id, efsys_dma_addr_t, addrp[i],
771                     size_t, size);
772
773                 EFX_POPULATE_QWORD_3(qword,
774                     ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
775                     ESF_DZ_RX_KER_BUF_ADDR_DW0,
776                     (uint32_t)(addrp[i] & 0xffffffff),
777                     ESF_DZ_RX_KER_BUF_ADDR_DW1,
778                     (uint32_t)(addrp[i] >> 32));
779
780                 offset = id * sizeof (efx_qword_t);
781                 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
782
783                 id = (id + 1) & (erp->er_mask);
784         }
785 }
786
787                         void
788 ef10_rx_qpush(
789         __in    efx_rxq_t *erp,
790         __in    unsigned int added,
791         __inout unsigned int *pushedp)
792 {
793         efx_nic_t *enp = erp->er_enp;
794         unsigned int pushed = *pushedp;
795         uint32_t wptr;
796         efx_dword_t dword;
797
798         /* Hardware has alignment restriction for WPTR */
799         wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
800         if (pushed == wptr)
801                 return;
802
803         *pushedp = wptr;
804
805         /* Push the populated descriptors out */
806         wptr &= erp->er_mask;
807
808         EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
809
810         /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
811         EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
812             wptr, pushed & erp->er_mask);
813         EFSYS_PIO_WRITE_BARRIER();
814         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
815                             erp->er_index, &dword, B_FALSE);
816 }
817
818 #if EFSYS_OPT_RX_PACKED_STREAM
819
820                         void
821 ef10_rx_qpush_ps_credits(
822         __in            efx_rxq_t *erp)
823 {
824         efx_nic_t *enp = erp->er_enp;
825         efx_dword_t dword;
826         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
827         uint32_t credits;
828
829         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
830
831         if (rxq_state->eers_rx_packed_stream_credits == 0)
832                 return;
833
834         /*
835          * It is a bug if we think that FW has utilized more
836          * credits than it is allowed to have (maximum). However,
837          * make sure that we do not credit more than maximum anyway.
838          */
839         credits = MIN(rxq_state->eers_rx_packed_stream_credits,
840             EFX_RX_PACKED_STREAM_MAX_CREDITS);
841         EFX_POPULATE_DWORD_3(dword,
842             ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
843             ERF_DZ_RX_DESC_MAGIC_CMD,
844             ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
845             ERF_DZ_RX_DESC_MAGIC_DATA, credits);
846         EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
847             erp->er_index, &dword, B_FALSE);
848
849         rxq_state->eers_rx_packed_stream_credits = 0;
850 }
851
852 /*
853  * In accordance with SF-112241-TC the received data has the following layout:
854  *  - 8 byte pseudo-header which consist of:
855  *    - 4 byte little-endian timestamp
856  *    - 2 byte little-endian captured length in bytes
857  *    - 2 byte little-endian original packet length in bytes
858  *  - captured packet bytes
859  *  - optional padding to align to 64 bytes boundary
860  *  - 64 bytes scratch space for the host software
861  */
862         __checkReturn   uint8_t *
863 ef10_rx_qps_packet_info(
864         __in            efx_rxq_t *erp,
865         __in            uint8_t *buffer,
866         __in            uint32_t buffer_length,
867         __in            uint32_t current_offset,
868         __out           uint16_t *lengthp,
869         __out           uint32_t *next_offsetp,
870         __out           uint32_t *timestamp)
871 {
872         uint16_t buf_len;
873         uint8_t *pkt_start;
874         efx_qword_t *qwordp;
875         efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate;
876
877         EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
878
879         buffer += current_offset;
880         pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
881
882         qwordp = (efx_qword_t *)buffer;
883         *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
884         *lengthp   = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
885         buf_len    = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
886
887         buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
888                             EFX_RX_PACKED_STREAM_ALIGNMENT);
889         *next_offsetp =
890             current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
891
892         EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
893         EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
894
895         if ((*next_offsetp ^ current_offset) &
896             EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
897                 rxq_state->eers_rx_packed_stream_credits++;
898
899         return (pkt_start);
900 }
901
902
903 #endif
904
905         __checkReturn   efx_rc_t
906 ef10_rx_qflush(
907         __in    efx_rxq_t *erp)
908 {
909         efx_nic_t *enp = erp->er_enp;
910         efx_rc_t rc;
911
912         if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
913                 goto fail1;
914
915         return (0);
916
917 fail1:
918         /*
919          * EALREADY is not an error, but indicates that the MC has rebooted and
920          * that the RXQ has already been destroyed. Callers need to know that
921          * the RXQ flush has completed to avoid waiting until timeout for a
922          * flush done event that will not be delivered.
923          */
924         if (rc != EALREADY)
925                 EFSYS_PROBE1(fail1, efx_rc_t, rc);
926
927         return (rc);
928 }
929
930                 void
931 ef10_rx_qenable(
932         __in    efx_rxq_t *erp)
933 {
934         /* FIXME */
935         _NOTE(ARGUNUSED(erp))
936         /* FIXME */
937 }
938
939         __checkReturn   efx_rc_t
940 ef10_rx_qcreate(
941         __in            efx_nic_t *enp,
942         __in            unsigned int index,
943         __in            unsigned int label,
944         __in            efx_rxq_type_t type,
945         __in            efsys_mem_t *esmp,
946         __in            size_t ndescs,
947         __in            uint32_t id,
948         __in            efx_evq_t *eep,
949         __in            efx_rxq_t *erp)
950 {
951         efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
952         efx_rc_t rc;
953         boolean_t disable_scatter;
954         unsigned int ps_buf_size;
955
956         _NOTE(ARGUNUSED(id, erp))
957
958         EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
959         EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
960         EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
961
962         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
963         EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
964
965         if (!ISP2(ndescs) ||
966             (ndescs < EFX_RXQ_MINNDESCS) || (ndescs > EFX_RXQ_MAXNDESCS)) {
967                 rc = EINVAL;
968                 goto fail1;
969         }
970         if (index >= encp->enc_rxq_limit) {
971                 rc = EINVAL;
972                 goto fail2;
973         }
974
975         switch (type) {
976         case EFX_RXQ_TYPE_DEFAULT:
977         case EFX_RXQ_TYPE_SCATTER:
978                 ps_buf_size = 0;
979                 break;
980 #if EFSYS_OPT_RX_PACKED_STREAM
981         case EFX_RXQ_TYPE_PACKED_STREAM_1M:
982                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
983                 break;
984         case EFX_RXQ_TYPE_PACKED_STREAM_512K:
985                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
986                 break;
987         case EFX_RXQ_TYPE_PACKED_STREAM_256K:
988                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
989                 break;
990         case EFX_RXQ_TYPE_PACKED_STREAM_128K:
991                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
992                 break;
993         case EFX_RXQ_TYPE_PACKED_STREAM_64K:
994                 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
995                 break;
996 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
997         default:
998                 rc = ENOTSUP;
999                 goto fail3;
1000         }
1001
1002 #if EFSYS_OPT_RX_PACKED_STREAM
1003         if (ps_buf_size != 0) {
1004                 /* Check if datapath firmware supports packed stream mode */
1005                 if (encp->enc_rx_packed_stream_supported == B_FALSE) {
1006                         rc = ENOTSUP;
1007                         goto fail4;
1008                 }
1009                 /* Check if packed stream allows configurable buffer sizes */
1010                 if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) &&
1011                     (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
1012                         rc = ENOTSUP;
1013                         goto fail5;
1014                 }
1015         }
1016 #else /* EFSYS_OPT_RX_PACKED_STREAM */
1017         EFSYS_ASSERT(ps_buf_size == 0);
1018 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1019
1020         /* Scatter can only be disabled if the firmware supports doing so */
1021         if (type == EFX_RXQ_TYPE_SCATTER)
1022                 disable_scatter = B_FALSE;
1023         else
1024                 disable_scatter = encp->enc_rx_disable_scatter_supported;
1025
1026         if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep->ee_index, label, index,
1027                     esmp, disable_scatter, ps_buf_size)) != 0)
1028                 goto fail6;
1029
1030         erp->er_eep = eep;
1031         erp->er_label = label;
1032
1033         ef10_ev_rxlabel_init(eep, erp, label, type);
1034
1035         erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label];
1036
1037         return (0);
1038
1039 fail6:
1040         EFSYS_PROBE(fail6);
1041 #if EFSYS_OPT_RX_PACKED_STREAM
1042 fail5:
1043         EFSYS_PROBE(fail5);
1044 fail4:
1045         EFSYS_PROBE(fail4);
1046 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1047 fail3:
1048         EFSYS_PROBE(fail3);
1049 fail2:
1050         EFSYS_PROBE(fail2);
1051 fail1:
1052         EFSYS_PROBE1(fail1, efx_rc_t, rc);
1053
1054         return (rc);
1055 }
1056
1057                 void
1058 ef10_rx_qdestroy(
1059         __in    efx_rxq_t *erp)
1060 {
1061         efx_nic_t *enp = erp->er_enp;
1062         efx_evq_t *eep = erp->er_eep;
1063         unsigned int label = erp->er_label;
1064
1065         ef10_ev_rxlabel_fini(eep, label);
1066
1067         EFSYS_ASSERT(enp->en_rx_qcount != 0);
1068         --enp->en_rx_qcount;
1069
1070         EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1071 }
1072
1073                 void
1074 ef10_rx_fini(
1075         __in    efx_nic_t *enp)
1076 {
1077 #if EFSYS_OPT_RX_SCALE
1078         if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE)
1079                 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
1080         enp->en_rss_context = 0;
1081         enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE;
1082 #else
1083         _NOTE(ARGUNUSED(enp))
1084 #endif /* EFSYS_OPT_RX_SCALE */
1085 }
1086
1087 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */