net/netvsc: implement link state change callback
[dpdk.git] / drivers / net / netvsc / hn_rndis.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright (c) 2009-2018 Microsoft Corp.
3  * Copyright (c) 2010-2012 Citrix Inc.
4  * Copyright (c) 2012 NetApp Inc.
5  * All rights reserved.
6  */
7
8 #include <stdint.h>
9 #include <string.h>
10 #include <stdio.h>
11 #include <errno.h>
12 #include <unistd.h>
13
14 #include <rte_ethdev_driver.h>
15 #include <rte_ethdev.h>
16 #include <rte_string_fns.h>
17 #include <rte_memzone.h>
18 #include <rte_malloc.h>
19 #include <rte_atomic.h>
20 #include <rte_branch_prediction.h>
21 #include <rte_ether.h>
22 #include <rte_common.h>
23 #include <rte_errno.h>
24 #include <rte_cycles.h>
25 #include <rte_memory.h>
26 #include <rte_eal.h>
27 #include <rte_dev.h>
28 #include <rte_bus_vmbus.h>
29
30 #include "hn_logs.h"
31 #include "hn_var.h"
32 #include "hn_nvs.h"
33 #include "hn_rndis.h"
34 #include "ndis.h"
35
36 #define HN_RNDIS_XFER_SIZE              0x4000
37
38 #define HN_NDIS_TXCSUM_CAP_IP4          \
39         (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT)
40 #define HN_NDIS_TXCSUM_CAP_TCP4         \
41         (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT)
42 #define HN_NDIS_TXCSUM_CAP_TCP6         \
43         (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \
44          NDIS_TXCSUM_CAP_IP6EXT)
45 #define HN_NDIS_TXCSUM_CAP_UDP6         \
46         (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT)
47 #define HN_NDIS_LSOV2_CAP_IP6           \
48         (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT)
49
50 /* Get unique request id */
51 static inline uint32_t
52 hn_rndis_rid(struct hn_data *hv)
53 {
54         uint32_t rid;
55
56         do {
57                 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1);
58         } while (rid == 0);
59
60         return rid;
61 }
62
63 static void *hn_rndis_alloc(struct hn_data *hv, size_t size)
64 {
65         return rte_zmalloc_socket("RNDIS", size, PAGE_SIZE,
66                                  hv->vmbus->device.numa_node);
67 }
68
69 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP
70 void hn_rndis_dump(const void *buf)
71 {
72         const union {
73                 struct rndis_msghdr hdr;
74                 struct rndis_packet_msg pkt;
75                 struct rndis_init_req init_request;
76                 struct rndis_init_comp init_complete;
77                 struct rndis_halt_req halt;
78                 struct rndis_query_req query_request;
79                 struct rndis_query_comp query_complete;
80                 struct rndis_set_req set_request;
81                 struct rndis_set_comp set_complete;
82                 struct rndis_reset_req reset_request;
83                 struct rndis_reset_comp reset_complete;
84                 struct rndis_keepalive_req keepalive_request;
85                 struct rndis_keepalive_comp keepalive_complete;
86                 struct rndis_status_msg indicate_status;
87         } *rndis_msg = buf;
88
89         switch (rndis_msg->hdr.type) {
90         case RNDIS_PACKET_MSG: {
91                 const struct rndis_pktinfo *ppi;
92                 unsigned int ppi_len;
93
94                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
95                             "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n",
96                             rndis_msg->pkt.len,
97                             rndis_msg->pkt.dataoffset,
98                             rndis_msg->pkt.datalen,
99                             rndis_msg->pkt.oobdataelements,
100                             rndis_msg->pkt.oobdataoffset,
101                             rndis_msg->pkt.oobdatalen,
102                             rndis_msg->pkt.pktinfooffset,
103                             rndis_msg->pkt.pktinfolen);
104
105                 ppi = (const struct rndis_pktinfo *)
106                         ((const char *)buf
107                          + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset));
108
109                 ppi_len = rndis_msg->pkt.pktinfolen;
110                 while (ppi_len > 0) {
111                         const void *ppi_data;
112
113                         ppi_data = ppi->data;
114
115                         rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
116                                 "    PPI (size %u, type %u, offs %u data %#x)\n",
117                                 ppi->size, ppi->type, ppi->offset,
118                                 *(const uint32_t *)ppi_data);
119                         if (ppi->size == 0)
120                                 break;
121                         ppi_len -= ppi->size;
122                         ppi = (const struct rndis_pktinfo *)
123                                 ((const char *)ppi + ppi->size);
124                 }
125                 break;
126         }
127         case RNDIS_INITIALIZE_MSG:
128                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
129                             "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n",
130                             rndis_msg->init_request.len,
131                             rndis_msg->init_request.rid,
132                             rndis_msg->init_request.ver_major,
133                             rndis_msg->init_request.ver_minor,
134                             rndis_msg->init_request.max_xfersz);
135                 break;
136
137         case RNDIS_INITIALIZE_CMPLT:
138                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
139                             "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, "
140                             "flags %d, max xfer %u, max pkts %u, aligned %u)\n",
141                             rndis_msg->init_complete.len,
142                             rndis_msg->init_complete.rid,
143                             rndis_msg->init_complete.status,
144                             rndis_msg->init_complete.ver_major,
145                             rndis_msg->init_complete.ver_minor,
146                             rndis_msg->init_complete.devflags,
147                             rndis_msg->init_complete.pktmaxsz,
148                             rndis_msg->init_complete.pktmaxcnt,
149                             rndis_msg->init_complete.align);
150                 break;
151
152         case RNDIS_HALT_MSG:
153                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
154                             "RNDIS_HALT (len %u id %#x)\n",
155                             rndis_msg->halt.len, rndis_msg->halt.rid);
156                 break;
157
158         case RNDIS_QUERY_MSG:
159                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
160                             "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n",
161                             rndis_msg->query_request.len,
162                             rndis_msg->query_request.rid,
163                             rndis_msg->query_request.oid,
164                             rndis_msg->query_request.infobuflen,
165                             rndis_msg->query_request.infobufoffset);
166                 break;
167
168         case RNDIS_QUERY_CMPLT:
169                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
170                             "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n",
171                             rndis_msg->query_complete.len,
172                             rndis_msg->query_complete.rid,
173                             rndis_msg->query_complete.status,
174                             rndis_msg->query_complete.infobuflen,
175                             rndis_msg->query_complete.infobufoffset);
176                 break;
177
178         case RNDIS_SET_MSG:
179                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
180                             "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n",
181                             rndis_msg->set_request.len,
182                             rndis_msg->set_request.rid,
183                             rndis_msg->set_request.oid,
184                             rndis_msg->set_request.infobuflen,
185                             rndis_msg->set_request.infobufoffset);
186                 break;
187
188         case RNDIS_SET_CMPLT:
189                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
190                             "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n",
191                             rndis_msg->set_complete.len,
192                             rndis_msg->set_complete.rid,
193                             rndis_msg->set_complete.status);
194                 break;
195
196         case RNDIS_INDICATE_STATUS_MSG:
197                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
198                             "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n",
199                             rndis_msg->indicate_status.len,
200                             rndis_msg->indicate_status.status,
201                             rndis_msg->indicate_status.stbuflen,
202                             rndis_msg->indicate_status.stbufoffset);
203                 break;
204
205         case RNDIS_RESET_MSG:
206                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
207                             "RNDIS_RESET (len %u, id %#x)\n",
208                             rndis_msg->reset_request.len,
209                             rndis_msg->reset_request.rid);
210                 break;
211
212         case RNDIS_RESET_CMPLT:
213                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
214                             "RNDIS_RESET_C (len %u, status %#x address %#x)\n",
215                             rndis_msg->reset_complete.len,
216                             rndis_msg->reset_complete.status,
217                             rndis_msg->reset_complete.adrreset);
218                 break;
219
220         case RNDIS_KEEPALIVE_MSG:
221                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
222                             "RNDIS_KEEPALIVE (len %u, id %#x)\n",
223                             rndis_msg->keepalive_request.len,
224                             rndis_msg->keepalive_request.rid);
225                 break;
226
227         case RNDIS_KEEPALIVE_CMPLT:
228                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
229                             "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n",
230                             rndis_msg->keepalive_complete.len,
231                             rndis_msg->keepalive_complete.rid,
232                             rndis_msg->keepalive_complete.status);
233                 break;
234
235         default:
236                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
237                             "RNDIS type %#x len %u\n",
238                             rndis_msg->hdr.type,
239                             rndis_msg->hdr.len);
240                 break;
241         }
242 }
243 #endif
244
245 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan,
246                                   const void *req, uint32_t reqlen)
247
248 {
249         struct hn_nvs_rndis nvs_rndis = {
250                 .type = NVS_TYPE_RNDIS,
251                 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL,
252                 .chim_idx = NVS_CHIM_IDX_INVALID,
253                 .chim_sz = 0
254         };
255         struct vmbus_gpa sg;
256         rte_iova_t addr;
257
258         addr = rte_malloc_virt2iova(req);
259         if (unlikely(addr == RTE_BAD_IOVA)) {
260                 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova");
261                 return -EINVAL;
262         }
263
264         if (unlikely(reqlen > PAGE_SIZE)) {
265                 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size",
266                             reqlen);
267                 return -EINVAL;
268         }
269
270         sg.page = addr / PAGE_SIZE;
271         sg.ofs  = addr & PAGE_MASK;
272         sg.len  = reqlen;
273
274         if (sg.ofs + reqlen >  PAGE_SIZE) {
275                 PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary");
276                 return -EINVAL;
277         }
278
279         hn_rndis_dump(req);
280
281         return hn_nvs_send_sglist(chan, &sg, 1,
282                                   &nvs_rndis, sizeof(nvs_rndis), 0U, NULL);
283 }
284
285 void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg)
286 {
287         const struct rndis_status_msg *indicate = msg;
288
289         hn_rndis_dump(msg);
290
291         PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status);
292
293         switch (indicate->status) {
294         case RNDIS_STATUS_NETWORK_CHANGE:
295         case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG:
296                 /* ignore not in DPDK API */
297                 break;
298
299         case RNDIS_STATUS_LINK_SPEED_CHANGE:
300         case RNDIS_STATUS_MEDIA_CONNECT:
301         case RNDIS_STATUS_MEDIA_DISCONNECT:
302                 if (dev->data->dev_conf.intr_conf.lsc &&
303                     hn_dev_link_update(dev, 0) == 0)
304                         _rte_eth_dev_callback_process(dev,
305                                                       RTE_ETH_EVENT_INTR_LSC,
306                                                       NULL);
307                 break;
308         default:
309                 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x",
310                             indicate->status);
311         }
312 }
313
314 /* Callback from hn_process_events when response is visible */
315 void hn_rndis_receive_response(struct hn_data *hv,
316                                const void *data, uint32_t len)
317 {
318         const struct rndis_init_comp *hdr = data;
319
320         hn_rndis_dump(data);
321
322         if (len < sizeof(3 * sizeof(uint32_t))) {
323                 PMD_DRV_LOG(ERR,
324                             "missing RNDIS header %u", len);
325                 return;
326         }
327
328         if (len < hdr->len) {
329                 PMD_DRV_LOG(ERR,
330                             "truncated RNDIS response %u", len);
331                 return;
332         }
333
334         if  (len > sizeof(hv->rndis_resp)) {
335                 PMD_DRV_LOG(NOTICE,
336                             "RNDIS response exceeds buffer");
337                 len = sizeof(hv->rndis_resp);
338         }
339
340         if (hdr->rid == 0) {
341                 PMD_DRV_LOG(NOTICE,
342                             "RNDIS response id zero!");
343         }
344
345         memcpy(hv->rndis_resp, data, len);
346
347         /* make sure response copied before update */
348         rte_smp_wmb();
349
350         if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) {
351                 PMD_DRV_LOG(ERR,
352                             "received id %#x pending id %#x",
353                             hdr->rid, (uint32_t)hv->rndis_pending);
354         }
355 }
356
357 /* Do request/response transaction */
358 static int hn_rndis_exec1(struct hn_data *hv,
359                           const void *req, uint32_t reqlen,
360                           void *comp, uint32_t comp_len)
361 {
362         const struct rndis_halt_req *hdr = req;
363         uint32_t rid = hdr->rid;
364         struct vmbus_channel *chan = hn_primary_chan(hv);
365         int error;
366
367         if (comp_len > sizeof(hv->rndis_resp)) {
368                 PMD_DRV_LOG(ERR,
369                             "Expected completion size %u exceeds buffer %zu",
370                             comp_len, sizeof(hv->rndis_resp));
371                 return -EIO;
372         }
373
374         if (comp != NULL &&
375             rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) {
376                 PMD_DRV_LOG(ERR,
377                             "Request already pending");
378                 return -EBUSY;
379         }
380
381         error = hn_nvs_send_rndis_ctrl(chan, req, reqlen);
382         if (error) {
383                 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error);
384                 return error;
385         }
386
387         if (comp) {
388                 /* Poll primary channel until response received */
389                 while (hv->rndis_pending == rid)
390                         hn_process_events(hv, 0, 1);
391
392                 memcpy(comp, hv->rndis_resp, comp_len);
393         }
394
395         return 0;
396 }
397
398 /* Do transaction and validate response */
399 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid,
400                             const void *req, uint32_t reqlen,
401                             void *comp, uint32_t comp_len, uint32_t comp_type)
402 {
403         const struct rndis_comp_hdr *hdr = comp;
404         int ret;
405
406         memset(comp, 0, comp_len);
407
408         ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len);
409         if (ret < 0)
410                 return ret;
411         /*
412          * Check this RNDIS complete message.
413          */
414         if (unlikely(hdr->type != comp_type)) {
415                 PMD_DRV_LOG(ERR,
416                             "unexpected RNDIS response complete %#x expect %#x",
417                             hdr->type, comp_type);
418
419                 return -ENXIO;
420         }
421         if (unlikely(hdr->rid != rid)) {
422                 PMD_DRV_LOG(ERR,
423                             "RNDIS comp rid mismatch %#x, expect %#x",
424                             hdr->rid, rid);
425                 return -EINVAL;
426         }
427
428         /* All pass! */
429         return 0;
430 }
431
432 static int
433 hn_rndis_query(struct hn_data *hv, uint32_t oid,
434                const void *idata, uint32_t idlen,
435                void *odata, uint32_t odlen)
436 {
437         struct rndis_query_req *req;
438         struct rndis_query_comp *comp;
439         uint32_t reqlen, comp_len;
440         int error = -EIO;
441         unsigned int ofs;
442         uint32_t rid;
443
444         reqlen = sizeof(*req) + idlen;
445         req = hn_rndis_alloc(hv, reqlen);
446         if (req == NULL)
447                 return -ENOMEM;
448
449         comp_len = sizeof(*comp) + odlen;
450         comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE);
451         if (!comp) {
452                 error = -ENOMEM;
453                 goto done;
454         }
455         comp->status = RNDIS_STATUS_PENDING;
456
457         rid = hn_rndis_rid(hv);
458
459         req->type = RNDIS_QUERY_MSG;
460         req->len = reqlen;
461         req->rid = rid;
462         req->oid = oid;
463         req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET;
464         req->infobuflen = idlen;
465
466         /* Input data immediately follows RNDIS query. */
467         memcpy(req + 1, idata, idlen);
468
469         error = hn_rndis_execute(hv, rid, req, reqlen,
470                                  comp, comp_len, RNDIS_QUERY_CMPLT);
471
472         if (error)
473                 goto done;
474
475         if (comp->status != RNDIS_STATUS_SUCCESS) {
476                 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x",
477                             oid, comp->status);
478                 error = -EINVAL;
479                 goto done;
480         }
481
482         if (comp->infobuflen == 0 || comp->infobufoffset == 0) {
483                 /* No output data! */
484                 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid);
485                 error = 0;
486                 goto done;
487         }
488
489         /*
490          * Check output data length and offset.
491          */
492         /* ofs is the offset from the beginning of comp. */
493         ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset);
494         if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) {
495                 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u",
496                             comp->infobufoffset, comp->infobuflen);
497                 error = -EINVAL;
498                 goto done;
499         }
500
501         /* Save output data. */
502         if (comp->infobuflen < odlen)
503                 odlen = comp->infobuflen;
504
505         /* ofs is the offset from the beginning of comp. */
506         memcpy(odata, (const char *)comp + ofs, odlen);
507
508         error = 0;
509 done:
510         rte_free(comp);
511         rte_free(req);
512         return error;
513 }
514
515 static int
516 hn_rndis_halt(struct hn_data *hv)
517 {
518         struct rndis_halt_req *halt;
519
520         halt = hn_rndis_alloc(hv, sizeof(*halt));
521         if (halt == NULL)
522                 return -ENOMEM;
523
524         halt->type = RNDIS_HALT_MSG;
525         halt->len = sizeof(*halt);
526         halt->rid = hn_rndis_rid(hv);
527
528         /* No RNDIS completion; rely on NVS message send completion */
529         hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0);
530
531         rte_free(halt);
532
533         PMD_INIT_LOG(DEBUG, "RNDIS halt done");
534         return 0;
535 }
536
537 static int
538 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps)
539 {
540         struct ndis_offload in;
541         uint32_t caps_len, size;
542         int error;
543
544         memset(caps, 0, sizeof(*caps));
545         memset(&in, 0, sizeof(in));
546         in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD;
547
548         if (hv->ndis_ver >= NDIS_VERSION_6_30) {
549                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3;
550                 size = NDIS_OFFLOAD_SIZE;
551         } else if (hv->ndis_ver >= NDIS_VERSION_6_1) {
552                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2;
553                 size = NDIS_OFFLOAD_SIZE_6_1;
554         } else {
555                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1;
556                 size = NDIS_OFFLOAD_SIZE_6_0;
557         }
558         in.ndis_hdr.ndis_size = size;
559
560         caps_len = NDIS_OFFLOAD_SIZE;
561         error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES,
562                                &in, size, caps, caps_len);
563         if (error)
564                 return error;
565
566         /* Preliminary verification. */
567         if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) {
568                 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x",
569                             caps->ndis_hdr.ndis_type);
570                 return -EINVAL;
571         }
572         if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) {
573                 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x",
574                             caps->ndis_hdr.ndis_rev);
575                 return -EINVAL;
576         }
577         if (caps->ndis_hdr.ndis_size > caps_len) {
578                 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u",
579                             caps->ndis_hdr.ndis_size, caps_len);
580                 return -EINVAL;
581         } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) {
582                 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u",
583                             caps->ndis_hdr.ndis_size);
584                 return -EINVAL;
585         }
586
587         return 0;
588 }
589
590 int
591 hn_rndis_query_rsscaps(struct hn_data *hv,
592                        unsigned int *rxr_cnt0)
593 {
594         struct ndis_rss_caps in, caps;
595         unsigned int indsz, rxr_cnt;
596         uint32_t caps_len;
597         int error;
598
599         *rxr_cnt0 = 0;
600
601         if (hv->ndis_ver < NDIS_VERSION_6_20) {
602                 PMD_DRV_LOG(DEBUG, "RSS not supported on this host");
603                 return -EOPNOTSUPP;
604         }
605
606         memset(&in, 0, sizeof(in));
607         in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS;
608         in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2;
609         in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE;
610
611         caps_len = NDIS_RSS_CAPS_SIZE;
612         error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES,
613                                &in, NDIS_RSS_CAPS_SIZE,
614                                &caps, caps_len);
615         if (error)
616                 return error;
617
618         PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x",
619                      caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps);
620         /*
621          * Preliminary verification.
622          */
623         if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) {
624                 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x",
625                             caps.ndis_hdr.ndis_type);
626                 return -EINVAL;
627         }
628         if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) {
629                 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x",
630                             caps.ndis_hdr.ndis_rev);
631                 return -EINVAL;
632         }
633         if (caps.ndis_hdr.ndis_size > caps_len) {
634                 PMD_DRV_LOG(ERR,
635                             "invalid NDIS objsize %u, data size %u",
636                             caps.ndis_hdr.ndis_size, caps_len);
637                 return -EINVAL;
638         } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) {
639                 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u",
640                             caps.ndis_hdr.ndis_size);
641                 return -EINVAL;
642         }
643
644         /*
645          * Save information for later RSS configuration.
646          */
647         if (caps.ndis_nrxr == 0) {
648                 PMD_DRV_LOG(ERR, "0 RX rings!?");
649                 return -EINVAL;
650         }
651         rxr_cnt = caps.ndis_nrxr;
652
653         if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE &&
654             caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) {
655                 if (caps.ndis_nind > NDIS_HASH_INDCNT) {
656                         PMD_DRV_LOG(ERR,
657                                     "too many RSS indirect table entries %u",
658                                     caps.ndis_nind);
659                         return -EOPNOTSUPP;
660                 }
661                 if (!rte_is_power_of_2(caps.ndis_nind)) {
662                         PMD_DRV_LOG(ERR,
663                                     "RSS indirect table size is not power-of-2 %u",
664                                     caps.ndis_nind);
665                 }
666
667                 indsz = caps.ndis_nind;
668         } else {
669                 indsz = NDIS_HASH_INDCNT;
670         }
671
672         if (indsz < rxr_cnt) {
673                 PMD_DRV_LOG(NOTICE,
674                             "# of RX rings (%d) > RSS indirect table size %d",
675                             rxr_cnt, indsz);
676                 rxr_cnt = indsz;
677         }
678
679         hv->rss_offloads = 0;
680         if (caps.ndis_caps & NDIS_RSS_CAP_IPV4)
681                 hv->rss_offloads |= ETH_RSS_IPV4
682                         | ETH_RSS_NONFRAG_IPV4_TCP
683                         | ETH_RSS_NONFRAG_IPV4_UDP;
684         if (caps.ndis_caps & NDIS_RSS_CAP_IPV6)
685                 hv->rss_offloads |= ETH_RSS_IPV6
686                         | ETH_RSS_NONFRAG_IPV6_TCP;
687         if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX)
688                 hv->rss_offloads |= ETH_RSS_IPV6_EX
689                         | ETH_RSS_IPV6_TCP_EX;
690
691         /* Commit! */
692         *rxr_cnt0 = rxr_cnt;
693
694         return 0;
695 }
696
697 static int
698 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen)
699 {
700         struct rndis_set_req *req;
701         struct rndis_set_comp comp;
702         uint32_t reqlen, comp_len;
703         uint32_t rid;
704         int error;
705
706         reqlen = sizeof(*req) + dlen;
707         req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE);
708         if (!req)
709                 return -ENOMEM;
710
711         rid = hn_rndis_rid(hv);
712         req->type = RNDIS_SET_MSG;
713         req->len = reqlen;
714         req->rid = rid;
715         req->oid = oid;
716         req->infobuflen = dlen;
717         req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET;
718
719         /* Data immediately follows RNDIS set. */
720         memcpy(req + 1, data, dlen);
721
722         comp_len = sizeof(comp);
723         error = hn_rndis_execute(hv, rid, req, reqlen,
724                                  &comp, comp_len,
725                                  RNDIS_SET_CMPLT);
726         if (error) {
727                 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed",
728                             oid);
729                 error = EIO;
730                 goto done;
731         }
732
733         if (comp.status != RNDIS_STATUS_SUCCESS) {
734                 PMD_DRV_LOG(ERR,
735                             "RNDIS set %#" PRIx32 " failed: status %#" PRIx32,
736                             oid, comp.status);
737                 error = EIO;
738                 goto done;
739         }
740
741 done:
742         rte_free(req);
743         return error;
744 }
745
746 int hn_rndis_conf_offload(struct hn_data *hv,
747                           uint64_t tx_offloads, uint64_t rx_offloads)
748 {
749         struct ndis_offload_params params;
750         struct ndis_offload hwcaps;
751         int error;
752
753         error = hn_rndis_query_hwcaps(hv, &hwcaps);
754         if (error) {
755                 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
756                 return error;
757         }
758
759         /* NOTE: 0 means "no change" */
760         memset(&params, 0, sizeof(params));
761
762         params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT;
763         if (hv->ndis_ver < NDIS_VERSION_6_30) {
764                 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2;
765                 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1;
766         } else {
767                 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3;
768                 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE;
769         }
770
771         if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) {
772                 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4)
773                         params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX;
774                 else
775                         goto unsupported;
776
777                 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6)
778                         params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX;
779                 else
780                         goto unsupported;
781         }
782
783         if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) {
784                 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4)
785                     == NDIS_RXCSUM_CAP_TCP4)
786                         params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX;
787                 else
788                         goto unsupported;
789
790                 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)
791                     == NDIS_RXCSUM_CAP_TCP6)
792                         params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX;
793                 else
794                         goto unsupported;
795         }
796
797         if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
798                 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4)
799                         params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX;
800                 else
801                         goto unsupported;
802
803                 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)
804                     == NDIS_TXCSUM_CAP_UDP6)
805                         params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX;
806                 else
807                         goto unsupported;
808         }
809
810         if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
811                 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4)
812                         params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX;
813                 else
814                         goto unsupported;
815
816                 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)
817                         params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX;
818                 else
819                         goto unsupported;
820         }
821
822         if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) {
823                 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4)
824                     == NDIS_TXCSUM_CAP_IP4)
825                         params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX;
826                 else
827                         goto unsupported;
828         }
829         if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) {
830                 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
831                         params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX;
832                 else
833                         goto unsupported;
834         }
835
836         if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) {
837                 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023)
838                         params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON;
839                 else
840                         goto unsupported;
841
842                 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
843                     == HN_NDIS_LSOV2_CAP_IP6)
844                         params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON;
845                 else
846                         goto unsupported;
847         }
848
849         error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, &params,
850                              params.ndis_hdr.ndis_size);
851         if (error) {
852                 PMD_DRV_LOG(ERR, "offload config failed");
853                 return error;
854         }
855
856         return 0;
857  unsupported:
858         PMD_DRV_LOG(NOTICE,
859                     "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version",
860                     tx_offloads, rx_offloads);
861         return -EINVAL;
862 }
863
864 int hn_rndis_get_offload(struct hn_data *hv,
865                          struct rte_eth_dev_info *dev_info)
866 {
867         struct ndis_offload hwcaps;
868         int error;
869
870         memset(&hwcaps, 0, sizeof(hwcaps));
871
872         error = hn_rndis_query_hwcaps(hv, &hwcaps);
873         if (error) {
874                 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
875                 return error;
876         }
877
878         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
879                                     DEV_TX_OFFLOAD_VLAN_INSERT;
880
881         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4)
882             == HN_NDIS_TXCSUM_CAP_IP4)
883                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM;
884
885         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4)
886             == HN_NDIS_TXCSUM_CAP_TCP4 &&
887             (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6)
888             == HN_NDIS_TXCSUM_CAP_TCP6)
889                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM;
890
891         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) &&
892             (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6))
893                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM;
894
895         if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) &&
896             (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
897             == HN_NDIS_LSOV2_CAP_IP6)
898                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
899
900         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP;
901
902         if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
903                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM;
904
905         if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) &&
906             (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6))
907                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM;
908
909         if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) &&
910             (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6))
911                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM;
912
913         return 0;
914 }
915
916 int
917 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter)
918 {
919         int error;
920
921         error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER,
922                              &filter, sizeof(filter));
923         if (error) {
924                 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d",
925                             filter, error);
926         } else {
927                 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter);
928         }
929
930         return error;
931 }
932
933 /* The default RSS key.
934  * This value is the same as MLX5 so that flows will be
935  * received on same path for both VF ans synthetic NIC.
936  */
937 static const uint8_t rss_default_key[NDIS_HASH_KEYSIZE_TOEPLITZ] = {
938         0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7,
939         0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94,
940         0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1,
941         0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59,
942         0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a,
943 };
944
945 int hn_rndis_conf_rss(struct hn_data *hv,
946                       const struct rte_eth_rss_conf *rss_conf)
947 {
948         struct ndis_rssprm_toeplitz rssp;
949         struct ndis_rss_params *prm = &rssp.rss_params;
950         const uint8_t *rss_key = rss_conf->rss_key ? : rss_default_key;
951         uint32_t rss_hash;
952         unsigned int i;
953         int error;
954
955         PMD_INIT_FUNC_TRACE();
956
957         memset(&rssp, 0, sizeof(rssp));
958
959         prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS;
960         prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2;
961         prm->ndis_hdr.ndis_size = sizeof(*prm);
962         prm->ndis_flags = 0;
963
964         rss_hash = NDIS_HASH_FUNCTION_TOEPLITZ;
965         if (rss_conf->rss_hf & ETH_RSS_IPV4)
966                 rss_hash |= NDIS_HASH_IPV4;
967         if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
968                 rss_hash |= NDIS_HASH_TCP_IPV4;
969         if (rss_conf->rss_hf & ETH_RSS_IPV6)
970                 rss_hash |=  NDIS_HASH_IPV6;
971         if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
972                 rss_hash |= NDIS_HASH_TCP_IPV6;
973
974         prm->ndis_hash = rss_hash;
975         prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT;
976         prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]);
977         prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ;
978         prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]);
979
980         for (i = 0; i < NDIS_HASH_INDCNT; i++)
981                 rssp.rss_ind[i] = i % hv->num_queues;
982
983         /* Set hask key values */
984         memcpy(&rssp.rss_key, rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ);
985
986         error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS,
987                              &rssp, sizeof(rssp));
988         if (error) {
989                 PMD_DRV_LOG(ERR,
990                             "RSS config num queues=%u failed: %d",
991                             hv->num_queues, error);
992         }
993         return error;
994 }
995
996 static int hn_rndis_init(struct hn_data *hv)
997 {
998         struct rndis_init_req *req;
999         struct rndis_init_comp comp;
1000         uint32_t comp_len, rid;
1001         int error;
1002
1003         req = hn_rndis_alloc(hv, sizeof(*req));
1004         if (!req) {
1005                 PMD_DRV_LOG(ERR, "no memory for RNDIS init");
1006                 return -ENXIO;
1007         }
1008
1009         rid = hn_rndis_rid(hv);
1010         req->type = RNDIS_INITIALIZE_MSG;
1011         req->len = sizeof(*req);
1012         req->rid = rid;
1013         req->ver_major = RNDIS_VERSION_MAJOR;
1014         req->ver_minor = RNDIS_VERSION_MINOR;
1015         req->max_xfersz = HN_RNDIS_XFER_SIZE;
1016
1017         comp_len = RNDIS_INIT_COMP_SIZE_MIN;
1018         error = hn_rndis_execute(hv, rid, req, sizeof(*req),
1019                                  &comp, comp_len,
1020                                  RNDIS_INITIALIZE_CMPLT);
1021         if (error)
1022                 goto done;
1023
1024         if (comp.status != RNDIS_STATUS_SUCCESS) {
1025                 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x",
1026                             comp.status);
1027                 error = -EIO;
1028                 goto done;
1029         }
1030
1031         hv->rndis_agg_size = comp.pktmaxsz;
1032         hv->rndis_agg_pkts = comp.pktmaxcnt;
1033         hv->rndis_agg_align = 1U << comp.align;
1034
1035         if (hv->rndis_agg_align < sizeof(uint32_t)) {
1036                 /*
1037                  * The RNDIS packet message encap assumes that the RNDIS
1038                  * packet message is at least 4 bytes aligned.  Fix up the
1039                  * alignment here, if the remote side sets the alignment
1040                  * too low.
1041                  */
1042                 PMD_DRV_LOG(NOTICE,
1043                             "fixup RNDIS aggpkt align: %u -> %zu",
1044                             hv->rndis_agg_align, sizeof(uint32_t));
1045                 hv->rndis_agg_align = sizeof(uint32_t);
1046         }
1047
1048         PMD_INIT_LOG(INFO,
1049                      "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u",
1050                      comp.ver_major, comp.ver_minor,
1051                      hv->rndis_agg_size, hv->rndis_agg_pkts,
1052                      hv->rndis_agg_align);
1053         error = 0;
1054 done:
1055         rte_free(req);
1056         return error;
1057 }
1058
1059 int
1060 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr)
1061 {
1062         uint32_t eaddr_len;
1063         int error;
1064
1065         eaddr_len = ETHER_ADDR_LEN;
1066         error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0,
1067                                eaddr, eaddr_len);
1068         if (error)
1069                 return error;
1070
1071         PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x",
1072                     eaddr[0], eaddr[1], eaddr[2],
1073                     eaddr[3], eaddr[4], eaddr[5]);
1074         return 0;
1075 }
1076
1077 int
1078 hn_rndis_get_linkstatus(struct hn_data *hv)
1079 {
1080         return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0,
1081                               &hv->link_status, sizeof(uint32_t));
1082 }
1083
1084 int
1085 hn_rndis_get_linkspeed(struct hn_data *hv)
1086 {
1087         return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0,
1088                               &hv->link_speed, sizeof(uint32_t));
1089 }
1090
1091 int
1092 hn_rndis_attach(struct hn_data *hv)
1093 {
1094         /* Initialize RNDIS. */
1095         return hn_rndis_init(hv);
1096 }
1097
1098 void
1099 hn_rndis_detach(struct hn_data *hv)
1100 {
1101         /* Halt the RNDIS. */
1102         hn_rndis_halt(hv);
1103 }