net/ice: fix GTPU down/uplink and extension conflict
[dpdk.git] / drivers / net / netvsc / hn_rndis.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright (c) 2009-2018 Microsoft Corp.
3  * Copyright (c) 2010-2012 Citrix Inc.
4  * Copyright (c) 2012 NetApp Inc.
5  * All rights reserved.
6  */
7
8 #include <stdint.h>
9 #include <string.h>
10 #include <stdio.h>
11 #include <errno.h>
12 #include <unistd.h>
13 #include <time.h>
14
15 #include <rte_ethdev_driver.h>
16 #include <rte_ethdev.h>
17 #include <rte_string_fns.h>
18 #include <rte_memzone.h>
19 #include <rte_malloc.h>
20 #include <rte_atomic.h>
21 #include <rte_alarm.h>
22 #include <rte_branch_prediction.h>
23 #include <rte_ether.h>
24 #include <rte_common.h>
25 #include <rte_errno.h>
26 #include <rte_cycles.h>
27 #include <rte_memory.h>
28 #include <rte_eal.h>
29 #include <rte_dev.h>
30 #include <rte_bus_vmbus.h>
31
32 #include "hn_logs.h"
33 #include "hn_var.h"
34 #include "hn_nvs.h"
35 #include "hn_rndis.h"
36 #include "ndis.h"
37
38 #define RNDIS_TIMEOUT_SEC 5
39 #define RNDIS_DELAY_MS    10
40
41 #define HN_RNDIS_XFER_SIZE              0x4000
42
43 #define HN_NDIS_TXCSUM_CAP_IP4          \
44         (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT)
45 #define HN_NDIS_TXCSUM_CAP_TCP4         \
46         (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT)
47 #define HN_NDIS_TXCSUM_CAP_TCP6         \
48         (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \
49          NDIS_TXCSUM_CAP_IP6EXT)
50 #define HN_NDIS_TXCSUM_CAP_UDP6         \
51         (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT)
52 #define HN_NDIS_LSOV2_CAP_IP6           \
53         (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT)
54
55 /* Get unique request id */
56 static inline uint32_t
57 hn_rndis_rid(struct hn_data *hv)
58 {
59         uint32_t rid;
60
61         do {
62                 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1);
63         } while (rid == 0);
64
65         return rid;
66 }
67
68 static void *hn_rndis_alloc(size_t size)
69 {
70         return rte_zmalloc("RNDIS", size, PAGE_SIZE);
71 }
72
73 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP
74 void hn_rndis_dump(const void *buf)
75 {
76         const union {
77                 struct rndis_msghdr hdr;
78                 struct rndis_packet_msg pkt;
79                 struct rndis_init_req init_request;
80                 struct rndis_init_comp init_complete;
81                 struct rndis_halt_req halt;
82                 struct rndis_query_req query_request;
83                 struct rndis_query_comp query_complete;
84                 struct rndis_set_req set_request;
85                 struct rndis_set_comp set_complete;
86                 struct rndis_reset_req reset_request;
87                 struct rndis_reset_comp reset_complete;
88                 struct rndis_keepalive_req keepalive_request;
89                 struct rndis_keepalive_comp keepalive_complete;
90                 struct rndis_status_msg indicate_status;
91         } *rndis_msg = buf;
92
93         switch (rndis_msg->hdr.type) {
94         case RNDIS_PACKET_MSG: {
95                 const struct rndis_pktinfo *ppi;
96                 unsigned int ppi_len;
97
98                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
99                             "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n",
100                             rndis_msg->pkt.len,
101                             rndis_msg->pkt.dataoffset,
102                             rndis_msg->pkt.datalen,
103                             rndis_msg->pkt.oobdataelements,
104                             rndis_msg->pkt.oobdataoffset,
105                             rndis_msg->pkt.oobdatalen,
106                             rndis_msg->pkt.pktinfooffset,
107                             rndis_msg->pkt.pktinfolen);
108
109                 ppi = (const struct rndis_pktinfo *)
110                         ((const char *)buf
111                          + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset));
112
113                 ppi_len = rndis_msg->pkt.pktinfolen;
114                 while (ppi_len > 0) {
115                         const void *ppi_data;
116
117                         ppi_data = ppi->data;
118
119                         rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
120                                 "    PPI (size %u, type %u, offs %u data %#x)\n",
121                                 ppi->size, ppi->type, ppi->offset,
122                                 *(const uint32_t *)ppi_data);
123                         if (ppi->size == 0)
124                                 break;
125                         ppi_len -= ppi->size;
126                         ppi = (const struct rndis_pktinfo *)
127                                 ((const char *)ppi + ppi->size);
128                 }
129                 break;
130         }
131         case RNDIS_INITIALIZE_MSG:
132                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
133                             "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n",
134                             rndis_msg->init_request.len,
135                             rndis_msg->init_request.rid,
136                             rndis_msg->init_request.ver_major,
137                             rndis_msg->init_request.ver_minor,
138                             rndis_msg->init_request.max_xfersz);
139                 break;
140
141         case RNDIS_INITIALIZE_CMPLT:
142                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
143                             "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, "
144                             "flags %d, max xfer %u, max pkts %u, aligned %u)\n",
145                             rndis_msg->init_complete.len,
146                             rndis_msg->init_complete.rid,
147                             rndis_msg->init_complete.status,
148                             rndis_msg->init_complete.ver_major,
149                             rndis_msg->init_complete.ver_minor,
150                             rndis_msg->init_complete.devflags,
151                             rndis_msg->init_complete.pktmaxsz,
152                             rndis_msg->init_complete.pktmaxcnt,
153                             rndis_msg->init_complete.align);
154                 break;
155
156         case RNDIS_HALT_MSG:
157                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
158                             "RNDIS_HALT (len %u id %#x)\n",
159                             rndis_msg->halt.len, rndis_msg->halt.rid);
160                 break;
161
162         case RNDIS_QUERY_MSG:
163                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
164                             "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n",
165                             rndis_msg->query_request.len,
166                             rndis_msg->query_request.rid,
167                             rndis_msg->query_request.oid,
168                             rndis_msg->query_request.infobuflen,
169                             rndis_msg->query_request.infobufoffset);
170                 break;
171
172         case RNDIS_QUERY_CMPLT:
173                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
174                             "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n",
175                             rndis_msg->query_complete.len,
176                             rndis_msg->query_complete.rid,
177                             rndis_msg->query_complete.status,
178                             rndis_msg->query_complete.infobuflen,
179                             rndis_msg->query_complete.infobufoffset);
180                 break;
181
182         case RNDIS_SET_MSG:
183                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
184                             "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n",
185                             rndis_msg->set_request.len,
186                             rndis_msg->set_request.rid,
187                             rndis_msg->set_request.oid,
188                             rndis_msg->set_request.infobuflen,
189                             rndis_msg->set_request.infobufoffset);
190                 break;
191
192         case RNDIS_SET_CMPLT:
193                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
194                             "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n",
195                             rndis_msg->set_complete.len,
196                             rndis_msg->set_complete.rid,
197                             rndis_msg->set_complete.status);
198                 break;
199
200         case RNDIS_INDICATE_STATUS_MSG:
201                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
202                             "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n",
203                             rndis_msg->indicate_status.len,
204                             rndis_msg->indicate_status.status,
205                             rndis_msg->indicate_status.stbuflen,
206                             rndis_msg->indicate_status.stbufoffset);
207                 break;
208
209         case RNDIS_RESET_MSG:
210                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
211                             "RNDIS_RESET (len %u, id %#x)\n",
212                             rndis_msg->reset_request.len,
213                             rndis_msg->reset_request.rid);
214                 break;
215
216         case RNDIS_RESET_CMPLT:
217                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
218                             "RNDIS_RESET_C (len %u, status %#x address %#x)\n",
219                             rndis_msg->reset_complete.len,
220                             rndis_msg->reset_complete.status,
221                             rndis_msg->reset_complete.adrreset);
222                 break;
223
224         case RNDIS_KEEPALIVE_MSG:
225                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
226                             "RNDIS_KEEPALIVE (len %u, id %#x)\n",
227                             rndis_msg->keepalive_request.len,
228                             rndis_msg->keepalive_request.rid);
229                 break;
230
231         case RNDIS_KEEPALIVE_CMPLT:
232                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
233                             "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n",
234                             rndis_msg->keepalive_complete.len,
235                             rndis_msg->keepalive_complete.rid,
236                             rndis_msg->keepalive_complete.status);
237                 break;
238
239         default:
240                 rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
241                             "RNDIS type %#x len %u\n",
242                             rndis_msg->hdr.type,
243                             rndis_msg->hdr.len);
244                 break;
245         }
246 }
247 #endif
248
249 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan,
250                                   const void *req, uint32_t reqlen)
251
252 {
253         struct hn_nvs_rndis nvs_rndis = {
254                 .type = NVS_TYPE_RNDIS,
255                 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL,
256                 .chim_idx = NVS_CHIM_IDX_INVALID,
257                 .chim_sz = 0
258         };
259         struct vmbus_gpa sg;
260         rte_iova_t addr;
261
262         addr = rte_malloc_virt2iova(req);
263         if (unlikely(addr == RTE_BAD_IOVA)) {
264                 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova");
265                 return -EINVAL;
266         }
267
268         if (unlikely(reqlen > PAGE_SIZE)) {
269                 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size",
270                             reqlen);
271                 return -EINVAL;
272         }
273
274         sg.page = addr / PAGE_SIZE;
275         sg.ofs  = addr & PAGE_MASK;
276         sg.len  = reqlen;
277
278         if (sg.ofs + reqlen >  PAGE_SIZE) {
279                 PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary");
280                 return -EINVAL;
281         }
282
283         hn_rndis_dump(req);
284
285         return hn_nvs_send_sglist(chan, &sg, 1,
286                                   &nvs_rndis, sizeof(nvs_rndis), 0U, NULL);
287 }
288
289 /*
290  * Alarm callback to process link changed notifications.
291  * Can not directly since link_status is discovered while reading ring
292  */
293 static void hn_rndis_link_alarm(void *arg)
294 {
295         _rte_eth_dev_callback_process(arg, RTE_ETH_EVENT_INTR_LSC, NULL);
296 }
297
298 void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg)
299 {
300         const struct rndis_status_msg *indicate = msg;
301
302         hn_rndis_dump(msg);
303
304         PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status);
305
306         switch (indicate->status) {
307         case RNDIS_STATUS_NETWORK_CHANGE:
308         case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG:
309                 /* ignore not in DPDK API */
310                 break;
311
312         case RNDIS_STATUS_LINK_SPEED_CHANGE:
313         case RNDIS_STATUS_MEDIA_CONNECT:
314         case RNDIS_STATUS_MEDIA_DISCONNECT:
315                 if (dev->data->dev_conf.intr_conf.lsc)
316                         rte_eal_alarm_set(10, hn_rndis_link_alarm, dev);
317                 break;
318         default:
319                 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x",
320                             indicate->status);
321         }
322 }
323
324 /* Callback from hn_process_events when response is visible */
325 void hn_rndis_receive_response(struct hn_data *hv,
326                                const void *data, uint32_t len)
327 {
328         const struct rndis_init_comp *hdr = data;
329
330         hn_rndis_dump(data);
331
332         if (len < sizeof(3 * sizeof(uint32_t))) {
333                 PMD_DRV_LOG(ERR,
334                             "missing RNDIS header %u", len);
335                 return;
336         }
337
338         if (len < hdr->len) {
339                 PMD_DRV_LOG(ERR,
340                             "truncated RNDIS response %u", len);
341                 return;
342         }
343
344         if  (len > sizeof(hv->rndis_resp)) {
345                 PMD_DRV_LOG(NOTICE,
346                             "RNDIS response exceeds buffer");
347                 len = sizeof(hv->rndis_resp);
348         }
349
350         if (hdr->rid == 0) {
351                 PMD_DRV_LOG(NOTICE,
352                             "RNDIS response id zero!");
353         }
354
355         memcpy(hv->rndis_resp, data, len);
356
357         /* make sure response copied before update */
358         rte_smp_wmb();
359
360         if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) {
361                 PMD_DRV_LOG(NOTICE,
362                             "received id %#x pending id %#x",
363                             hdr->rid, (uint32_t)hv->rndis_pending);
364         }
365 }
366
367 /* Do request/response transaction */
368 static int hn_rndis_exec1(struct hn_data *hv,
369                           const void *req, uint32_t reqlen,
370                           void *comp, uint32_t comp_len)
371 {
372         const struct rndis_halt_req *hdr = req;
373         uint32_t rid = hdr->rid;
374         struct vmbus_channel *chan = hn_primary_chan(hv);
375         int error;
376
377         if (comp_len > sizeof(hv->rndis_resp)) {
378                 PMD_DRV_LOG(ERR,
379                             "Expected completion size %u exceeds buffer %zu",
380                             comp_len, sizeof(hv->rndis_resp));
381                 return -EIO;
382         }
383
384         if (rid == 0) {
385                 PMD_DRV_LOG(ERR, "Invalid request id");
386                 return -EINVAL;
387         }
388
389         if (comp != NULL &&
390             rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) {
391                 PMD_DRV_LOG(ERR,
392                             "Request already pending");
393                 return -EBUSY;
394         }
395
396         error = hn_nvs_send_rndis_ctrl(chan, req, reqlen);
397         if (error) {
398                 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error);
399                 return error;
400         }
401
402         if (comp) {
403                 time_t start = time(NULL);
404
405                 /* Poll primary channel until response received */
406                 while (hv->rndis_pending == rid) {
407                         if (hv->closed)
408                                 return -ENETDOWN;
409
410                         if (time(NULL) - start > RNDIS_TIMEOUT_SEC) {
411                                 PMD_DRV_LOG(ERR,
412                                             "RNDIS response timed out");
413
414                                 rte_atomic32_cmpset(&hv->rndis_pending, rid, 0);
415                                 return -ETIMEDOUT;
416                         }
417
418                         if (rte_vmbus_chan_rx_empty(hv->primary->chan))
419                                 rte_delay_ms(RNDIS_DELAY_MS);
420
421                         hn_process_events(hv, 0, 1);
422                 }
423
424                 memcpy(comp, hv->rndis_resp, comp_len);
425         }
426
427         return 0;
428 }
429
430 /* Do transaction and validate response */
431 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid,
432                             const void *req, uint32_t reqlen,
433                             void *comp, uint32_t comp_len, uint32_t comp_type)
434 {
435         const struct rndis_comp_hdr *hdr = comp;
436         int ret;
437
438         memset(comp, 0, comp_len);
439
440         ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len);
441         if (ret < 0)
442                 return ret;
443         /*
444          * Check this RNDIS complete message.
445          */
446         if (unlikely(hdr->type != comp_type)) {
447                 PMD_DRV_LOG(ERR,
448                             "unexpected RNDIS response complete %#x expect %#x",
449                             hdr->type, comp_type);
450
451                 return -ENXIO;
452         }
453         if (unlikely(hdr->rid != rid)) {
454                 PMD_DRV_LOG(ERR,
455                             "RNDIS comp rid mismatch %#x, expect %#x",
456                             hdr->rid, rid);
457                 return -EINVAL;
458         }
459
460         /* All pass! */
461         return 0;
462 }
463
464 static int
465 hn_rndis_query(struct hn_data *hv, uint32_t oid,
466                const void *idata, uint32_t idlen,
467                void *odata, uint32_t odlen)
468 {
469         struct rndis_query_req *req;
470         struct rndis_query_comp *comp;
471         uint32_t reqlen, comp_len;
472         int error = -EIO;
473         unsigned int ofs;
474         uint32_t rid;
475
476         reqlen = sizeof(*req) + idlen;
477         req = hn_rndis_alloc(reqlen);
478         if (req == NULL)
479                 return -ENOMEM;
480
481         comp_len = sizeof(*comp) + odlen;
482         comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE);
483         if (!comp) {
484                 error = -ENOMEM;
485                 goto done;
486         }
487         comp->status = RNDIS_STATUS_PENDING;
488
489         rid = hn_rndis_rid(hv);
490
491         req->type = RNDIS_QUERY_MSG;
492         req->len = reqlen;
493         req->rid = rid;
494         req->oid = oid;
495         req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET;
496         req->infobuflen = idlen;
497
498         /* Input data immediately follows RNDIS query. */
499         memcpy(req + 1, idata, idlen);
500
501         error = hn_rndis_execute(hv, rid, req, reqlen,
502                                  comp, comp_len, RNDIS_QUERY_CMPLT);
503
504         if (error)
505                 goto done;
506
507         if (comp->status != RNDIS_STATUS_SUCCESS) {
508                 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x",
509                             oid, comp->status);
510                 error = -EINVAL;
511                 goto done;
512         }
513
514         if (comp->infobuflen == 0 || comp->infobufoffset == 0) {
515                 /* No output data! */
516                 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid);
517                 error = 0;
518                 goto done;
519         }
520
521         /*
522          * Check output data length and offset.
523          */
524         /* ofs is the offset from the beginning of comp. */
525         ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset);
526         if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) {
527                 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u",
528                             comp->infobufoffset, comp->infobuflen);
529                 error = -EINVAL;
530                 goto done;
531         }
532
533         /* Save output data. */
534         if (comp->infobuflen < odlen)
535                 odlen = comp->infobuflen;
536
537         /* ofs is the offset from the beginning of comp. */
538         memcpy(odata, (const char *)comp + ofs, odlen);
539
540         error = 0;
541 done:
542         rte_free(comp);
543         rte_free(req);
544         return error;
545 }
546
547 static int
548 hn_rndis_halt(struct hn_data *hv)
549 {
550         struct rndis_halt_req *halt;
551
552         halt = hn_rndis_alloc(sizeof(*halt));
553         if (halt == NULL)
554                 return -ENOMEM;
555
556         halt->type = RNDIS_HALT_MSG;
557         halt->len = sizeof(*halt);
558         halt->rid = hn_rndis_rid(hv);
559
560         /* No RNDIS completion; rely on NVS message send completion */
561         hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0);
562
563         rte_free(halt);
564
565         PMD_INIT_LOG(DEBUG, "RNDIS halt done");
566         return 0;
567 }
568
569 static int
570 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps)
571 {
572         struct ndis_offload in;
573         uint32_t caps_len, size;
574         int error;
575
576         memset(caps, 0, sizeof(*caps));
577         memset(&in, 0, sizeof(in));
578         in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD;
579
580         if (hv->ndis_ver >= NDIS_VERSION_6_30) {
581                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3;
582                 size = NDIS_OFFLOAD_SIZE;
583         } else if (hv->ndis_ver >= NDIS_VERSION_6_1) {
584                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2;
585                 size = NDIS_OFFLOAD_SIZE_6_1;
586         } else {
587                 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1;
588                 size = NDIS_OFFLOAD_SIZE_6_0;
589         }
590         in.ndis_hdr.ndis_size = size;
591
592         caps_len = NDIS_OFFLOAD_SIZE;
593         error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES,
594                                &in, size, caps, caps_len);
595         if (error)
596                 return error;
597
598         /* Preliminary verification. */
599         if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) {
600                 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x",
601                             caps->ndis_hdr.ndis_type);
602                 return -EINVAL;
603         }
604         if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) {
605                 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x",
606                             caps->ndis_hdr.ndis_rev);
607                 return -EINVAL;
608         }
609         if (caps->ndis_hdr.ndis_size > caps_len) {
610                 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u",
611                             caps->ndis_hdr.ndis_size, caps_len);
612                 return -EINVAL;
613         } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) {
614                 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u",
615                             caps->ndis_hdr.ndis_size);
616                 return -EINVAL;
617         }
618
619         return 0;
620 }
621
622 int
623 hn_rndis_query_rsscaps(struct hn_data *hv,
624                        unsigned int *rxr_cnt0)
625 {
626         struct ndis_rss_caps in, caps;
627         unsigned int indsz, rxr_cnt;
628         uint32_t caps_len;
629         int error;
630
631         *rxr_cnt0 = 0;
632
633         if (hv->ndis_ver < NDIS_VERSION_6_20) {
634                 PMD_DRV_LOG(DEBUG, "RSS not supported on this host");
635                 return -EOPNOTSUPP;
636         }
637
638         memset(&in, 0, sizeof(in));
639         in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS;
640         in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2;
641         in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE;
642
643         caps_len = NDIS_RSS_CAPS_SIZE;
644         error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES,
645                                &in, NDIS_RSS_CAPS_SIZE,
646                                &caps, caps_len);
647         if (error)
648                 return error;
649
650         PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x",
651                      caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps);
652         /*
653          * Preliminary verification.
654          */
655         if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) {
656                 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x",
657                             caps.ndis_hdr.ndis_type);
658                 return -EINVAL;
659         }
660         if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) {
661                 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x",
662                             caps.ndis_hdr.ndis_rev);
663                 return -EINVAL;
664         }
665         if (caps.ndis_hdr.ndis_size > caps_len) {
666                 PMD_DRV_LOG(ERR,
667                             "invalid NDIS objsize %u, data size %u",
668                             caps.ndis_hdr.ndis_size, caps_len);
669                 return -EINVAL;
670         } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) {
671                 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u",
672                             caps.ndis_hdr.ndis_size);
673                 return -EINVAL;
674         }
675
676         /*
677          * Save information for later RSS configuration.
678          */
679         if (caps.ndis_nrxr == 0) {
680                 PMD_DRV_LOG(ERR, "0 RX rings!?");
681                 return -EINVAL;
682         }
683         rxr_cnt = caps.ndis_nrxr;
684
685         if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE &&
686             caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) {
687                 if (caps.ndis_nind > NDIS_HASH_INDCNT) {
688                         PMD_DRV_LOG(ERR,
689                                     "too many RSS indirect table entries %u",
690                                     caps.ndis_nind);
691                         return -EOPNOTSUPP;
692                 }
693                 if (!rte_is_power_of_2(caps.ndis_nind)) {
694                         PMD_DRV_LOG(ERR,
695                                     "RSS indirect table size is not power-of-2 %u",
696                                     caps.ndis_nind);
697                 }
698
699                 indsz = caps.ndis_nind;
700         } else {
701                 indsz = NDIS_HASH_INDCNT;
702         }
703
704         if (indsz < rxr_cnt) {
705                 PMD_DRV_LOG(NOTICE,
706                             "# of RX rings (%d) > RSS indirect table size %d",
707                             rxr_cnt, indsz);
708                 rxr_cnt = indsz;
709         }
710
711         hv->rss_offloads = 0;
712         if (caps.ndis_caps & NDIS_RSS_CAP_IPV4)
713                 hv->rss_offloads |= ETH_RSS_IPV4
714                         | ETH_RSS_NONFRAG_IPV4_TCP
715                         | ETH_RSS_NONFRAG_IPV4_UDP;
716         if (caps.ndis_caps & NDIS_RSS_CAP_IPV6)
717                 hv->rss_offloads |= ETH_RSS_IPV6
718                         | ETH_RSS_NONFRAG_IPV6_TCP;
719         if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX)
720                 hv->rss_offloads |= ETH_RSS_IPV6_EX
721                         | ETH_RSS_IPV6_TCP_EX;
722
723         /* Commit! */
724         *rxr_cnt0 = rxr_cnt;
725
726         return 0;
727 }
728
729 static int
730 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen)
731 {
732         struct rndis_set_req *req;
733         struct rndis_set_comp comp;
734         uint32_t reqlen, comp_len;
735         uint32_t rid;
736         int error;
737
738         reqlen = sizeof(*req) + dlen;
739         req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE);
740         if (!req)
741                 return -ENOMEM;
742
743         rid = hn_rndis_rid(hv);
744         req->type = RNDIS_SET_MSG;
745         req->len = reqlen;
746         req->rid = rid;
747         req->oid = oid;
748         req->infobuflen = dlen;
749         req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET;
750
751         /* Data immediately follows RNDIS set. */
752         memcpy(req + 1, data, dlen);
753
754         comp_len = sizeof(comp);
755         error = hn_rndis_execute(hv, rid, req, reqlen,
756                                  &comp, comp_len,
757                                  RNDIS_SET_CMPLT);
758         if (error) {
759                 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed",
760                             oid);
761                 error = EIO;
762                 goto done;
763         }
764
765         if (comp.status != RNDIS_STATUS_SUCCESS) {
766                 PMD_DRV_LOG(ERR,
767                             "RNDIS set %#" PRIx32 " failed: status %#" PRIx32,
768                             oid, comp.status);
769                 error = EIO;
770                 goto done;
771         }
772
773 done:
774         rte_free(req);
775         return error;
776 }
777
778 int hn_rndis_conf_offload(struct hn_data *hv,
779                           uint64_t tx_offloads, uint64_t rx_offloads)
780 {
781         struct ndis_offload_params params;
782         struct ndis_offload hwcaps;
783         int error;
784
785         error = hn_rndis_query_hwcaps(hv, &hwcaps);
786         if (error) {
787                 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
788                 return error;
789         }
790
791         /* NOTE: 0 means "no change" */
792         memset(&params, 0, sizeof(params));
793
794         params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT;
795         if (hv->ndis_ver < NDIS_VERSION_6_30) {
796                 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2;
797                 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1;
798         } else {
799                 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3;
800                 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE;
801         }
802
803         if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) {
804                 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4)
805                         params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX;
806                 else
807                         goto unsupported;
808
809                 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6)
810                         params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX;
811                 else
812                         goto unsupported;
813         }
814
815         if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) {
816                 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4)
817                     == NDIS_RXCSUM_CAP_TCP4)
818                         params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX;
819                 else
820                         goto unsupported;
821
822                 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)
823                     == NDIS_RXCSUM_CAP_TCP6)
824                         params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX;
825                 else
826                         goto unsupported;
827         }
828
829         if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
830                 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4)
831                         params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX;
832                 else
833                         goto unsupported;
834
835                 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)
836                     == NDIS_TXCSUM_CAP_UDP6)
837                         params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX;
838                 else
839                         goto unsupported;
840         }
841
842         if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
843                 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4)
844                         params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX;
845                 else
846                         goto unsupported;
847
848                 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)
849                         params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX;
850                 else
851                         goto unsupported;
852         }
853
854         if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) {
855                 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4)
856                     == NDIS_TXCSUM_CAP_IP4)
857                         params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX;
858                 else
859                         goto unsupported;
860         }
861         if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) {
862                 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
863                         params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX;
864                 else
865                         goto unsupported;
866         }
867
868         if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) {
869                 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023)
870                         params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON;
871                 else
872                         goto unsupported;
873
874                 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
875                     == HN_NDIS_LSOV2_CAP_IP6)
876                         params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON;
877                 else
878                         goto unsupported;
879         }
880
881         error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, &params,
882                              params.ndis_hdr.ndis_size);
883         if (error) {
884                 PMD_DRV_LOG(ERR, "offload config failed");
885                 return error;
886         }
887
888         return 0;
889  unsupported:
890         PMD_DRV_LOG(NOTICE,
891                     "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version",
892                     tx_offloads, rx_offloads);
893         return -EINVAL;
894 }
895
896 int hn_rndis_get_offload(struct hn_data *hv,
897                          struct rte_eth_dev_info *dev_info)
898 {
899         struct ndis_offload hwcaps;
900         int error;
901
902         memset(&hwcaps, 0, sizeof(hwcaps));
903
904         error = hn_rndis_query_hwcaps(hv, &hwcaps);
905         if (error) {
906                 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
907                 return error;
908         }
909
910         dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
911                                     DEV_TX_OFFLOAD_VLAN_INSERT;
912
913         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4)
914             == HN_NDIS_TXCSUM_CAP_IP4)
915                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM;
916
917         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4)
918             == HN_NDIS_TXCSUM_CAP_TCP4 &&
919             (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6)
920             == HN_NDIS_TXCSUM_CAP_TCP6)
921                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM;
922
923         if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) &&
924             (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6))
925                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM;
926
927         if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) &&
928             (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
929             == HN_NDIS_LSOV2_CAP_IP6)
930                 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
931
932         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
933                                     DEV_RX_OFFLOAD_RSS_HASH;
934
935         if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
936                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM;
937
938         if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) &&
939             (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6))
940                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM;
941
942         if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) &&
943             (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6))
944                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM;
945
946         return 0;
947 }
948
949 uint32_t
950 hn_rndis_get_ptypes(struct hn_data *hv)
951 {
952         struct ndis_offload hwcaps;
953         uint32_t ptypes;
954         int error;
955
956         memset(&hwcaps, 0, sizeof(hwcaps));
957
958         error = hn_rndis_query_hwcaps(hv, &hwcaps);
959         if (error) {
960                 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
961                 return RTE_PTYPE_L2_ETHER;
962         }
963
964         ptypes = RTE_PTYPE_L2_ETHER;
965
966         if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
967                 ptypes |= RTE_PTYPE_L3_IPV4;
968
969         if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) ||
970             (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6))
971                 ptypes |= RTE_PTYPE_L4_TCP;
972
973         if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) ||
974             (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6))
975                 ptypes |= RTE_PTYPE_L4_UDP;
976
977         return ptypes;
978 }
979
980 int
981 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter)
982 {
983         int error;
984
985         error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER,
986                              &filter, sizeof(filter));
987         if (error) {
988                 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d",
989                             filter, error);
990         } else {
991                 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter);
992         }
993
994         return error;
995 }
996
997 int hn_rndis_conf_rss(struct hn_data *hv, uint32_t flags)
998 {
999         struct ndis_rssprm_toeplitz rssp;
1000         struct ndis_rss_params *prm = &rssp.rss_params;
1001         unsigned int i;
1002         int error;
1003
1004         memset(&rssp, 0, sizeof(rssp));
1005
1006         prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS;
1007         prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2;
1008         prm->ndis_hdr.ndis_size = sizeof(*prm);
1009         prm->ndis_flags = flags;
1010         prm->ndis_hash = hv->rss_hash;
1011         prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT;
1012         prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]);
1013         prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ;
1014         prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]);
1015
1016         for (i = 0; i < NDIS_HASH_INDCNT; i++)
1017                 rssp.rss_ind[i] = hv->rss_ind[i];
1018
1019         /* Set hask key values */
1020         memcpy(&rssp.rss_key, hv->rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ);
1021
1022         error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS,
1023                              &rssp, sizeof(rssp));
1024         if (error != 0) {
1025                 PMD_DRV_LOG(ERR,
1026                             "RSS config num queues=%u failed: %d",
1027                             hv->num_queues, error);
1028         }
1029         return error;
1030 }
1031
1032 static int hn_rndis_init(struct hn_data *hv)
1033 {
1034         struct rndis_init_req *req;
1035         struct rndis_init_comp comp;
1036         uint32_t comp_len, rid;
1037         int error;
1038
1039         req = hn_rndis_alloc(sizeof(*req));
1040         if (!req) {
1041                 PMD_DRV_LOG(ERR, "no memory for RNDIS init");
1042                 return -ENXIO;
1043         }
1044
1045         rid = hn_rndis_rid(hv);
1046         req->type = RNDIS_INITIALIZE_MSG;
1047         req->len = sizeof(*req);
1048         req->rid = rid;
1049         req->ver_major = RNDIS_VERSION_MAJOR;
1050         req->ver_minor = RNDIS_VERSION_MINOR;
1051         req->max_xfersz = HN_RNDIS_XFER_SIZE;
1052
1053         comp_len = RNDIS_INIT_COMP_SIZE_MIN;
1054         error = hn_rndis_execute(hv, rid, req, sizeof(*req),
1055                                  &comp, comp_len,
1056                                  RNDIS_INITIALIZE_CMPLT);
1057         if (error)
1058                 goto done;
1059
1060         if (comp.status != RNDIS_STATUS_SUCCESS) {
1061                 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x",
1062                             comp.status);
1063                 error = -EIO;
1064                 goto done;
1065         }
1066
1067         hv->rndis_agg_size = comp.pktmaxsz;
1068         hv->rndis_agg_pkts = comp.pktmaxcnt;
1069         hv->rndis_agg_align = 1U << comp.align;
1070
1071         if (hv->rndis_agg_align < sizeof(uint32_t)) {
1072                 /*
1073                  * The RNDIS packet message encap assumes that the RNDIS
1074                  * packet message is at least 4 bytes aligned.  Fix up the
1075                  * alignment here, if the remote side sets the alignment
1076                  * too low.
1077                  */
1078                 PMD_DRV_LOG(NOTICE,
1079                             "fixup RNDIS aggpkt align: %u -> %zu",
1080                             hv->rndis_agg_align, sizeof(uint32_t));
1081                 hv->rndis_agg_align = sizeof(uint32_t);
1082         }
1083
1084         PMD_INIT_LOG(INFO,
1085                      "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u",
1086                      comp.ver_major, comp.ver_minor,
1087                      hv->rndis_agg_size, hv->rndis_agg_pkts,
1088                      hv->rndis_agg_align);
1089         error = 0;
1090 done:
1091         rte_free(req);
1092         return error;
1093 }
1094
1095 int
1096 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr)
1097 {
1098         uint32_t eaddr_len;
1099         int error;
1100
1101         eaddr_len = RTE_ETHER_ADDR_LEN;
1102         error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0,
1103                                eaddr, eaddr_len);
1104         if (error)
1105                 return error;
1106
1107         PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x",
1108                     eaddr[0], eaddr[1], eaddr[2],
1109                     eaddr[3], eaddr[4], eaddr[5]);
1110         return 0;
1111 }
1112
1113 int
1114 hn_rndis_get_linkstatus(struct hn_data *hv)
1115 {
1116         return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0,
1117                               &hv->link_status, sizeof(uint32_t));
1118 }
1119
1120 int
1121 hn_rndis_get_linkspeed(struct hn_data *hv)
1122 {
1123         return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0,
1124                               &hv->link_speed, sizeof(uint32_t));
1125 }
1126
1127 int
1128 hn_rndis_attach(struct hn_data *hv)
1129 {
1130         /* Initialize RNDIS. */
1131         return hn_rndis_init(hv);
1132 }
1133
1134 void
1135 hn_rndis_detach(struct hn_data *hv)
1136 {
1137         struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id];
1138
1139         rte_eal_alarm_cancel(hn_rndis_link_alarm, dev);
1140
1141         /* Halt the RNDIS. */
1142         hn_rndis_halt(hv);
1143 }