drivers/net/netvsc/hn_nvs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2018 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  */
   7
   8 /*
   9  * Network Virtualization Service.
  10  */
  11
  12
  13 #include <stdint.h>
  14 #include <string.h>
  15 #include <stdio.h>
  16 #include <errno.h>
  17 #include <unistd.h>
  18
  19 #include <rte_ethdev.h>
  20 #include <rte_string_fns.h>
  21 #include <rte_memzone.h>
  22 #include <rte_malloc.h>
  23 #include <rte_atomic.h>
  24 #include <rte_branch_prediction.h>
  25 #include <rte_ether.h>
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_cycles.h>
  29 #include <rte_memory.h>
  30 #include <rte_eal.h>
  31 #include <rte_dev.h>
  32 #include <rte_bus_vmbus.h>
  33
  34 #include "hn_logs.h"
  35 #include "hn_var.h"
  36 #include "hn_nvs.h"
  37
  38 static const uint32_t hn_nvs_version[] = {
  39         NVS_VERSION_61,
  40         NVS_VERSION_6,
  41         NVS_VERSION_5,
  42         NVS_VERSION_4,
  43         NVS_VERSION_2,
  44         NVS_VERSION_1
  45 };
  46
  47 static int hn_nvs_req_send(struct hn_data *hv,
  48                            void *req, uint32_t reqlen)
  49 {
  50         return rte_vmbus_chan_send(hn_primary_chan(hv),
  51                                    VMBUS_CHANPKT_TYPE_INBAND,
  52                                    req, reqlen, 0,
  53                                    VMBUS_CHANPKT_FLAG_NONE, NULL);
  54 }
  55
  56 static int
  57 __hn_nvs_execute(struct hn_data *hv,
  58                void *req, uint32_t reqlen,
  59                void *resp, uint32_t resplen,
  60                uint32_t type)
  61 {
  62         struct vmbus_channel *chan = hn_primary_chan(hv);
  63         char buffer[NVS_RESPSIZE_MAX];
  64         const struct hn_nvs_hdr *hdr;
  65         uint64_t xactid;
  66         uint32_t len;
  67         int ret;
  68
  69         /* Send request to ring buffer */
  70         ret = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND,
  71                                   req, reqlen, 0,
  72                                   VMBUS_CHANPKT_FLAG_RC, NULL);
  73
  74         if (ret) {
  75                 PMD_DRV_LOG(ERR, "send request failed: %d", ret);
  76                 return ret;
  77         }
  78
  79  retry:
  80         len = sizeof(buffer);
  81         ret = rte_vmbus_chan_recv(chan, buffer, &len, &xactid);
  82         if (ret == -EAGAIN) {
  83                 rte_delay_us(HN_CHAN_INTERVAL_US);
  84                 goto retry;
  85         }
  86
  87         if (ret < 0) {
  88                 PMD_DRV_LOG(ERR, "recv response failed: %d", ret);
  89                 return ret;
  90         }
  91
  92         if (len < sizeof(*hdr)) {
  93                 PMD_DRV_LOG(ERR, "response missing NVS header");
  94                 return -EINVAL;
  95         }
  96
  97         hdr = (struct hn_nvs_hdr *)buffer;
  98
  99         /* Silently drop received packets while waiting for response */
 100         if (hdr->type == NVS_TYPE_RNDIS) {
 101                 hn_nvs_ack_rxbuf(chan, xactid);
 102                 goto retry;
 103         }
 104
 105         if (hdr->type != type) {
 106                 PMD_DRV_LOG(ERR, "unexpected NVS resp %#x, expect %#x",
 107                             hdr->type, type);
 108                 return -EINVAL;
 109         }
 110
 111         if (len < resplen) {
 112                 PMD_DRV_LOG(ERR,
 113                             "invalid NVS resp len %u (expect %u)",
 114                             len, resplen);
 115                 return -EINVAL;
 116         }
 117
 118         memcpy(resp, buffer, resplen);
 119
 120         /* All pass! */
 121         return 0;
 122 }
 123
 124
 125 /*
 126  * Execute one control command and get the response.
 127  * Only one command can be active on a channel at once
 128  * Unlike BSD, DPDK does not have an interrupt context
 129  * so the polling is required to wait for response.
 130  */
 131 static int
 132 hn_nvs_execute(struct hn_data *hv,
 133                void *req, uint32_t reqlen,
 134                void *resp, uint32_t resplen,
 135                uint32_t type)
 136 {
 137         struct hn_rx_queue *rxq = hv->primary;
 138         int ret;
 139
 140         rte_spinlock_lock(&rxq->ring_lock);
 141         ret = __hn_nvs_execute(hv, req, reqlen, resp, resplen, type);
 142         rte_spinlock_unlock(&rxq->ring_lock);
 143
 144         return ret;
 145 }
 146
 147 static int
 148 hn_nvs_doinit(struct hn_data *hv, uint32_t nvs_ver)
 149 {
 150         struct hn_nvs_init init;
 151         struct hn_nvs_init_resp resp;
 152         uint32_t status;
 153         int error;
 154
 155         memset(&init, 0, sizeof(init));
 156         init.type = NVS_TYPE_INIT;
 157         init.ver_min = nvs_ver;
 158         init.ver_max = nvs_ver;
 159
 160         error = hn_nvs_execute(hv, &init, sizeof(init),
 161                                &resp, sizeof(resp),
 162                                NVS_TYPE_INIT_RESP);
 163         if (error)
 164                 return error;
 165
 166         status = resp.status;
 167         if (status != NVS_STATUS_OK) {
 168                 /* Not fatal, try other versions */
 169                 PMD_INIT_LOG(DEBUG, "nvs init failed for ver 0x%x",
 170                              nvs_ver);
 171                 return -EINVAL;
 172         }
 173
 174         return 0;
 175 }
 176
 177 static int
 178 hn_nvs_conn_rxbuf(struct hn_data *hv)
 179 {
 180         struct hn_nvs_rxbuf_conn conn;
 181         struct hn_nvs_rxbuf_connresp resp;
 182         uint32_t status;
 183         int error;
 184
 185         /* Kernel has already setup RXBUF on primary channel. */
 186
 187         /*
 188          * Connect RXBUF to NVS.
 189          */
 190         conn.type = NVS_TYPE_RXBUF_CONN;
 191         conn.gpadl = hv->rxbuf_res->phys_addr;
 192         conn.sig = NVS_RXBUF_SIG;
 193         PMD_DRV_LOG(DEBUG, "connect rxbuff va=%p gpad=%#" PRIx64,
 194                     hv->rxbuf_res->addr,
 195                     hv->rxbuf_res->phys_addr);
 196
 197         error = hn_nvs_execute(hv, &conn, sizeof(conn),
 198                                &resp, sizeof(resp),
 199                                NVS_TYPE_RXBUF_CONNRESP);
 200         if (error) {
 201                 PMD_DRV_LOG(ERR,
 202                             "exec nvs rxbuf conn failed: %d",
 203                             error);
 204                 return error;
 205         }
 206
 207         status = resp.status;
 208         if (status != NVS_STATUS_OK) {
 209                 PMD_DRV_LOG(ERR,
 210                             "nvs rxbuf conn failed: %x", status);
 211                 return -EIO;
 212         }
 213         if (resp.nsect != 1) {
 214                 PMD_DRV_LOG(ERR,
 215                             "nvs rxbuf response num sections %u != 1",
 216                             resp.nsect);
 217                 return -EIO;
 218         }
 219
 220         PMD_DRV_LOG(INFO,
 221                     "receive buffer size %u count %u",
 222                     resp.nvs_sect[0].slotsz,
 223                     resp.nvs_sect[0].slotcnt);
 224         hv->rxbuf_section_cnt = resp.nvs_sect[0].slotcnt;
 225
 226         hv->rxbuf_info = rte_calloc("HN_RXBUF_INFO", hv->rxbuf_section_cnt,
 227                                     sizeof(*hv->rxbuf_info), RTE_CACHE_LINE_SIZE);
 228         if (!hv->rxbuf_info) {
 229                 PMD_DRV_LOG(ERR,
 230                             "could not allocate rxbuf info");
 231                 return -ENOMEM;
 232         }
 233
 234         return 0;
 235 }
 236
 237 static void
 238 hn_nvs_disconn_rxbuf(struct hn_data *hv)
 239 {
 240         struct hn_nvs_rxbuf_disconn disconn;
 241         int error;
 242
 243         /*
 244          * Disconnect RXBUF from NVS.
 245          */
 246         memset(&disconn, 0, sizeof(disconn));
 247         disconn.type = NVS_TYPE_RXBUF_DISCONN;
 248         disconn.sig = NVS_RXBUF_SIG;
 249
 250         /* NOTE: No response. */
 251         error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
 252         if (error) {
 253                 PMD_DRV_LOG(ERR,
 254                             "send nvs rxbuf disconn failed: %d",
 255                             error);
 256         }
 257
 258         rte_free(hv->rxbuf_info);
 259         /*
 260          * Linger long enough for NVS to disconnect RXBUF.
 261          */
 262         rte_delay_ms(200);
 263 }
 264
 265 static void
 266 hn_nvs_disconn_chim(struct hn_data *hv)
 267 {
 268         int error;
 269
 270         if (hv->chim_cnt != 0) {
 271                 struct hn_nvs_chim_disconn disconn;
 272
 273                 /* Disconnect chimney sending buffer from NVS. */
 274                 memset(&disconn, 0, sizeof(disconn));
 275                 disconn.type = NVS_TYPE_CHIM_DISCONN;
 276                 disconn.sig = NVS_CHIM_SIG;
 277
 278                 /* NOTE: No response. */
 279                 error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
 280
 281                 if (error) {
 282                         PMD_DRV_LOG(ERR,
 283                                     "send nvs chim disconn failed: %d", error);
 284                 }
 285
 286                 hv->chim_cnt = 0;
 287                 /*
 288                  * Linger long enough for NVS to disconnect chimney
 289                  * sending buffer.
 290                  */
 291                 rte_delay_ms(200);
 292         }
 293 }
 294
 295 static int
 296 hn_nvs_conn_chim(struct hn_data *hv)
 297 {
 298         struct hn_nvs_chim_conn chim;
 299         struct hn_nvs_chim_connresp resp;
 300         uint32_t sectsz;
 301         unsigned long len = hv->chim_res->len;
 302         int error;
 303
 304         /* Connect chimney sending buffer to NVS */
 305         memset(&chim, 0, sizeof(chim));
 306         chim.type = NVS_TYPE_CHIM_CONN;
 307         chim.gpadl = hv->chim_res->phys_addr;
 308         chim.sig = NVS_CHIM_SIG;
 309         PMD_DRV_LOG(DEBUG, "connect send buf va=%p gpad=%#" PRIx64,
 310                     hv->chim_res->addr,
 311                     hv->chim_res->phys_addr);
 312
 313         error = hn_nvs_execute(hv, &chim, sizeof(chim),
 314                                &resp, sizeof(resp),
 315                                NVS_TYPE_CHIM_CONNRESP);
 316         if (error) {
 317                 PMD_DRV_LOG(ERR, "exec nvs chim conn failed");
 318                 return error;
 319         }
 320
 321         if (resp.status != NVS_STATUS_OK) {
 322                 PMD_DRV_LOG(ERR, "nvs chim conn failed: %x",
 323                             resp.status);
 324                 return -EIO;
 325         }
 326
 327         sectsz = resp.sectsz;
 328         if (sectsz == 0 || sectsz & (sizeof(uint32_t) - 1)) {
 329                 /* Can't use chimney sending buffer; done! */
 330                 PMD_DRV_LOG(NOTICE,
 331                             "invalid chimney sending buffer section size: %u",
 332                             sectsz);
 333                 error = -EINVAL;
 334                 goto cleanup;
 335         }
 336
 337         hv->chim_szmax = sectsz;
 338         hv->chim_cnt = len / sectsz;
 339
 340         PMD_DRV_LOG(INFO, "send buffer %lu section size:%u, count:%u",
 341                     len, hv->chim_szmax, hv->chim_cnt);
 342
 343         /* Done! */
 344         return 0;
 345
 346 cleanup:
 347         hn_nvs_disconn_chim(hv);
 348         return error;
 349 }
 350
 351 /*
 352  * Configure MTU and enable VLAN.
 353  */
 354 static int
 355 hn_nvs_conf_ndis(struct hn_data *hv, unsigned int mtu)
 356 {
 357         struct hn_nvs_ndis_conf conf;
 358         int error;
 359
 360         memset(&conf, 0, sizeof(conf));
 361         conf.type = NVS_TYPE_NDIS_CONF;
 362         conf.mtu = mtu + RTE_ETHER_HDR_LEN;
 363         conf.caps = NVS_NDIS_CONF_VLAN;
 364
 365         /* enable SRIOV */
 366         if (hv->nvs_ver >= NVS_VERSION_5)
 367                 conf.caps |= NVS_NDIS_CONF_SRIOV;
 368
 369         /* NOTE: No response. */
 370         error = hn_nvs_req_send(hv, &conf, sizeof(conf));
 371         if (error) {
 372                 PMD_DRV_LOG(ERR,
 373                             "send nvs ndis conf failed: %d", error);
 374                 return error;
 375         }
 376
 377         return 0;
 378 }
 379
 380 static int
 381 hn_nvs_init_ndis(struct hn_data *hv)
 382 {
 383         struct hn_nvs_ndis_init ndis;
 384         int error;
 385
 386         memset(&ndis, 0, sizeof(ndis));
 387         ndis.type = NVS_TYPE_NDIS_INIT;
 388         ndis.ndis_major = NDIS_VERSION_MAJOR(hv->ndis_ver);
 389         ndis.ndis_minor = NDIS_VERSION_MINOR(hv->ndis_ver);
 390
 391         /* NOTE: No response. */
 392         error = hn_nvs_req_send(hv, &ndis, sizeof(ndis));
 393         if (error)
 394                 PMD_DRV_LOG(ERR,
 395                             "send nvs ndis init failed: %d", error);
 396
 397         return error;
 398 }
 399
 400 static int
 401 hn_nvs_init(struct hn_data *hv)
 402 {
 403         unsigned int i;
 404         int error;
 405
 406         /*
 407          * Find the supported NVS version and set NDIS version accordingly.
 408          */
 409         for (i = 0; i < RTE_DIM(hn_nvs_version); ++i) {
 410                 error = hn_nvs_doinit(hv, hn_nvs_version[i]);
 411                 if (error) {
 412                         PMD_INIT_LOG(DEBUG, "version %#x error %d",
 413                                      hn_nvs_version[i], error);
 414                         continue;
 415                 }
 416
 417                 hv->nvs_ver = hn_nvs_version[i];
 418
 419                 /* Set NDIS version according to NVS version. */
 420                 hv->ndis_ver = NDIS_VERSION_6_30;
 421                 if (hv->nvs_ver <= NVS_VERSION_4)
 422                         hv->ndis_ver = NDIS_VERSION_6_1;
 423
 424                 PMD_INIT_LOG(DEBUG,
 425                              "NVS version %#x, NDIS version %u.%u",
 426                              hv->nvs_ver, NDIS_VERSION_MAJOR(hv->ndis_ver),
 427                              NDIS_VERSION_MINOR(hv->ndis_ver));
 428                 return 0;
 429         }
 430
 431         PMD_DRV_LOG(ERR,
 432                     "no NVS compatible version available");
 433         return -ENXIO;
 434 }
 435
 436 int
 437 hn_nvs_attach(struct hn_data *hv, unsigned int mtu)
 438 {
 439         int error;
 440
 441         /*
 442          * Initialize NVS.
 443          */
 444         error = hn_nvs_init(hv);
 445         if (error)
 446                 return error;
 447
 448         /** Configure NDIS before initializing it. */
 449         if (hv->nvs_ver >= NVS_VERSION_2) {
 450                 error = hn_nvs_conf_ndis(hv, mtu);
 451                 if (error)
 452                         return error;
 453         }
 454
 455         /*
 456          * Initialize NDIS.
 457          */
 458         error = hn_nvs_init_ndis(hv);
 459         if (error)
 460                 return error;
 461
 462         /*
 463          * Connect RXBUF.
 464          */
 465         error = hn_nvs_conn_rxbuf(hv);
 466         if (error)
 467                 return error;
 468
 469         /*
 470          * Connect chimney sending buffer.
 471          */
 472         error = hn_nvs_conn_chim(hv);
 473         if (error) {
 474                 hn_nvs_disconn_rxbuf(hv);
 475                 return error;
 476         }
 477
 478         return 0;
 479 }
 480
 481 void
 482 hn_nvs_detach(struct hn_data *hv __rte_unused)
 483 {
 484         PMD_INIT_FUNC_TRACE();
 485
 486         /* NOTE: there are no requests to stop the NVS. */
 487         hn_nvs_disconn_rxbuf(hv);
 488         hn_nvs_disconn_chim(hv);
 489 }
 490
 491 /*
 492  * Ack the consumed RXBUF associated w/ this channel packet,
 493  * so that this RXBUF can be recycled by the hypervisor.
 494  */
 495 void
 496 hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid)
 497 {
 498         unsigned int retries = 0;
 499         struct hn_nvs_rndis_ack ack = {
 500                 .type = NVS_TYPE_RNDIS_ACK,
 501                 .status = NVS_STATUS_OK,
 502         };
 503         int error;
 504
 505         PMD_RX_LOG(DEBUG, "ack RX id %" PRIu64, tid);
 506
 507  again:
 508         error = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP,
 509                                     &ack, sizeof(ack), tid,
 510                                     VMBUS_CHANPKT_FLAG_NONE, NULL);
 511
 512         if (error == 0)
 513                 return;
 514
 515         if (error == -EAGAIN) {
 516                 /*
 517                  * NOTE:
 518                  * This should _not_ happen in real world, since the
 519                  * consumption of the TX bufring from the TX path is
 520                  * controlled.
 521                  */
 522                 PMD_RX_LOG(NOTICE, "RXBUF ack retry");
 523                 if (++retries < 10) {
 524                         rte_delay_ms(1);
 525                         goto again;
 526                 }
 527         }
 528         /* RXBUF leaks! */
 529         PMD_DRV_LOG(ERR, "RXBUF ack failed");
 530 }
 531
 532 int
 533 hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch)
 534 {
 535         struct hn_nvs_subch_req req;
 536         struct hn_nvs_subch_resp resp;
 537         int error;
 538
 539         memset(&req, 0, sizeof(req));
 540         req.type = NVS_TYPE_SUBCH_REQ;
 541         req.op = NVS_SUBCH_OP_ALLOC;
 542         req.nsubch = *nsubch;
 543
 544         error = hn_nvs_execute(hv, &req, sizeof(req),
 545                                &resp, sizeof(resp),
 546                                NVS_TYPE_SUBCH_RESP);
 547         if (error)
 548                 return error;
 549
 550         if (resp.status != NVS_STATUS_OK) {
 551                 PMD_INIT_LOG(ERR,
 552                              "nvs subch alloc failed: %#x",
 553                              resp.status);
 554                 return -EIO;
 555         }
 556
 557         if (resp.nsubch > *nsubch) {
 558                 PMD_INIT_LOG(NOTICE,
 559                              "%u subchans are allocated, requested %u",
 560                              resp.nsubch, *nsubch);
 561         }
 562         *nsubch = resp.nsubch;
 563
 564         return 0;
 565 }
 566
 567 void
 568 hn_nvs_set_datapath(struct hn_data *hv, uint32_t path)
 569 {
 570         struct hn_nvs_datapath dp;
 571         int error;
 572
 573         PMD_DRV_LOG(DEBUG, "set datapath %s",
 574                     path ? "VF" : "Synthetic");
 575
 576         memset(&dp, 0, sizeof(dp));
 577         dp.type = NVS_TYPE_SET_DATAPATH;
 578         dp.active_path = path;
 579
 580         error = hn_nvs_req_send(hv, &dp, sizeof(dp));
 581         if (error) {
 582                 PMD_DRV_LOG(ERR,
 583                             "send set datapath failed: %d",
 584                             error);
 585         }
 586 }