drivers/net/netvsc/hn_nvs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2018 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  */
   7
   8 /*
   9  * Network Virtualization Service.
  10  */
  11
  12
  13 #include <stdint.h>
  14 #include <string.h>
  15 #include <stdio.h>
  16 #include <errno.h>
  17 #include <unistd.h>
  18
  19 #include <rte_ethdev.h>
  20 #include <rte_string_fns.h>
  21 #include <rte_memzone.h>
  22 #include <rte_malloc.h>
  23 #include <rte_atomic.h>
  24 #include <rte_branch_prediction.h>
  25 #include <rte_ether.h>
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_cycles.h>
  29 #include <rte_memory.h>
  30 #include <rte_eal.h>
  31 #include <rte_dev.h>
  32 #include <rte_bus_vmbus.h>
  33
  34 #include "hn_logs.h"
  35 #include "hn_var.h"
  36 #include "hn_nvs.h"
  37
  38 static const uint32_t hn_nvs_version[] = {
  39         NVS_VERSION_61,
  40         NVS_VERSION_6,
  41         NVS_VERSION_5,
  42         NVS_VERSION_4,
  43         NVS_VERSION_2,
  44         NVS_VERSION_1
  45 };
  46
  47 static int hn_nvs_req_send(struct hn_data *hv,
  48                            void *req, uint32_t reqlen)
  49 {
  50         return rte_vmbus_chan_send(hn_primary_chan(hv),
  51                                    VMBUS_CHANPKT_TYPE_INBAND,
  52                                    req, reqlen, 0,
  53                                    VMBUS_CHANPKT_FLAG_NONE, NULL);
  54 }
  55
  56 static int
  57 __hn_nvs_execute(struct hn_data *hv,
  58                void *req, uint32_t reqlen,
  59                void *resp, uint32_t resplen,
  60                uint32_t type)
  61 {
  62         struct vmbus_channel *chan = hn_primary_chan(hv);
  63         char buffer[NVS_RESPSIZE_MAX];
  64         const struct hn_nvs_hdr *hdr;
  65         uint64_t xactid;
  66         uint32_t len;
  67         int ret;
  68
  69         /* Send request to ring buffer */
  70         ret = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND,
  71                                   req, reqlen, 0,
  72                                   VMBUS_CHANPKT_FLAG_RC, NULL);
  73
  74         if (ret) {
  75                 PMD_DRV_LOG(ERR, "send request failed: %d", ret);
  76                 return ret;
  77         }
  78
  79  retry:
  80         len = sizeof(buffer);
  81         ret = rte_vmbus_chan_recv(chan, buffer, &len, &xactid);
  82         if (ret == -EAGAIN) {
  83                 rte_delay_us(HN_CHAN_INTERVAL_US);
  84                 goto retry;
  85         }
  86
  87         if (ret < 0) {
  88                 PMD_DRV_LOG(ERR, "recv response failed: %d", ret);
  89                 return ret;
  90         }
  91
  92         if (len < sizeof(*hdr)) {
  93                 PMD_DRV_LOG(ERR, "response missing NVS header");
  94                 return -EINVAL;
  95         }
  96
  97         hdr = (struct hn_nvs_hdr *)buffer;
  98
  99         /* Silently drop received packets while waiting for response */
 100         if (hdr->type == NVS_TYPE_RNDIS) {
 101                 hn_nvs_ack_rxbuf(chan, xactid);
 102                 --hv->rxbuf_outstanding;
 103                 goto retry;
 104         }
 105
 106         if (hdr->type != type) {
 107                 PMD_DRV_LOG(ERR, "unexpected NVS resp %#x, expect %#x",
 108                             hdr->type, type);
 109                 return -EINVAL;
 110         }
 111
 112         if (len < resplen) {
 113                 PMD_DRV_LOG(ERR,
 114                             "invalid NVS resp len %u (expect %u)",
 115                             len, resplen);
 116                 return -EINVAL;
 117         }
 118
 119         memcpy(resp, buffer, resplen);
 120
 121         /* All pass! */
 122         return 0;
 123 }
 124
 125
 126 /*
 127  * Execute one control command and get the response.
 128  * Only one command can be active on a channel at once
 129  * Unlike BSD, DPDK does not have an interrupt context
 130  * so the polling is required to wait for response.
 131  */
 132 static int
 133 hn_nvs_execute(struct hn_data *hv,
 134                void *req, uint32_t reqlen,
 135                void *resp, uint32_t resplen,
 136                uint32_t type)
 137 {
 138         struct hn_rx_queue *rxq = hv->primary;
 139         int ret;
 140
 141         rte_spinlock_lock(&rxq->ring_lock);
 142         ret = __hn_nvs_execute(hv, req, reqlen, resp, resplen, type);
 143         rte_spinlock_unlock(&rxq->ring_lock);
 144
 145         return ret;
 146 }
 147
 148 static int
 149 hn_nvs_doinit(struct hn_data *hv, uint32_t nvs_ver)
 150 {
 151         struct hn_nvs_init init;
 152         struct hn_nvs_init_resp resp;
 153         uint32_t status;
 154         int error;
 155
 156         memset(&init, 0, sizeof(init));
 157         init.type = NVS_TYPE_INIT;
 158         init.ver_min = nvs_ver;
 159         init.ver_max = nvs_ver;
 160
 161         error = hn_nvs_execute(hv, &init, sizeof(init),
 162                                &resp, sizeof(resp),
 163                                NVS_TYPE_INIT_RESP);
 164         if (error)
 165                 return error;
 166
 167         status = resp.status;
 168         if (status != NVS_STATUS_OK) {
 169                 /* Not fatal, try other versions */
 170                 PMD_INIT_LOG(DEBUG, "nvs init failed for ver 0x%x",
 171                              nvs_ver);
 172                 return -EINVAL;
 173         }
 174
 175         return 0;
 176 }
 177
 178 static int
 179 hn_nvs_conn_rxbuf(struct hn_data *hv)
 180 {
 181         struct hn_nvs_rxbuf_conn conn;
 182         struct hn_nvs_rxbuf_connresp resp;
 183         uint32_t status;
 184         int error;
 185
 186         /* Kernel has already setup RXBUF on primary channel. */
 187
 188         /*
 189          * Connect RXBUF to NVS.
 190          */
 191         conn.type = NVS_TYPE_RXBUF_CONN;
 192         conn.gpadl = hv->rxbuf_res->phys_addr;
 193         conn.sig = NVS_RXBUF_SIG;
 194         PMD_DRV_LOG(DEBUG, "connect rxbuff va=%p gpad=%#" PRIx64,
 195                     hv->rxbuf_res->addr,
 196                     hv->rxbuf_res->phys_addr);
 197
 198         error = hn_nvs_execute(hv, &conn, sizeof(conn),
 199                                &resp, sizeof(resp),
 200                                NVS_TYPE_RXBUF_CONNRESP);
 201         if (error) {
 202                 PMD_DRV_LOG(ERR,
 203                             "exec nvs rxbuf conn failed: %d",
 204                             error);
 205                 return error;
 206         }
 207
 208         status = resp.status;
 209         if (status != NVS_STATUS_OK) {
 210                 PMD_DRV_LOG(ERR,
 211                             "nvs rxbuf conn failed: %x", status);
 212                 return -EIO;
 213         }
 214         if (resp.nsect != 1) {
 215                 PMD_DRV_LOG(ERR,
 216                             "nvs rxbuf response num sections %u != 1",
 217                             resp.nsect);
 218                 return -EIO;
 219         }
 220
 221         PMD_DRV_LOG(INFO,
 222                     "receive buffer size %u count %u",
 223                     resp.nvs_sect[0].slotsz,
 224                     resp.nvs_sect[0].slotcnt);
 225         hv->rxbuf_section_cnt = resp.nvs_sect[0].slotcnt;
 226
 227         hv->rxbuf_info = rte_calloc("HN_RXBUF_INFO", hv->rxbuf_section_cnt,
 228                                     sizeof(*hv->rxbuf_info), RTE_CACHE_LINE_SIZE);
 229         if (!hv->rxbuf_info) {
 230                 PMD_DRV_LOG(ERR,
 231                             "could not allocate rxbuf info");
 232                 return -ENOMEM;
 233         }
 234
 235         return 0;
 236 }
 237
 238 static void
 239 hn_nvs_disconn_rxbuf(struct hn_data *hv)
 240 {
 241         struct hn_nvs_rxbuf_disconn disconn;
 242         int error;
 243
 244         /*
 245          * Disconnect RXBUF from NVS.
 246          */
 247         memset(&disconn, 0, sizeof(disconn));
 248         disconn.type = NVS_TYPE_RXBUF_DISCONN;
 249         disconn.sig = NVS_RXBUF_SIG;
 250
 251         /* NOTE: No response. */
 252         error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
 253         if (error) {
 254                 PMD_DRV_LOG(ERR,
 255                             "send nvs rxbuf disconn failed: %d",
 256                             error);
 257         }
 258
 259         rte_free(hv->rxbuf_info);
 260         /*
 261          * Linger long enough for NVS to disconnect RXBUF.
 262          */
 263         rte_delay_ms(200);
 264 }
 265
 266 static void
 267 hn_nvs_disconn_chim(struct hn_data *hv)
 268 {
 269         int error;
 270
 271         if (hv->chim_cnt != 0) {
 272                 struct hn_nvs_chim_disconn disconn;
 273
 274                 /* Disconnect chimney sending buffer from NVS. */
 275                 memset(&disconn, 0, sizeof(disconn));
 276                 disconn.type = NVS_TYPE_CHIM_DISCONN;
 277                 disconn.sig = NVS_CHIM_SIG;
 278
 279                 /* NOTE: No response. */
 280                 error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
 281
 282                 if (error) {
 283                         PMD_DRV_LOG(ERR,
 284                                     "send nvs chim disconn failed: %d", error);
 285                 }
 286
 287                 hv->chim_cnt = 0;
 288                 /*
 289                  * Linger long enough for NVS to disconnect chimney
 290                  * sending buffer.
 291                  */
 292                 rte_delay_ms(200);
 293         }
 294 }
 295
 296 static int
 297 hn_nvs_conn_chim(struct hn_data *hv)
 298 {
 299         struct hn_nvs_chim_conn chim;
 300         struct hn_nvs_chim_connresp resp;
 301         uint32_t sectsz;
 302         unsigned long len = hv->chim_res->len;
 303         int error;
 304
 305         /* Connect chimney sending buffer to NVS */
 306         memset(&chim, 0, sizeof(chim));
 307         chim.type = NVS_TYPE_CHIM_CONN;
 308         chim.gpadl = hv->chim_res->phys_addr;
 309         chim.sig = NVS_CHIM_SIG;
 310         PMD_DRV_LOG(DEBUG, "connect send buf va=%p gpad=%#" PRIx64,
 311                     hv->chim_res->addr,
 312                     hv->chim_res->phys_addr);
 313
 314         error = hn_nvs_execute(hv, &chim, sizeof(chim),
 315                                &resp, sizeof(resp),
 316                                NVS_TYPE_CHIM_CONNRESP);
 317         if (error) {
 318                 PMD_DRV_LOG(ERR, "exec nvs chim conn failed");
 319                 return error;
 320         }
 321
 322         if (resp.status != NVS_STATUS_OK) {
 323                 PMD_DRV_LOG(ERR, "nvs chim conn failed: %x",
 324                             resp.status);
 325                 return -EIO;
 326         }
 327
 328         sectsz = resp.sectsz;
 329         if (sectsz == 0 || sectsz & (sizeof(uint32_t) - 1)) {
 330                 /* Can't use chimney sending buffer; done! */
 331                 PMD_DRV_LOG(NOTICE,
 332                             "invalid chimney sending buffer section size: %u",
 333                             sectsz);
 334                 error = -EINVAL;
 335                 goto cleanup;
 336         }
 337
 338         hv->chim_szmax = sectsz;
 339         hv->chim_cnt = len / sectsz;
 340
 341         PMD_DRV_LOG(INFO, "send buffer %lu section size:%u, count:%u",
 342                     len, hv->chim_szmax, hv->chim_cnt);
 343
 344         /* Done! */
 345         return 0;
 346
 347 cleanup:
 348         hn_nvs_disconn_chim(hv);
 349         return error;
 350 }
 351
 352 /*
 353  * Configure MTU and enable VLAN.
 354  */
 355 static int
 356 hn_nvs_conf_ndis(struct hn_data *hv, unsigned int mtu)
 357 {
 358         struct hn_nvs_ndis_conf conf;
 359         int error;
 360
 361         memset(&conf, 0, sizeof(conf));
 362         conf.type = NVS_TYPE_NDIS_CONF;
 363         conf.mtu = mtu + RTE_ETHER_HDR_LEN;
 364         conf.caps = NVS_NDIS_CONF_VLAN;
 365
 366         /* enable SRIOV */
 367         if (hv->nvs_ver >= NVS_VERSION_5)
 368                 conf.caps |= NVS_NDIS_CONF_SRIOV;
 369
 370         /* NOTE: No response. */
 371         error = hn_nvs_req_send(hv, &conf, sizeof(conf));
 372         if (error) {
 373                 PMD_DRV_LOG(ERR,
 374                             "send nvs ndis conf failed: %d", error);
 375                 return error;
 376         }
 377
 378         return 0;
 379 }
 380
 381 static int
 382 hn_nvs_init_ndis(struct hn_data *hv)
 383 {
 384         struct hn_nvs_ndis_init ndis;
 385         int error;
 386
 387         memset(&ndis, 0, sizeof(ndis));
 388         ndis.type = NVS_TYPE_NDIS_INIT;
 389         ndis.ndis_major = NDIS_VERSION_MAJOR(hv->ndis_ver);
 390         ndis.ndis_minor = NDIS_VERSION_MINOR(hv->ndis_ver);
 391
 392         /* NOTE: No response. */
 393         error = hn_nvs_req_send(hv, &ndis, sizeof(ndis));
 394         if (error)
 395                 PMD_DRV_LOG(ERR,
 396                             "send nvs ndis init failed: %d", error);
 397
 398         return error;
 399 }
 400
 401 static int
 402 hn_nvs_init(struct hn_data *hv)
 403 {
 404         unsigned int i;
 405         int error;
 406
 407         /*
 408          * Find the supported NVS version and set NDIS version accordingly.
 409          */
 410         for (i = 0; i < RTE_DIM(hn_nvs_version); ++i) {
 411                 error = hn_nvs_doinit(hv, hn_nvs_version[i]);
 412                 if (error) {
 413                         PMD_INIT_LOG(DEBUG, "version %#x error %d",
 414                                      hn_nvs_version[i], error);
 415                         continue;
 416                 }
 417
 418                 hv->nvs_ver = hn_nvs_version[i];
 419
 420                 /* Set NDIS version according to NVS version. */
 421                 hv->ndis_ver = NDIS_VERSION_6_30;
 422                 if (hv->nvs_ver <= NVS_VERSION_4)
 423                         hv->ndis_ver = NDIS_VERSION_6_1;
 424
 425                 PMD_INIT_LOG(DEBUG,
 426                              "NVS version %#x, NDIS version %u.%u",
 427                              hv->nvs_ver, NDIS_VERSION_MAJOR(hv->ndis_ver),
 428                              NDIS_VERSION_MINOR(hv->ndis_ver));
 429                 return 0;
 430         }
 431
 432         PMD_DRV_LOG(ERR,
 433                     "no NVS compatible version available");
 434         return -ENXIO;
 435 }
 436
 437 int
 438 hn_nvs_attach(struct hn_data *hv, unsigned int mtu)
 439 {
 440         int error;
 441
 442         /*
 443          * Initialize NVS.
 444          */
 445         error = hn_nvs_init(hv);
 446         if (error)
 447                 return error;
 448
 449         /** Configure NDIS before initializing it. */
 450         if (hv->nvs_ver >= NVS_VERSION_2) {
 451                 error = hn_nvs_conf_ndis(hv, mtu);
 452                 if (error)
 453                         return error;
 454         }
 455
 456         /*
 457          * Initialize NDIS.
 458          */
 459         error = hn_nvs_init_ndis(hv);
 460         if (error)
 461                 return error;
 462
 463         /*
 464          * Connect RXBUF.
 465          */
 466         error = hn_nvs_conn_rxbuf(hv);
 467         if (error)
 468                 return error;
 469
 470         /*
 471          * Connect chimney sending buffer.
 472          */
 473         error = hn_nvs_conn_chim(hv);
 474         if (error) {
 475                 hn_nvs_disconn_rxbuf(hv);
 476                 return error;
 477         }
 478
 479         return 0;
 480 }
 481
 482 void
 483 hn_nvs_detach(struct hn_data *hv __rte_unused)
 484 {
 485         PMD_INIT_FUNC_TRACE();
 486
 487         /* NOTE: there are no requests to stop the NVS. */
 488         hn_nvs_disconn_rxbuf(hv);
 489         hn_nvs_disconn_chim(hv);
 490 }
 491
 492 /*
 493  * Ack the consumed RXBUF associated w/ this channel packet,
 494  * so that this RXBUF can be recycled by the hypervisor.
 495  */
 496 void
 497 hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid)
 498 {
 499         unsigned int retries = 0;
 500         struct hn_nvs_rndis_ack ack = {
 501                 .type = NVS_TYPE_RNDIS_ACK,
 502                 .status = NVS_STATUS_OK,
 503         };
 504         int error;
 505
 506         PMD_RX_LOG(DEBUG, "ack RX id %" PRIu64, tid);
 507
 508  again:
 509         error = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP,
 510                                     &ack, sizeof(ack), tid,
 511                                     VMBUS_CHANPKT_FLAG_NONE, NULL);
 512
 513         if (error == 0)
 514                 return;
 515
 516         if (error == -EAGAIN) {
 517                 /*
 518                  * NOTE:
 519                  * This should _not_ happen in real world, since the
 520                  * consumption of the TX bufring from the TX path is
 521                  * controlled.
 522                  */
 523                 PMD_RX_LOG(NOTICE, "RXBUF ack retry");
 524                 if (++retries < 10) {
 525                         rte_delay_ms(1);
 526                         goto again;
 527                 }
 528         }
 529         /* RXBUF leaks! */
 530         PMD_DRV_LOG(ERR, "RXBUF ack failed");
 531 }
 532
 533 int
 534 hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch)
 535 {
 536         struct hn_nvs_subch_req req;
 537         struct hn_nvs_subch_resp resp;
 538         int error;
 539
 540         memset(&req, 0, sizeof(req));
 541         req.type = NVS_TYPE_SUBCH_REQ;
 542         req.op = NVS_SUBCH_OP_ALLOC;
 543         req.nsubch = *nsubch;
 544
 545         error = hn_nvs_execute(hv, &req, sizeof(req),
 546                                &resp, sizeof(resp),
 547                                NVS_TYPE_SUBCH_RESP);
 548         if (error)
 549                 return error;
 550
 551         if (resp.status != NVS_STATUS_OK) {
 552                 PMD_INIT_LOG(ERR,
 553                              "nvs subch alloc failed: %#x",
 554                              resp.status);
 555                 return -EIO;
 556         }
 557
 558         if (resp.nsubch > *nsubch) {
 559                 PMD_INIT_LOG(NOTICE,
 560                              "%u subchans are allocated, requested %u",
 561                              resp.nsubch, *nsubch);
 562         }
 563         *nsubch = resp.nsubch;
 564
 565         return 0;
 566 }
 567
 568 void
 569 hn_nvs_set_datapath(struct hn_data *hv, uint32_t path)
 570 {
 571         struct hn_nvs_datapath dp;
 572         int error;
 573
 574         PMD_DRV_LOG(DEBUG, "set datapath %s",
 575                     path ? "VF" : "Synthetic");
 576
 577         memset(&dp, 0, sizeof(dp));
 578         dp.type = NVS_TYPE_SET_DATAPATH;
 579         dp.active_path = path;
 580
 581         error = hn_nvs_req_send(hv, &dp, sizeof(dp));
 582         if (error) {
 583                 PMD_DRV_LOG(ERR,
 584                             "send set datapath failed: %d",
 585                             error);
 586         }
 587 }