examples/vm_power_manager/channel_manager.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <sys/un.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <inttypes.h>
  11 #include <dirent.h>
  12 #include <errno.h>
  13
  14 #include <sys/queue.h>
  15 #include <sys/types.h>
  16 #include <sys/socket.h>
  17 #include <sys/select.h>
  18
  19 #include <rte_malloc.h>
  20 #include <rte_memory.h>
  21 #include <rte_mempool.h>
  22 #include <rte_log.h>
  23 #include <rte_atomic.h>
  24 #include <rte_spinlock.h>
  25
  26 #include <libvirt/libvirt.h>
  27
  28 #include "channel_manager.h"
  29 #include "channel_commands.h"
  30 #include "channel_monitor.h"
  31
  32
  33 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  34
  35 #define ITERATIVE_BITMASK_CHECK_64(mask_u64b, i) \
  36                 for (i = 0; mask_u64b; mask_u64b &= ~(1ULL << i++)) \
  37                 if ((mask_u64b >> i) & 1) \
  38
  39 /* Global pointer to libvirt connection */
  40 static virConnectPtr global_vir_conn_ptr;
  41
  42 static unsigned char *global_cpumaps;
  43 static virVcpuInfo *global_vircpuinfo;
  44 static size_t global_maplen;
  45
  46 static unsigned int global_n_host_cpus;
  47 static bool global_hypervisor_available;
  48
  49 /*
  50  * Represents a single Virtual Machine
  51  */
  52 struct virtual_machine_info {
  53         char name[CHANNEL_MGR_MAX_NAME_LEN];
  54         rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS];
  55         struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS];
  56         uint64_t channel_mask;
  57         uint8_t num_channels;
  58         enum vm_status status;
  59         virDomainPtr domainPtr;
  60         virDomainInfo info;
  61         rte_spinlock_t config_spinlock;
  62         LIST_ENTRY(virtual_machine_info) vms_info;
  63 };
  64
  65 LIST_HEAD(, virtual_machine_info) vm_list_head;
  66
  67 static struct virtual_machine_info *
  68 find_domain_by_name(const char *name)
  69 {
  70         struct virtual_machine_info *info;
  71         LIST_FOREACH(info, &vm_list_head, vms_info) {
  72                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
  73                         return info;
  74         }
  75         return NULL;
  76 }
  77
  78 static int
  79 update_pcpus_mask(struct virtual_machine_info *vm_info)
  80 {
  81         virVcpuInfoPtr cpuinfo;
  82         unsigned i, j;
  83         int n_vcpus;
  84         uint64_t mask;
  85
  86         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
  87
  88         if (!virDomainIsActive(vm_info->domainPtr)) {
  89                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
  90                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
  91                                 VIR_DOMAIN_AFFECT_CONFIG);
  92                 if (n_vcpus < 0) {
  93                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
  94                                         "in-active VM '%s'\n", vm_info->name);
  95                         return -1;
  96                 }
  97                 goto update_pcpus;
  98         }
  99
 100         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
 101                         CHANNEL_CMDS_MAX_CPUS);
 102
 103         cpuinfo = global_vircpuinfo;
 104
 105         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 106                         CHANNEL_CMDS_MAX_CPUS, global_cpumaps, global_maplen);
 107         if (n_vcpus < 0) {
 108                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 109                                 "active VM '%s'\n", vm_info->name);
 110                 return -1;
 111         }
 112 update_pcpus:
 113         if (n_vcpus >= CHANNEL_CMDS_MAX_CPUS) {
 114                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 115                                 "0...%d\n", n_vcpus, CHANNEL_CMDS_MAX_CPUS-1);
 116                 return -1;
 117         }
 118         if (n_vcpus != vm_info->info.nrVirtCpu) {
 119                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 120                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 121                                 n_vcpus);
 122                 vm_info->info.nrVirtCpu = n_vcpus;
 123         }
 124         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 125                 mask = 0;
 126                 for (j = 0; j < global_n_host_cpus; j++) {
 127                         if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j) > 0) {
 128                                 mask |= 1ULL << j;
 129                         }
 130                 }
 131                 rte_atomic64_set(&vm_info->pcpu_mask[i], mask);
 132         }
 133         return 0;
 134 }
 135
 136 int
 137 set_pcpus_mask(char *vm_name, unsigned vcpu, uint64_t core_mask)
 138 {
 139         unsigned i = 0;
 140         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 141         struct virtual_machine_info *vm_info;
 142         uint64_t mask = core_mask;
 143
 144         if (vcpu >= CHANNEL_CMDS_MAX_CPUS) {
 145                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 146                                 vcpu, CHANNEL_CMDS_MAX_CPUS-1);
 147                 return -1;
 148         }
 149
 150         vm_info = find_domain_by_name(vm_name);
 151         if (vm_info == NULL) {
 152                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 153                 return -1;
 154         }
 155
 156         if (!virDomainIsActive(vm_info->domainPtr)) {
 157                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 158                                 "mask(0x%"PRIx64") for VM '%s', VM is not active\n",
 159                                 vcpu, core_mask, vm_info->name);
 160                 return -1;
 161         }
 162
 163         if (vcpu >= vm_info->info.nrVirtCpu) {
 164                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 165                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 166                 return -1;
 167         }
 168         memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen);
 169         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 170                 VIR_USE_CPU(global_cpumaps, i);
 171                 if (i >= global_n_host_cpus) {
 172                         RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 173                                         "number of CPUs(%u)\n", i, global_n_host_cpus);
 174                         return -1;
 175                 }
 176         }
 177         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 178                         global_maplen, flags) < 0) {
 179                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 180                                 "mask(0x%"PRIx64") for VM '%s'\n", vcpu, core_mask,
 181                                 vm_info->name);
 182                 return -1;
 183         }
 184         rte_atomic64_set(&vm_info->pcpu_mask[vcpu], core_mask);
 185         return 0;
 186
 187 }
 188
 189 int
 190 set_pcpu(char *vm_name, unsigned vcpu, unsigned core_num)
 191 {
 192         uint64_t mask = 1ULL << core_num;
 193
 194         return set_pcpus_mask(vm_name, vcpu, mask);
 195 }
 196
 197 uint64_t
 198 get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu)
 199 {
 200         struct virtual_machine_info *vm_info =
 201                         (struct virtual_machine_info *)chan_info->priv_info;
 202
 203         if (global_hypervisor_available && (vm_info != NULL))
 204                 return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]);
 205         else
 206                 return 0;
 207 }
 208
 209 static inline int
 210 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 211 {
 212         rte_spinlock_lock(&(vm_info->config_spinlock));
 213         if (vm_info->channel_mask & (1ULL << channel_num)) {
 214                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 215                 return 1;
 216         }
 217         rte_spinlock_unlock(&(vm_info->config_spinlock));
 218         return 0;
 219 }
 220
 221
 222
 223 static int
 224 open_non_blocking_channel(struct channel_info *info)
 225 {
 226         int ret, flags;
 227         struct sockaddr_un sock_addr;
 228         fd_set soc_fd_set;
 229         struct timeval tv;
 230
 231         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 232         if (info->fd == -1) {
 233                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 234                                 strerror(errno),
 235                                 info->channel_path);
 236                 return -1;
 237         }
 238         sock_addr.sun_family = AF_UNIX;
 239         memcpy(&sock_addr.sun_path, info->channel_path,
 240                         strlen(info->channel_path)+1);
 241
 242         /* Get current flags */
 243         flags = fcntl(info->fd, F_GETFL, 0);
 244         if (flags < 0) {
 245                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 246                                 "'%s'\n", strerror(errno), info->channel_path);
 247                 return 1;
 248         }
 249         /* Set to Non Blocking */
 250         flags |= O_NONBLOCK;
 251         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 252                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 253                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 254                 return -1;
 255         }
 256         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 257                         sizeof(sock_addr));
 258         if (ret < 0) {
 259                 /* ECONNREFUSED error is given when VM is not active */
 260                 if (errno == ECONNREFUSED) {
 261                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 262                                         "activated its endpoint to channel %s\n",
 263                                         info->channel_path);
 264                         return -1;
 265                 }
 266                 /* Wait for tv_sec if in progress */
 267                 else if (errno == EINPROGRESS) {
 268                         tv.tv_sec = 2;
 269                         tv.tv_usec = 0;
 270                         FD_ZERO(&soc_fd_set);
 271                         FD_SET(info->fd, &soc_fd_set);
 272                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 273                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 274                                                 "'%s'\n", info->channel_path);
 275                                 return -1;
 276                         }
 277                 } else {
 278                         /* Any other error */
 279                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 280                                         " for '%s'\n", strerror(errno), info->channel_path);
 281                         return -1;
 282                 }
 283         }
 284         return 0;
 285 }
 286
 287 static int
 288 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 289                 struct channel_info **chan_info_dptr, unsigned channel_num)
 290 {
 291         struct channel_info *chan_info = *chan_info_dptr;
 292         struct virtual_machine_info *vm_info = *vm_info_dptr;
 293
 294         chan_info->channel_num = channel_num;
 295         chan_info->priv_info = (void *)vm_info;
 296         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 297         if (open_non_blocking_channel(chan_info) < 0) {
 298                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 299                                 "'%s' for VM '%s'\n",
 300                                 chan_info->channel_path, vm_info->name);
 301                 return -1;
 302         }
 303         if (add_channel_to_monitor(&chan_info) < 0) {
 304                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 305                                 "'%s' to epoll ctl for VM '%s'\n",
 306                                 chan_info->channel_path, vm_info->name);
 307                 return -1;
 308
 309         }
 310         rte_spinlock_lock(&(vm_info->config_spinlock));
 311         vm_info->num_channels++;
 312         vm_info->channel_mask |= 1ULL << channel_num;
 313         vm_info->channels[channel_num] = chan_info;
 314         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 315         rte_spinlock_unlock(&(vm_info->config_spinlock));
 316         return 0;
 317 }
 318
 319 int
 320 add_all_channels(const char *vm_name)
 321 {
 322         DIR *d;
 323         struct dirent *dir;
 324         struct virtual_machine_info *vm_info;
 325         struct channel_info *chan_info;
 326         char *token, *remaining, *tail_ptr;
 327         char socket_name[PATH_MAX];
 328         unsigned channel_num;
 329         int num_channels_enabled = 0;
 330
 331         /* verify VM exists */
 332         vm_info = find_domain_by_name(vm_name);
 333         if (vm_info == NULL) {
 334                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 335                                 " during channel discovery\n", vm_name);
 336                 return 0;
 337         }
 338         if (!virDomainIsActive(vm_info->domainPtr)) {
 339                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 340                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 341                 return 0;
 342         }
 343         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 344         if (d == NULL) {
 345                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 346                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 347                 return -1;
 348         }
 349         while ((dir = readdir(d)) != NULL) {
 350                 if (!strncmp(dir->d_name, ".", 1) ||
 351                                 !strncmp(dir->d_name, "..", 2))
 352                         continue;
 353
 354                 snprintf(socket_name, sizeof(socket_name), "%s", dir->d_name);
 355                 remaining = socket_name;
 356                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 357                 token = strsep(&remaining, ".");
 358                 if (remaining == NULL)
 359                         continue;
 360                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 361                         continue;
 362
 363                 /* remaining should contain only <channel_num> */
 364                 errno = 0;
 365                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 366                 if ((errno != 0) || (remaining[0] == '\0') ||
 367                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 368                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 369                                         "'%s' found it should be in the form of "
 370                                         "'<guest_name>.<channel_num>(decimal)'\n",
 371                                         dir->d_name);
 372                         continue;
 373                 }
 374                 if (channel_num >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 375                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 376                                         "greater than max allowable: %d, skipping '%s%s'\n",
 377                                         channel_num, CHANNEL_CMDS_MAX_VM_CHANNELS-1,
 378                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 379                         continue;
 380                 }
 381                 /* if channel has not been added previously */
 382                 if (channel_exists(vm_info, channel_num))
 383                         continue;
 384
 385                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 386                                 RTE_CACHE_LINE_SIZE);
 387                 if (chan_info == NULL) {
 388                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 389                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 390                         continue;
 391                 }
 392
 393                 snprintf(chan_info->channel_path,
 394                                 sizeof(chan_info->channel_path), "%s%s",
 395                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 396
 397                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 398                         rte_free(chan_info);
 399                         continue;
 400                 }
 401
 402                 num_channels_enabled++;
 403         }
 404         closedir(d);
 405         return num_channels_enabled;
 406 }
 407
 408 int
 409 add_channels(const char *vm_name, unsigned *channel_list,
 410                 unsigned len_channel_list)
 411 {
 412         struct virtual_machine_info *vm_info;
 413         struct channel_info *chan_info;
 414         char socket_path[PATH_MAX];
 415         unsigned i;
 416         int num_channels_enabled = 0;
 417
 418         vm_info = find_domain_by_name(vm_name);
 419         if (vm_info == NULL) {
 420                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 421                                 "not found\n", vm_name);
 422                 return 0;
 423         }
 424
 425         if (!virDomainIsActive(vm_info->domainPtr)) {
 426                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 427                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 428                 return 0;
 429         }
 430
 431         for (i = 0; i < len_channel_list; i++) {
 432
 433                 if (channel_list[i] >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 434                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 435                                                         "0...%d\n", channel_list[i],
 436                                                         CHANNEL_CMDS_MAX_VM_CHANNELS-1);
 437                         continue;
 438                 }
 439                 if (channel_exists(vm_info, channel_list[i])) {
 440                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 441                                         "'%s.%u'\n", vm_name, i);
 442                         continue;
 443                 }
 444
 445                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 446                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 447                 errno = 0;
 448                 if (access(socket_path, F_OK) < 0) {
 449                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 450                                         "%s\n", socket_path, strerror(errno));
 451                         continue;
 452                 }
 453                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 454                                 RTE_CACHE_LINE_SIZE);
 455                 if (chan_info == NULL) {
 456                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 457                                         "channel '%s'\n", socket_path);
 458                         continue;
 459                 }
 460                 snprintf(chan_info->channel_path,
 461                                 sizeof(chan_info->channel_path), "%s%s.%u",
 462                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 463                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 464                         rte_free(chan_info);
 465                         continue;
 466                 }
 467                 num_channels_enabled++;
 468
 469         }
 470         return num_channels_enabled;
 471 }
 472
 473 int
 474 remove_channel(struct channel_info **chan_info_dptr)
 475 {
 476         struct virtual_machine_info *vm_info;
 477         struct channel_info *chan_info = *chan_info_dptr;
 478
 479         close(chan_info->fd);
 480
 481         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 482
 483         rte_spinlock_lock(&(vm_info->config_spinlock));
 484         vm_info->channel_mask &= ~(1ULL << chan_info->channel_num);
 485         vm_info->num_channels--;
 486         rte_spinlock_unlock(&(vm_info->config_spinlock));
 487
 488         rte_free(chan_info);
 489         return 0;
 490 }
 491
 492 int
 493 set_channel_status_all(const char *vm_name, enum channel_status status)
 494 {
 495         struct virtual_machine_info *vm_info;
 496         unsigned i;
 497         uint64_t mask;
 498         int num_channels_changed = 0;
 499
 500         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 501                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 502                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 503                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 504         }
 505         vm_info = find_domain_by_name(vm_name);
 506         if (vm_info == NULL) {
 507                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 508                                 "not found\n", vm_name);
 509                 return 0;
 510         }
 511
 512         rte_spinlock_lock(&(vm_info->config_spinlock));
 513         mask = vm_info->channel_mask;
 514         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 515                 vm_info->channels[i]->status = status;
 516                 num_channels_changed++;
 517         }
 518         rte_spinlock_unlock(&(vm_info->config_spinlock));
 519         return num_channels_changed;
 520
 521 }
 522
 523 int
 524 set_channel_status(const char *vm_name, unsigned *channel_list,
 525                 unsigned len_channel_list, enum channel_status status)
 526 {
 527         struct virtual_machine_info *vm_info;
 528         unsigned i;
 529         int num_channels_changed = 0;
 530
 531         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 532                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 533                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 534                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 535         }
 536         vm_info = find_domain_by_name(vm_name);
 537         if (vm_info == NULL) {
 538                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 539                                 "not found\n", vm_name);
 540                 return 0;
 541         }
 542         for (i = 0; i < len_channel_list; i++) {
 543                 if (channel_exists(vm_info, channel_list[i])) {
 544                         rte_spinlock_lock(&(vm_info->config_spinlock));
 545                         vm_info->channels[channel_list[i]]->status = status;
 546                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 547                         num_channels_changed++;
 548                 }
 549         }
 550         return num_channels_changed;
 551 }
 552
 553 void
 554 get_all_vm(int *num_vm, int *num_vcpu)
 555 {
 556
 557         virNodeInfo node_info;
 558         virDomainPtr *domptr;
 559         uint64_t mask;
 560         int i, ii, numVcpus[MAX_VCPUS], cpu, n_vcpus;
 561         unsigned int jj;
 562         const char *vm_name;
 563         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 564                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 565         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 566
 567         if (!global_hypervisor_available)
 568                 return;
 569
 570         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
 571         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 572                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 573                 return;
 574         }
 575
 576         /* Returns number of pcpus */
 577         global_n_host_cpus = (unsigned int)node_info.cpus;
 578
 579         /* Returns number of active domains */
 580         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 581                                         domain_flags);
 582         if (*num_vm <= 0) {
 583                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 584                 return;
 585         }
 586
 587         for (i = 0; i < *num_vm; i++) {
 588
 589                 /* Get Domain Names */
 590                 vm_name = virDomainGetName(domptr[i]);
 591                 lvm_info[i].vm_name = vm_name;
 592
 593                 /* Get Number of Vcpus */
 594                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 595
 596                 /* Get Number of VCpus & VcpuPinInfo */
 597                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 598                                 numVcpus[i], global_cpumaps,
 599                                 global_maplen, domain_flag);
 600
 601                 if ((int)n_vcpus > 0) {
 602                         *num_vcpu = n_vcpus;
 603                         lvm_info[i].num_cpus = n_vcpus;
 604                 }
 605
 606                 /* Save pcpu in use by libvirt VMs */
 607                 for (ii = 0; ii < n_vcpus; ii++) {
 608                         mask = 0;
 609                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 610                                 if (VIR_CPU_USABLE(global_cpumaps,
 611                                                 global_maplen, ii, jj) > 0) {
 612                                         mask |= 1ULL << jj;
 613                                 }
 614                         }
 615                         ITERATIVE_BITMASK_CHECK_64(mask, cpu) {
 616                                 lvm_info[i].pcpus[ii] = cpu;
 617                         }
 618                 }
 619         }
 620 }
 621
 622 int
 623 get_info_vm(const char *vm_name, struct vm_info *info)
 624 {
 625         struct virtual_machine_info *vm_info;
 626         unsigned i, channel_num = 0;
 627         uint64_t mask;
 628
 629         vm_info = find_domain_by_name(vm_name);
 630         if (vm_info == NULL) {
 631                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 632                 return -1;
 633         }
 634         info->status = CHANNEL_MGR_VM_ACTIVE;
 635         if (!virDomainIsActive(vm_info->domainPtr))
 636                 info->status = CHANNEL_MGR_VM_INACTIVE;
 637
 638         rte_spinlock_lock(&(vm_info->config_spinlock));
 639
 640         mask = vm_info->channel_mask;
 641         ITERATIVE_BITMASK_CHECK_64(mask, i) {
 642                 info->channels[channel_num].channel_num = i;
 643                 memcpy(info->channels[channel_num].channel_path,
 644                                 vm_info->channels[i]->channel_path, UNIX_PATH_MAX);
 645                 info->channels[channel_num].status = vm_info->channels[i]->status;
 646                 info->channels[channel_num].fd = vm_info->channels[i]->fd;
 647                 channel_num++;
 648         }
 649
 650         info->num_channels = channel_num;
 651         info->num_vcpus = vm_info->info.nrVirtCpu;
 652         rte_spinlock_unlock(&(vm_info->config_spinlock));
 653
 654         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 655         for (i = 0; i < info->num_vcpus; i++) {
 656                 info->pcpu_mask[i] = rte_atomic64_read(&vm_info->pcpu_mask[i]);
 657         }
 658         return 0;
 659 }
 660
 661 int
 662 add_vm(const char *vm_name)
 663 {
 664         struct virtual_machine_info *new_domain;
 665         virDomainPtr dom_ptr;
 666         int i;
 667
 668         if (find_domain_by_name(vm_name) != NULL) {
 669                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 670                                 "already exists\n", vm_name);
 671                 return -1;
 672         }
 673
 674         if (global_vir_conn_ptr == NULL) {
 675                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 676                 return -1;
 677         }
 678         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 679         if (dom_ptr == NULL) {
 680                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 681                                 "VM '%s' not found\n", vm_name);
 682                 return -1;
 683         }
 684
 685         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 686                         RTE_CACHE_LINE_SIZE);
 687         if (new_domain == NULL) {
 688                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 689                                 "info\n");
 690                 return -1;
 691         }
 692         new_domain->domainPtr = dom_ptr;
 693         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 694                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 695                 rte_free(new_domain);
 696                 return -1;
 697         }
 698         if (new_domain->info.nrVirtCpu > CHANNEL_CMDS_MAX_CPUS) {
 699                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 700                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 701                                 CHANNEL_CMDS_MAX_CPUS);
 702                 rte_free(new_domain);
 703                 return -1;
 704         }
 705
 706         for (i = 0; i < CHANNEL_CMDS_MAX_CPUS; i++) {
 707                 rte_atomic64_init(&new_domain->pcpu_mask[i]);
 708         }
 709         if (update_pcpus_mask(new_domain) < 0) {
 710                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 711                 rte_free(new_domain);
 712                 return -1;
 713         }
 714         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 715         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 716         new_domain->channel_mask = 0;
 717         new_domain->num_channels = 0;
 718
 719         if (!virDomainIsActive(dom_ptr))
 720                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 721         else
 722                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 723
 724         rte_spinlock_init(&(new_domain->config_spinlock));
 725         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 726         return 0;
 727 }
 728
 729 int
 730 remove_vm(const char *vm_name)
 731 {
 732         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 733
 734         if (vm_info == NULL) {
 735                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 736                                 "not found\n", vm_name);
 737                 return -1;
 738         }
 739         rte_spinlock_lock(&vm_info->config_spinlock);
 740         if (vm_info->num_channels != 0) {
 741                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 742                                 "%"PRId8" channels still active\n",
 743                                 vm_name, vm_info->num_channels);
 744                 rte_spinlock_unlock(&vm_info->config_spinlock);
 745                 return -1;
 746         }
 747         LIST_REMOVE(vm_info, vms_info);
 748         rte_spinlock_unlock(&vm_info->config_spinlock);
 749         rte_free(vm_info);
 750         return 0;
 751 }
 752
 753 static void
 754 disconnect_hypervisor(void)
 755 {
 756         if (global_vir_conn_ptr != NULL) {
 757                 virConnectClose(global_vir_conn_ptr);
 758                 global_vir_conn_ptr = NULL;
 759         }
 760 }
 761
 762 static int
 763 connect_hypervisor(const char *path)
 764 {
 765         if (global_vir_conn_ptr != NULL) {
 766                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 767                                 "already established\n", path);
 768                 return -1;
 769         }
 770         global_vir_conn_ptr = virConnectOpen(path);
 771         if (global_vir_conn_ptr == NULL) {
 772                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 773                                 "Hypervisor '%s'\n", path);
 774                 return -1;
 775         }
 776         return 0;
 777 }
 778 int
 779 channel_manager_init(const char *path __rte_unused)
 780 {
 781         virNodeInfo info;
 782
 783         LIST_INIT(&vm_list_head);
 784         if (connect_hypervisor(path) < 0) {
 785                 global_n_host_cpus = 64;
 786                 global_hypervisor_available = 0;
 787                 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 788         } else {
 789                 global_hypervisor_available = 1;
 790
 791                 global_maplen = VIR_CPU_MAPLEN(CHANNEL_CMDS_MAX_CPUS);
 792
 793                 global_vircpuinfo = rte_zmalloc(NULL,
 794                                 sizeof(*global_vircpuinfo) *
 795                                 CHANNEL_CMDS_MAX_CPUS, RTE_CACHE_LINE_SIZE);
 796                 if (global_vircpuinfo == NULL) {
 797                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 798                         goto error;
 799                 }
 800                 global_cpumaps = rte_zmalloc(NULL,
 801                                 CHANNEL_CMDS_MAX_CPUS * global_maplen,
 802                                 RTE_CACHE_LINE_SIZE);
 803                 if (global_cpumaps == NULL)
 804                         goto error;
 805
 806                 if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 807                         RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 808                         goto error;
 809                 }
 810                 global_n_host_cpus = (unsigned int)info.cpus;
 811         }
 812
 813
 814
 815         if (global_n_host_cpus > CHANNEL_CMDS_MAX_CPUS) {
 816                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 817                                 "maximum of %u. No cores over %u should be used.\n",
 818                                 global_n_host_cpus, CHANNEL_CMDS_MAX_CPUS,
 819                                 CHANNEL_CMDS_MAX_CPUS - 1);
 820                 global_n_host_cpus = CHANNEL_CMDS_MAX_CPUS;
 821         }
 822
 823         return 0;
 824 error:
 825         if (global_hypervisor_available)
 826                 disconnect_hypervisor();
 827         return -1;
 828 }
 829
 830 void
 831 channel_manager_exit(void)
 832 {
 833         unsigned i;
 834         uint64_t mask;
 835         struct virtual_machine_info *vm_info;
 836
 837         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
 838
 839                 rte_spinlock_lock(&(vm_info->config_spinlock));
 840
 841                 mask = vm_info->channel_mask;
 842                 ITERATIVE_BITMASK_CHECK_64(mask, i) {
 843                         remove_channel_from_monitor(vm_info->channels[i]);
 844                         close(vm_info->channels[i]->fd);
 845                         rte_free(vm_info->channels[i]);
 846                 }
 847                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 848
 849                 LIST_REMOVE(vm_info, vms_info);
 850                 rte_free(vm_info);
 851         }
 852
 853         if (global_hypervisor_available) {
 854                 /* Only needed if hypervisor available */
 855                 rte_free(global_cpumaps);
 856                 rte_free(global_vircpuinfo);
 857                 disconnect_hypervisor();
 858         }
 859 }