examples/vm_power_manager/channel_manager.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <sys/un.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <inttypes.h>
  11 #include <dirent.h>
  12 #include <errno.h>
  13
  14 #include <sys/queue.h>
  15 #include <sys/types.h>
  16 #include <sys/stat.h>
  17 #include <sys/socket.h>
  18 #include <sys/select.h>
  19
  20 #include <rte_malloc.h>
  21 #include <rte_memory.h>
  22 #include <rte_mempool.h>
  23 #include <rte_log.h>
  24 #include <rte_atomic.h>
  25 #include <rte_spinlock.h>
  26
  27 #include <libvirt/libvirt.h>
  28
  29 #include "channel_manager.h"
  30 #include "channel_commands.h"
  31 #include "channel_monitor.h"
  32 #include "power_manager.h"
  33
  34
  35 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  36
  37 /* Global pointer to libvirt connection */
  38 static virConnectPtr global_vir_conn_ptr;
  39
  40 static unsigned char *global_cpumaps;
  41 static virVcpuInfo *global_vircpuinfo;
  42 static size_t global_maplen;
  43
  44 static unsigned int global_n_host_cpus;
  45 static bool global_hypervisor_available;
  46
  47 /*
  48  * Represents a single Virtual Machine
  49  */
  50 struct virtual_machine_info {
  51         char name[CHANNEL_MGR_MAX_NAME_LEN];
  52         rte_atomic64_t pcpu_mask[CHANNEL_CMDS_MAX_CPUS];
  53         struct channel_info *channels[CHANNEL_CMDS_MAX_VM_CHANNELS];
  54         char channel_mask[POWER_MGR_MAX_CPUS];
  55         uint8_t num_channels;
  56         enum vm_status status;
  57         virDomainPtr domainPtr;
  58         virDomainInfo info;
  59         rte_spinlock_t config_spinlock;
  60         LIST_ENTRY(virtual_machine_info) vms_info;
  61 };
  62
  63 LIST_HEAD(, virtual_machine_info) vm_list_head;
  64
  65 static struct virtual_machine_info *
  66 find_domain_by_name(const char *name)
  67 {
  68         struct virtual_machine_info *info;
  69         LIST_FOREACH(info, &vm_list_head, vms_info) {
  70                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
  71                         return info;
  72         }
  73         return NULL;
  74 }
  75
  76 static int
  77 update_pcpus_mask(struct virtual_machine_info *vm_info)
  78 {
  79         virVcpuInfoPtr cpuinfo;
  80         unsigned i, j;
  81         int n_vcpus;
  82         uint64_t mask;
  83
  84         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
  85
  86         if (!virDomainIsActive(vm_info->domainPtr)) {
  87                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
  88                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
  89                                 VIR_DOMAIN_AFFECT_CONFIG);
  90                 if (n_vcpus < 0) {
  91                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
  92                                         "in-active VM '%s'\n", vm_info->name);
  93                         return -1;
  94                 }
  95                 goto update_pcpus;
  96         }
  97
  98         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
  99                         CHANNEL_CMDS_MAX_CPUS);
 100
 101         cpuinfo = global_vircpuinfo;
 102
 103         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 104                         CHANNEL_CMDS_MAX_CPUS, global_cpumaps, global_maplen);
 105         if (n_vcpus < 0) {
 106                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 107                                 "active VM '%s'\n", vm_info->name);
 108                 return -1;
 109         }
 110 update_pcpus:
 111         if (n_vcpus >= CHANNEL_CMDS_MAX_CPUS) {
 112                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 113                                 "0...%d\n", n_vcpus, CHANNEL_CMDS_MAX_CPUS-1);
 114                 return -1;
 115         }
 116         if (n_vcpus != vm_info->info.nrVirtCpu) {
 117                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 118                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 119                                 n_vcpus);
 120                 vm_info->info.nrVirtCpu = n_vcpus;
 121         }
 122         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 123                 mask = 0;
 124                 for (j = 0; j < global_n_host_cpus; j++) {
 125                         if (VIR_CPU_USABLE(global_cpumaps, global_maplen, i, j) > 0) {
 126                                 mask |= 1ULL << j;
 127                         }
 128                 }
 129                 rte_atomic64_set(&vm_info->pcpu_mask[i], mask);
 130         }
 131         return 0;
 132 }
 133
 134 int
 135 set_pcpus_mask(char *vm_name, unsigned int vcpu, char *core_mask)
 136 {
 137         unsigned i = 0;
 138         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 139         struct virtual_machine_info *vm_info;
 140         char mask[POWER_MGR_MAX_CPUS];
 141
 142         memcpy(mask, core_mask, POWER_MGR_MAX_CPUS);
 143
 144         if (vcpu >= CHANNEL_CMDS_MAX_CPUS) {
 145                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 146                                 vcpu, CHANNEL_CMDS_MAX_CPUS-1);
 147                 return -1;
 148         }
 149
 150         vm_info = find_domain_by_name(vm_name);
 151         if (vm_info == NULL) {
 152                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 153                 return -1;
 154         }
 155
 156         if (!virDomainIsActive(vm_info->domainPtr)) {
 157                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 158                                 " for VM '%s', VM is not active\n",
 159                                 vcpu, vm_info->name);
 160                 return -1;
 161         }
 162
 163         if (vcpu >= vm_info->info.nrVirtCpu) {
 164                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 165                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 166                 return -1;
 167         }
 168         memset(global_cpumaps, 0 , CHANNEL_CMDS_MAX_CPUS * global_maplen);
 169         for (i = 0; i < POWER_MGR_MAX_CPUS; i++) {
 170                 if (mask[i] != 1)
 171                         continue;
 172                 VIR_USE_CPU(global_cpumaps, i);
 173                 if (i >= global_n_host_cpus) {
 174                         RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 175                                         "number of CPUs(%u)\n",
 176                                         i, global_n_host_cpus);
 177                         return -1;
 178                 }
 179         }
 180         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 181                         global_maplen, flags) < 0) {
 182                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 183                                 " for VM '%s'\n", vcpu,
 184                                 vm_info->name);
 185                 return -1;
 186         }
 187         rte_spinlock_lock(&(vm_info->config_spinlock));
 188         memcpy(&vm_info->pcpu_mask[vcpu], mask, POWER_MGR_MAX_CPUS);
 189         rte_spinlock_unlock(&(vm_info->config_spinlock));
 190         return 0;
 191
 192 }
 193
 194 int
 195 set_pcpu(char *vm_name, unsigned vcpu, unsigned core_num)
 196 {
 197         char mask[POWER_MGR_MAX_CPUS];
 198
 199         memset(mask, 0, POWER_MGR_MAX_CPUS);
 200
 201         mask[core_num] = 1;
 202
 203         return set_pcpus_mask(vm_name, vcpu, mask);
 204 }
 205
 206 uint64_t
 207 get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu)
 208 {
 209         struct virtual_machine_info *vm_info =
 210                         (struct virtual_machine_info *)chan_info->priv_info;
 211
 212         if (global_hypervisor_available && (vm_info != NULL))
 213                 return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]);
 214         else
 215                 return 0;
 216 }
 217
 218 static inline int
 219 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 220 {
 221         rte_spinlock_lock(&(vm_info->config_spinlock));
 222         if (vm_info->channel_mask[channel_num] == 1) {
 223                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 224                 return 1;
 225         }
 226         rte_spinlock_unlock(&(vm_info->config_spinlock));
 227         return 0;
 228 }
 229
 230
 231
 232 static int
 233 open_non_blocking_channel(struct channel_info *info)
 234 {
 235         int ret, flags;
 236         struct sockaddr_un sock_addr;
 237         fd_set soc_fd_set;
 238         struct timeval tv;
 239
 240         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 241         if (info->fd == -1) {
 242                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 243                                 strerror(errno),
 244                                 info->channel_path);
 245                 return -1;
 246         }
 247         sock_addr.sun_family = AF_UNIX;
 248         memcpy(&sock_addr.sun_path, info->channel_path,
 249                         strlen(info->channel_path)+1);
 250
 251         /* Get current flags */
 252         flags = fcntl(info->fd, F_GETFL, 0);
 253         if (flags < 0) {
 254                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 255                                 "'%s'\n", strerror(errno), info->channel_path);
 256                 return 1;
 257         }
 258         /* Set to Non Blocking */
 259         flags |= O_NONBLOCK;
 260         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 261                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 262                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 263                 return -1;
 264         }
 265         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 266                         sizeof(sock_addr));
 267         if (ret < 0) {
 268                 /* ECONNREFUSED error is given when VM is not active */
 269                 if (errno == ECONNREFUSED) {
 270                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 271                                         "activated its endpoint to channel %s\n",
 272                                         info->channel_path);
 273                         return -1;
 274                 }
 275                 /* Wait for tv_sec if in progress */
 276                 else if (errno == EINPROGRESS) {
 277                         tv.tv_sec = 2;
 278                         tv.tv_usec = 0;
 279                         FD_ZERO(&soc_fd_set);
 280                         FD_SET(info->fd, &soc_fd_set);
 281                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 282                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 283                                                 "'%s'\n", info->channel_path);
 284                                 return -1;
 285                         }
 286                 } else {
 287                         /* Any other error */
 288                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 289                                         " for '%s'\n", strerror(errno), info->channel_path);
 290                         return -1;
 291                 }
 292         }
 293         return 0;
 294 }
 295
 296 static int
 297 open_host_channel(struct channel_info *info)
 298 {
 299         int flags;
 300
 301         info->fd = open(info->channel_path, O_RDWR | O_RSYNC);
 302         if (info->fd == -1) {
 303                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n",
 304                                 strerror(errno),
 305                                 info->channel_path);
 306                 return -1;
 307         }
 308
 309         /* Get current flags */
 310         flags = fcntl(info->fd, F_GETFL, 0);
 311         if (flags < 0) {
 312                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 313                                 "'%s'\n", strerror(errno), info->channel_path);
 314                 return 1;
 315         }
 316         /* Set to Non Blocking */
 317         flags |= O_NONBLOCK;
 318         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 319                 RTE_LOG(WARNING, CHANNEL_MANAGER,
 320                                 "Error(%s) setting non-blocking "
 321                                 "socket for '%s'\n",
 322                                 strerror(errno), info->channel_path);
 323                 return -1;
 324         }
 325         return 0;
 326 }
 327
 328 static int
 329 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 330                 struct channel_info **chan_info_dptr, unsigned channel_num)
 331 {
 332         struct channel_info *chan_info = *chan_info_dptr;
 333         struct virtual_machine_info *vm_info = *vm_info_dptr;
 334
 335         chan_info->channel_num = channel_num;
 336         chan_info->priv_info = (void *)vm_info;
 337         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 338         chan_info->type = CHANNEL_TYPE_BINARY;
 339         if (open_non_blocking_channel(chan_info) < 0) {
 340                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 341                                 "'%s' for VM '%s'\n",
 342                                 chan_info->channel_path, vm_info->name);
 343                 return -1;
 344         }
 345         if (add_channel_to_monitor(&chan_info) < 0) {
 346                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 347                                 "'%s' to epoll ctl for VM '%s'\n",
 348                                 chan_info->channel_path, vm_info->name);
 349                 return -1;
 350
 351         }
 352         rte_spinlock_lock(&(vm_info->config_spinlock));
 353         vm_info->num_channels++;
 354         vm_info->channel_mask[channel_num] = 1;
 355         vm_info->channels[channel_num] = chan_info;
 356         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 357         rte_spinlock_unlock(&(vm_info->config_spinlock));
 358         return 0;
 359 }
 360
 361 static void
 362 fifo_path(char *dst, unsigned int len)
 363 {
 364         snprintf(dst, len, "%sfifo", CHANNEL_MGR_SOCKET_PATH);
 365 }
 366
 367 static int
 368 setup_host_channel_info(struct channel_info **chan_info_dptr,
 369                 unsigned int channel_num)
 370 {
 371         struct channel_info *chan_info = *chan_info_dptr;
 372
 373         chan_info->channel_num = channel_num;
 374         chan_info->priv_info = (void *)NULL;
 375         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 376         chan_info->type = CHANNEL_TYPE_JSON;
 377
 378         fifo_path(chan_info->channel_path, sizeof(chan_info->channel_path));
 379
 380         if (open_host_channel(chan_info) < 0) {
 381                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: "
 382                                 "'%s'\n",
 383                                 chan_info->channel_path);
 384                 return -1;
 385         }
 386         if (add_channel_to_monitor(&chan_info) < 0) {
 387                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 388                                 "'%s' to epoll ctl\n",
 389                                 chan_info->channel_path);
 390                 return -1;
 391
 392         }
 393         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 394         return 0;
 395 }
 396
 397 int
 398 add_all_channels(const char *vm_name)
 399 {
 400         DIR *d;
 401         struct dirent *dir;
 402         struct virtual_machine_info *vm_info;
 403         struct channel_info *chan_info;
 404         char *token, *remaining, *tail_ptr;
 405         char socket_name[PATH_MAX];
 406         unsigned channel_num;
 407         int num_channels_enabled = 0;
 408
 409         /* verify VM exists */
 410         vm_info = find_domain_by_name(vm_name);
 411         if (vm_info == NULL) {
 412                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 413                                 " during channel discovery\n", vm_name);
 414                 return 0;
 415         }
 416         if (!virDomainIsActive(vm_info->domainPtr)) {
 417                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 418                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 419                 return 0;
 420         }
 421         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 422         if (d == NULL) {
 423                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 424                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 425                 return -1;
 426         }
 427         while ((dir = readdir(d)) != NULL) {
 428                 if (!strncmp(dir->d_name, ".", 1) ||
 429                                 !strncmp(dir->d_name, "..", 2))
 430                         continue;
 431
 432                 snprintf(socket_name, sizeof(socket_name), "%s", dir->d_name);
 433                 remaining = socket_name;
 434                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 435                 token = strsep(&remaining, ".");
 436                 if (remaining == NULL)
 437                         continue;
 438                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 439                         continue;
 440
 441                 /* remaining should contain only <channel_num> */
 442                 errno = 0;
 443                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 444                 if ((errno != 0) || (remaining[0] == '\0') ||
 445                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 446                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 447                                         "'%s' found it should be in the form of "
 448                                         "'<guest_name>.<channel_num>(decimal)'\n",
 449                                         dir->d_name);
 450                         continue;
 451                 }
 452                 if (channel_num >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 453                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 454                                         "greater than max allowable: %d, skipping '%s%s'\n",
 455                                         channel_num, CHANNEL_CMDS_MAX_VM_CHANNELS-1,
 456                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 457                         continue;
 458                 }
 459                 /* if channel has not been added previously */
 460                 if (channel_exists(vm_info, channel_num))
 461                         continue;
 462
 463                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 464                                 RTE_CACHE_LINE_SIZE);
 465                 if (chan_info == NULL) {
 466                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 467                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 468                         continue;
 469                 }
 470
 471                 snprintf(chan_info->channel_path,
 472                                 sizeof(chan_info->channel_path), "%s%s",
 473                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 474
 475                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 476                         rte_free(chan_info);
 477                         continue;
 478                 }
 479
 480                 num_channels_enabled++;
 481         }
 482         closedir(d);
 483         return num_channels_enabled;
 484 }
 485
 486 int
 487 add_channels(const char *vm_name, unsigned *channel_list,
 488                 unsigned len_channel_list)
 489 {
 490         struct virtual_machine_info *vm_info;
 491         struct channel_info *chan_info;
 492         char socket_path[PATH_MAX];
 493         unsigned i;
 494         int num_channels_enabled = 0;
 495
 496         vm_info = find_domain_by_name(vm_name);
 497         if (vm_info == NULL) {
 498                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 499                                 "not found\n", vm_name);
 500                 return 0;
 501         }
 502
 503         if (!virDomainIsActive(vm_info->domainPtr)) {
 504                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 505                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 506                 return 0;
 507         }
 508
 509         for (i = 0; i < len_channel_list; i++) {
 510
 511                 if (channel_list[i] >= CHANNEL_CMDS_MAX_VM_CHANNELS) {
 512                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 513                                                         "0...%d\n", channel_list[i],
 514                                                         CHANNEL_CMDS_MAX_VM_CHANNELS-1);
 515                         continue;
 516                 }
 517                 if (channel_exists(vm_info, channel_list[i])) {
 518                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 519                                         "'%s.%u'\n", vm_name, i);
 520                         continue;
 521                 }
 522
 523                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 524                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 525                 errno = 0;
 526                 if (access(socket_path, F_OK) < 0) {
 527                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 528                                         "%s\n", socket_path, strerror(errno));
 529                         continue;
 530                 }
 531                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 532                                 RTE_CACHE_LINE_SIZE);
 533                 if (chan_info == NULL) {
 534                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 535                                         "channel '%s'\n", socket_path);
 536                         continue;
 537                 }
 538                 snprintf(chan_info->channel_path,
 539                                 sizeof(chan_info->channel_path), "%s%s.%u",
 540                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 541                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 542                         rte_free(chan_info);
 543                         continue;
 544                 }
 545                 num_channels_enabled++;
 546
 547         }
 548         return num_channels_enabled;
 549 }
 550
 551 int
 552 add_host_channel(void)
 553 {
 554         struct channel_info *chan_info;
 555         char socket_path[PATH_MAX];
 556         int num_channels_enabled = 0;
 557         int ret;
 558
 559         fifo_path(socket_path, sizeof(socket_path));
 560
 561         ret = mkfifo(socket_path, 0660);
 562         if ((errno != EEXIST) && (ret < 0)) {
 563                 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: "
 564                                 "%s\n", socket_path, strerror(errno));
 565                 return 0;
 566         }
 567
 568         if (access(socket_path, F_OK) < 0) {
 569                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 570                                 "%s\n", socket_path, strerror(errno));
 571                 return 0;
 572         }
 573         chan_info = rte_malloc(NULL, sizeof(*chan_info), 0);
 574         if (chan_info == NULL) {
 575                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 576                                 "channel '%s'\n", socket_path);
 577                 return 0;
 578         }
 579         snprintf(chan_info->channel_path,
 580                         sizeof(chan_info->channel_path), "%s", socket_path);
 581         if (setup_host_channel_info(&chan_info, 0) < 0) {
 582                 rte_free(chan_info);
 583                 return 0;
 584         }
 585         num_channels_enabled++;
 586
 587         return num_channels_enabled;
 588 }
 589
 590 int
 591 remove_channel(struct channel_info **chan_info_dptr)
 592 {
 593         struct virtual_machine_info *vm_info;
 594         struct channel_info *chan_info = *chan_info_dptr;
 595
 596         close(chan_info->fd);
 597
 598         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 599
 600         rte_spinlock_lock(&(vm_info->config_spinlock));
 601         vm_info->channel_mask[chan_info->channel_num] = 0;
 602         vm_info->num_channels--;
 603         rte_spinlock_unlock(&(vm_info->config_spinlock));
 604
 605         rte_free(chan_info);
 606         return 0;
 607 }
 608
 609 int
 610 set_channel_status_all(const char *vm_name, enum channel_status status)
 611 {
 612         struct virtual_machine_info *vm_info;
 613         unsigned i;
 614         char mask[POWER_MGR_MAX_CPUS];
 615         int num_channels_changed = 0;
 616
 617         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 618                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 619                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 620                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 621         }
 622         vm_info = find_domain_by_name(vm_name);
 623         if (vm_info == NULL) {
 624                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 625                                 "not found\n", vm_name);
 626                 return 0;
 627         }
 628
 629         rte_spinlock_lock(&(vm_info->config_spinlock));
 630         memcpy(mask, (char *)vm_info->channel_mask, POWER_MGR_MAX_CPUS);
 631         for (i = 0; i < POWER_MGR_MAX_CPUS; i++) {
 632                 if (mask[i] != 1)
 633                         continue;
 634                 vm_info->channels[i]->status = status;
 635                 num_channels_changed++;
 636         }
 637         rte_spinlock_unlock(&(vm_info->config_spinlock));
 638         return num_channels_changed;
 639
 640 }
 641
 642 int
 643 set_channel_status(const char *vm_name, unsigned *channel_list,
 644                 unsigned len_channel_list, enum channel_status status)
 645 {
 646         struct virtual_machine_info *vm_info;
 647         unsigned i;
 648         int num_channels_changed = 0;
 649
 650         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 651                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 652                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 653                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 654         }
 655         vm_info = find_domain_by_name(vm_name);
 656         if (vm_info == NULL) {
 657                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 658                                 "not found\n", vm_name);
 659                 return 0;
 660         }
 661         for (i = 0; i < len_channel_list; i++) {
 662                 if (channel_exists(vm_info, channel_list[i])) {
 663                         rte_spinlock_lock(&(vm_info->config_spinlock));
 664                         vm_info->channels[channel_list[i]]->status = status;
 665                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 666                         num_channels_changed++;
 667                 }
 668         }
 669         return num_channels_changed;
 670 }
 671
 672 void
 673 get_all_vm(int *num_vm, int *num_vcpu)
 674 {
 675
 676         virNodeInfo node_info;
 677         virDomainPtr *domptr;
 678         int i, ii, numVcpus[MAX_VCPUS], n_vcpus;
 679         unsigned int jj;
 680         const char *vm_name;
 681         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 682                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 683         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 684
 685         if (!global_hypervisor_available)
 686                 return;
 687
 688         memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
 689         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 690                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 691                 return;
 692         }
 693
 694         /* Returns number of pcpus */
 695         global_n_host_cpus = (unsigned int)node_info.cpus;
 696
 697         /* Returns number of active domains */
 698         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 699                                         domain_flags);
 700         if (*num_vm <= 0) {
 701                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 702                 return;
 703         }
 704
 705         for (i = 0; i < *num_vm; i++) {
 706
 707                 /* Get Domain Names */
 708                 vm_name = virDomainGetName(domptr[i]);
 709                 lvm_info[i].vm_name = vm_name;
 710
 711                 /* Get Number of Vcpus */
 712                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 713
 714                 /* Get Number of VCpus & VcpuPinInfo */
 715                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 716                                 numVcpus[i], global_cpumaps,
 717                                 global_maplen, domain_flag);
 718
 719                 if ((int)n_vcpus > 0) {
 720                         *num_vcpu = n_vcpus;
 721                         lvm_info[i].num_cpus = n_vcpus;
 722                 }
 723
 724                 /* Save pcpu in use by libvirt VMs */
 725                 for (ii = 0; ii < n_vcpus; ii++) {
 726                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 727                                 if (VIR_CPU_USABLE(global_cpumaps,
 728                                                 global_maplen, ii, jj) > 0) {
 729                                         lvm_info[i].pcpus[ii] = jj;
 730                                 }
 731                         }
 732                 }
 733         }
 734 }
 735
 736 int
 737 get_info_vm(const char *vm_name, struct vm_info *info)
 738 {
 739         struct virtual_machine_info *vm_info;
 740         unsigned i, channel_num = 0;
 741         char mask[POWER_MGR_MAX_CPUS];
 742
 743         vm_info = find_domain_by_name(vm_name);
 744         if (vm_info == NULL) {
 745                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 746                 return -1;
 747         }
 748         info->status = CHANNEL_MGR_VM_ACTIVE;
 749         if (!virDomainIsActive(vm_info->domainPtr))
 750                 info->status = CHANNEL_MGR_VM_INACTIVE;
 751
 752         rte_spinlock_lock(&(vm_info->config_spinlock));
 753
 754         memcpy(mask, (char *)vm_info->channel_mask, POWER_MGR_MAX_CPUS);
 755         for (i = 0; i < POWER_MGR_MAX_CPUS; i++) {
 756                 if (mask[i] != 1)
 757                         continue;
 758                 info->channels[channel_num].channel_num = i;
 759                 memcpy(info->channels[channel_num].channel_path,
 760                                 vm_info->channels[i]->channel_path,
 761                                 UNIX_PATH_MAX);
 762                 info->channels[channel_num].status =
 763                                 vm_info->channels[i]->status;
 764                 info->channels[channel_num].fd =
 765                                 vm_info->channels[i]->fd;
 766                 channel_num++;
 767         }
 768
 769         info->num_channels = channel_num;
 770         info->num_vcpus = vm_info->info.nrVirtCpu;
 771         rte_spinlock_unlock(&(vm_info->config_spinlock));
 772
 773         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 774         for (i = 0; i < info->num_vcpus; i++) {
 775                 info->pcpu_mask[i] = rte_atomic64_read(&vm_info->pcpu_mask[i]);
 776         }
 777         return 0;
 778 }
 779
 780 int
 781 add_vm(const char *vm_name)
 782 {
 783         struct virtual_machine_info *new_domain;
 784         virDomainPtr dom_ptr;
 785         int i;
 786
 787         if (find_domain_by_name(vm_name) != NULL) {
 788                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 789                                 "already exists\n", vm_name);
 790                 return -1;
 791         }
 792
 793         if (global_vir_conn_ptr == NULL) {
 794                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 795                 return -1;
 796         }
 797         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 798         if (dom_ptr == NULL) {
 799                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 800                                 "VM '%s' not found\n", vm_name);
 801                 return -1;
 802         }
 803
 804         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 805                         RTE_CACHE_LINE_SIZE);
 806         if (new_domain == NULL) {
 807                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 808                                 "info\n");
 809                 return -1;
 810         }
 811         new_domain->domainPtr = dom_ptr;
 812         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 813                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 814                 rte_free(new_domain);
 815                 return -1;
 816         }
 817         if (new_domain->info.nrVirtCpu > CHANNEL_CMDS_MAX_CPUS) {
 818                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 819                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 820                                 CHANNEL_CMDS_MAX_CPUS);
 821                 rte_free(new_domain);
 822                 return -1;
 823         }
 824
 825         for (i = 0; i < CHANNEL_CMDS_MAX_CPUS; i++) {
 826                 rte_atomic64_init(&new_domain->pcpu_mask[i]);
 827         }
 828         if (update_pcpus_mask(new_domain) < 0) {
 829                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 830                 rte_free(new_domain);
 831                 return -1;
 832         }
 833         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 834         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 835         memset(new_domain->channel_mask, 0, POWER_MGR_MAX_CPUS);
 836         new_domain->num_channels = 0;
 837
 838         if (!virDomainIsActive(dom_ptr))
 839                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 840         else
 841                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 842
 843         rte_spinlock_init(&(new_domain->config_spinlock));
 844         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 845         return 0;
 846 }
 847
 848 int
 849 remove_vm(const char *vm_name)
 850 {
 851         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 852
 853         if (vm_info == NULL) {
 854                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 855                                 "not found\n", vm_name);
 856                 return -1;
 857         }
 858         rte_spinlock_lock(&vm_info->config_spinlock);
 859         if (vm_info->num_channels != 0) {
 860                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 861                                 "%"PRId8" channels still active\n",
 862                                 vm_name, vm_info->num_channels);
 863                 rte_spinlock_unlock(&vm_info->config_spinlock);
 864                 return -1;
 865         }
 866         LIST_REMOVE(vm_info, vms_info);
 867         rte_spinlock_unlock(&vm_info->config_spinlock);
 868         rte_free(vm_info);
 869         return 0;
 870 }
 871
 872 static void
 873 disconnect_hypervisor(void)
 874 {
 875         if (global_vir_conn_ptr != NULL) {
 876                 virConnectClose(global_vir_conn_ptr);
 877                 global_vir_conn_ptr = NULL;
 878         }
 879 }
 880
 881 static int
 882 connect_hypervisor(const char *path)
 883 {
 884         if (global_vir_conn_ptr != NULL) {
 885                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 886                                 "already established\n", path);
 887                 return -1;
 888         }
 889         global_vir_conn_ptr = virConnectOpen(path);
 890         if (global_vir_conn_ptr == NULL) {
 891                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 892                                 "Hypervisor '%s'\n", path);
 893                 return -1;
 894         }
 895         return 0;
 896 }
 897 int
 898 channel_manager_init(const char *path __rte_unused)
 899 {
 900         virNodeInfo info;
 901
 902         LIST_INIT(&vm_list_head);
 903         if (connect_hypervisor(path) < 0) {
 904                 global_n_host_cpus = 64;
 905                 global_hypervisor_available = 0;
 906                 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 907         } else {
 908                 global_hypervisor_available = 1;
 909
 910                 global_maplen = VIR_CPU_MAPLEN(CHANNEL_CMDS_MAX_CPUS);
 911
 912                 global_vircpuinfo = rte_zmalloc(NULL,
 913                                 sizeof(*global_vircpuinfo) *
 914                                 CHANNEL_CMDS_MAX_CPUS, RTE_CACHE_LINE_SIZE);
 915                 if (global_vircpuinfo == NULL) {
 916                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 917                         goto error;
 918                 }
 919                 global_cpumaps = rte_zmalloc(NULL,
 920                                 CHANNEL_CMDS_MAX_CPUS * global_maplen,
 921                                 RTE_CACHE_LINE_SIZE);
 922                 if (global_cpumaps == NULL)
 923                         goto error;
 924
 925                 if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 926                         RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 927                         goto error;
 928                 }
 929                 global_n_host_cpus = (unsigned int)info.cpus;
 930         }
 931
 932
 933
 934         if (global_n_host_cpus > CHANNEL_CMDS_MAX_CPUS) {
 935                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 936                                 "maximum of %u. No cores over %u should be used.\n",
 937                                 global_n_host_cpus, CHANNEL_CMDS_MAX_CPUS,
 938                                 CHANNEL_CMDS_MAX_CPUS - 1);
 939                 global_n_host_cpus = CHANNEL_CMDS_MAX_CPUS;
 940         }
 941
 942         return 0;
 943 error:
 944         if (global_hypervisor_available)
 945                 disconnect_hypervisor();
 946         return -1;
 947 }
 948
 949 void
 950 channel_manager_exit(void)
 951 {
 952         unsigned i;
 953         char mask[POWER_MGR_MAX_CPUS];
 954         struct virtual_machine_info *vm_info;
 955
 956         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
 957
 958                 rte_spinlock_lock(&(vm_info->config_spinlock));
 959
 960                 memcpy(mask, (char *)vm_info->channel_mask, POWER_MGR_MAX_CPUS);
 961                 for (i = 0; i < POWER_MGR_MAX_CPUS; i++) {
 962                         if (mask[i] != 1)
 963                                 continue;
 964                         remove_channel_from_monitor(
 965                                         vm_info->channels[i]);
 966                         close(vm_info->channels[i]->fd);
 967                         rte_free(vm_info->channels[i]);
 968                 }
 969                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 970
 971                 LIST_REMOVE(vm_info, vms_info);
 972                 rte_free(vm_info);
 973         }
 974
 975         if (global_hypervisor_available) {
 976                 /* Only needed if hypervisor available */
 977                 rte_free(global_cpumaps);
 978                 rte_free(global_vircpuinfo);
 979                 disconnect_hypervisor();
 980         }
 981 }