examples/vm_power_manager/channel_manager.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <fcntl.h>
   8 #include <unistd.h>
   9 #include <inttypes.h>
  10 #include <dirent.h>
  11 #include <errno.h>
  12
  13 #include <sys/queue.h>
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include <sys/socket.h>
  17 #include <sys/select.h>
  18
  19 #include <rte_string_fns.h>
  20 #include <rte_malloc.h>
  21 #include <rte_memory.h>
  22 #include <rte_mempool.h>
  23 #include <rte_log.h>
  24 #include <rte_atomic.h>
  25 #include <rte_spinlock.h>
  26
  27 #include <libvirt/libvirt.h>
  28
  29 #include "channel_manager.h"
  30 #include "channel_commands.h"
  31 #include "channel_monitor.h"
  32 #include "power_manager.h"
  33
  34
  35 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  36
  37 struct libvirt_vm_info lvm_info[MAX_CLIENTS];
  38
  39 /* Global pointer to libvirt connection */
  40 static virConnectPtr global_vir_conn_ptr;
  41
  42 static unsigned char *global_cpumaps;
  43 static virVcpuInfo *global_vircpuinfo;
  44 static size_t global_maplen;
  45
  46 static unsigned int global_n_host_cpus;
  47 static bool global_hypervisor_available;
  48
  49 /*
  50  * Represents a single Virtual Machine
  51  */
  52 struct virtual_machine_info {
  53         char name[CHANNEL_MGR_MAX_NAME_LEN];
  54         uint16_t pcpu_map[RTE_MAX_LCORE];
  55         struct channel_info *channels[RTE_MAX_LCORE];
  56         char channel_mask[RTE_MAX_LCORE];
  57         uint8_t num_channels;
  58         enum vm_status status;
  59         virDomainPtr domainPtr;
  60         virDomainInfo info;
  61         rte_spinlock_t config_spinlock;
  62         int allow_query;
  63         LIST_ENTRY(virtual_machine_info) vms_info;
  64 };
  65
  66 LIST_HEAD(, virtual_machine_info) vm_list_head;
  67
  68 static struct virtual_machine_info *
  69 find_domain_by_name(const char *name)
  70 {
  71         struct virtual_machine_info *info;
  72         LIST_FOREACH(info, &vm_list_head, vms_info) {
  73                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
  74                         return info;
  75         }
  76         return NULL;
  77 }
  78
  79 static int
  80 update_pcpus_mask(struct virtual_machine_info *vm_info)
  81 {
  82         virVcpuInfoPtr cpuinfo;
  83         unsigned i, j;
  84         int n_vcpus;
  85
  86         memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen);
  87
  88         if (!virDomainIsActive(vm_info->domainPtr)) {
  89                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
  90                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
  91                                 VIR_DOMAIN_AFFECT_CONFIG);
  92                 if (n_vcpus < 0) {
  93                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
  94                                         "in-active VM '%s'\n", vm_info->name);
  95                         return -1;
  96                 }
  97                 goto update_pcpus;
  98         }
  99
 100         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
 101                         RTE_MAX_LCORE);
 102
 103         cpuinfo = global_vircpuinfo;
 104
 105         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 106                         RTE_MAX_LCORE, global_cpumaps, global_maplen);
 107         if (n_vcpus < 0) {
 108                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 109                                 "active VM '%s'\n", vm_info->name);
 110                 return -1;
 111         }
 112 update_pcpus:
 113         if (n_vcpus >= RTE_MAX_LCORE) {
 114                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 115                                 "0...%d\n", n_vcpus, RTE_MAX_LCORE-1);
 116                 return -1;
 117         }
 118         if (n_vcpus != vm_info->info.nrVirtCpu) {
 119                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 120                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 121                                 n_vcpus);
 122                 vm_info->info.nrVirtCpu = n_vcpus;
 123         }
 124         rte_spinlock_lock(&(vm_info->config_spinlock));
 125         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 126                 for (j = 0; j < global_n_host_cpus; j++) {
 127                         if (VIR_CPU_USABLE(global_cpumaps,
 128                                         global_maplen, i, j) <= 0)
 129                                 continue;
 130                         vm_info->pcpu_map[i] = j;
 131                 }
 132         }
 133         rte_spinlock_unlock(&(vm_info->config_spinlock));
 134         return 0;
 135 }
 136
 137 int
 138 set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu)
 139 {
 140         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 141         struct virtual_machine_info *vm_info;
 142
 143         if (vcpu >= RTE_MAX_LCORE) {
 144                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 145                                 vcpu, RTE_MAX_LCORE-1);
 146                 return -1;
 147         }
 148
 149         vm_info = find_domain_by_name(vm_name);
 150         if (vm_info == NULL) {
 151                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 152                 return -1;
 153         }
 154
 155         if (!virDomainIsActive(vm_info->domainPtr)) {
 156                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 157                                 " for VM '%s', VM is not active\n",
 158                                 vcpu, vm_info->name);
 159                 return -1;
 160         }
 161
 162         if (vcpu >= vm_info->info.nrVirtCpu) {
 163                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 164                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 165                 return -1;
 166         }
 167         memset(global_cpumaps, 0, RTE_MAX_LCORE * global_maplen);
 168
 169         VIR_USE_CPU(global_cpumaps, pcpu);
 170
 171         if (pcpu >= global_n_host_cpus) {
 172                 RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 173                                 "number of CPUs(%u)\n",
 174                                 pcpu, global_n_host_cpus);
 175                 return -1;
 176         }
 177
 178         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 179                         global_maplen, flags) < 0) {
 180                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 181                                 " for VM '%s'\n", vcpu,
 182                                 vm_info->name);
 183                 return -1;
 184         }
 185         rte_spinlock_lock(&(vm_info->config_spinlock));
 186         vm_info->pcpu_map[vcpu] = pcpu;
 187         rte_spinlock_unlock(&(vm_info->config_spinlock));
 188         return 0;
 189 }
 190
 191 uint16_t
 192 get_pcpu(struct channel_info *chan_info, unsigned int vcpu)
 193 {
 194         struct virtual_machine_info *vm_info =
 195                         (struct virtual_machine_info *)chan_info->priv_info;
 196
 197         if (global_hypervisor_available && (vm_info != NULL)) {
 198                 uint16_t pcpu;
 199                 rte_spinlock_lock(&(vm_info->config_spinlock));
 200                 pcpu = vm_info->pcpu_map[vcpu];
 201                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 202                 return pcpu;
 203         } else
 204                 return 0;
 205 }
 206
 207 static inline int
 208 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 209 {
 210         rte_spinlock_lock(&(vm_info->config_spinlock));
 211         if (vm_info->channel_mask[channel_num] == 1) {
 212                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 213                 return 1;
 214         }
 215         rte_spinlock_unlock(&(vm_info->config_spinlock));
 216         return 0;
 217 }
 218
 219
 220
 221 static int
 222 open_non_blocking_channel(struct channel_info *info)
 223 {
 224         int ret, flags;
 225         struct sockaddr_un sock_addr;
 226         fd_set soc_fd_set;
 227         struct timeval tv;
 228
 229         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 230         if (info->fd < 0) {
 231                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 232                                 strerror(errno),
 233                                 info->channel_path);
 234                 return -1;
 235         }
 236         sock_addr.sun_family = AF_UNIX;
 237         memcpy(&sock_addr.sun_path, info->channel_path,
 238                         strlen(info->channel_path)+1);
 239
 240         /* Get current flags */
 241         flags = fcntl(info->fd, F_GETFL, 0);
 242         if (flags < 0) {
 243                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 244                                 "'%s'\n", strerror(errno), info->channel_path);
 245                 return 1;
 246         }
 247         /* Set to Non Blocking */
 248         flags |= O_NONBLOCK;
 249         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 250                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 251                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 252                 return -1;
 253         }
 254         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 255                         sizeof(sock_addr));
 256         if (ret < 0) {
 257                 /* ECONNREFUSED error is given when VM is not active */
 258                 if (errno == ECONNREFUSED) {
 259                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 260                                         "activated its endpoint to channel %s\n",
 261                                         info->channel_path);
 262                         return -1;
 263                 }
 264                 /* Wait for tv_sec if in progress */
 265                 else if (errno == EINPROGRESS) {
 266                         tv.tv_sec = 2;
 267                         tv.tv_usec = 0;
 268                         FD_ZERO(&soc_fd_set);
 269                         FD_SET(info->fd, &soc_fd_set);
 270                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 271                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 272                                                 "'%s'\n", info->channel_path);
 273                                 return -1;
 274                         }
 275                 } else {
 276                         /* Any other error */
 277                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 278                                         " for '%s'\n", strerror(errno), info->channel_path);
 279                         return -1;
 280                 }
 281         }
 282         return 0;
 283 }
 284
 285 static int
 286 open_host_channel(struct channel_info *info)
 287 {
 288         int flags;
 289
 290         info->fd = open(info->channel_path, O_RDWR | O_RSYNC);
 291         if (info->fd < 0) {
 292                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n",
 293                                 strerror(errno),
 294                                 info->channel_path);
 295                 return -1;
 296         }
 297
 298         /* Get current flags */
 299         flags = fcntl(info->fd, F_GETFL, 0);
 300         if (flags < 0) {
 301                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 302                                 "'%s'\n", strerror(errno), info->channel_path);
 303                 return 1;
 304         }
 305         /* Set to Non Blocking */
 306         flags |= O_NONBLOCK;
 307         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 308                 RTE_LOG(WARNING, CHANNEL_MANAGER,
 309                                 "Error(%s) setting non-blocking "
 310                                 "socket for '%s'\n",
 311                                 strerror(errno), info->channel_path);
 312                 return -1;
 313         }
 314         return 0;
 315 }
 316
 317 static int
 318 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 319                 struct channel_info **chan_info_dptr, unsigned channel_num)
 320 {
 321         struct channel_info *chan_info = *chan_info_dptr;
 322         struct virtual_machine_info *vm_info = *vm_info_dptr;
 323
 324         chan_info->channel_num = channel_num;
 325         chan_info->priv_info = (void *)vm_info;
 326         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 327         chan_info->type = CHANNEL_TYPE_BINARY;
 328         if (open_non_blocking_channel(chan_info) < 0) {
 329                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 330                                 "'%s' for VM '%s'\n",
 331                                 chan_info->channel_path, vm_info->name);
 332                 return -1;
 333         }
 334         if (add_channel_to_monitor(&chan_info) < 0) {
 335                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 336                                 "'%s' to epoll ctl for VM '%s'\n",
 337                                 chan_info->channel_path, vm_info->name);
 338                 return -1;
 339
 340         }
 341         rte_spinlock_lock(&(vm_info->config_spinlock));
 342         vm_info->num_channels++;
 343         vm_info->channel_mask[channel_num] = 1;
 344         vm_info->channels[channel_num] = chan_info;
 345         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 346         rte_spinlock_unlock(&(vm_info->config_spinlock));
 347         return 0;
 348 }
 349
 350 static int
 351 fifo_path(char *dst, unsigned int len, unsigned int id)
 352 {
 353         int cnt;
 354
 355         cnt = snprintf(dst, len, "%s%s%d", CHANNEL_MGR_SOCKET_PATH,
 356                         CHANNEL_MGR_FIFO_PATTERN_NAME, id);
 357
 358         if ((cnt < 0) || (cnt > (int)len - 1)) {
 359                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not create proper "
 360                         "string for fifo path\n");
 361
 362                 return -1;
 363         }
 364
 365         return 0;
 366 }
 367
 368 static int
 369 setup_host_channel_info(struct channel_info **chan_info_dptr,
 370                 unsigned int channel_num)
 371 {
 372         struct channel_info *chan_info = *chan_info_dptr;
 373
 374         chan_info->channel_num = channel_num;
 375         chan_info->priv_info = (void *)NULL;
 376         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 377         chan_info->type = CHANNEL_TYPE_JSON;
 378
 379         if (open_host_channel(chan_info) < 0) {
 380                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: "
 381                                 "'%s'\n",
 382                                 chan_info->channel_path);
 383                 return -1;
 384         }
 385         if (add_channel_to_monitor(&chan_info) < 0) {
 386                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 387                                 "'%s' to epoll ctl\n",
 388                                 chan_info->channel_path);
 389                 return -1;
 390
 391         }
 392         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 393         return 0;
 394 }
 395
 396 int
 397 add_all_channels(const char *vm_name)
 398 {
 399         DIR *d;
 400         struct dirent *dir;
 401         struct virtual_machine_info *vm_info;
 402         struct channel_info *chan_info;
 403         char *token, *remaining, *tail_ptr;
 404         char socket_name[PATH_MAX];
 405         unsigned channel_num;
 406         int num_channels_enabled = 0;
 407
 408         /* verify VM exists */
 409         vm_info = find_domain_by_name(vm_name);
 410         if (vm_info == NULL) {
 411                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 412                                 " during channel discovery\n", vm_name);
 413                 return 0;
 414         }
 415         if (!virDomainIsActive(vm_info->domainPtr)) {
 416                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 417                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 418                 return 0;
 419         }
 420         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 421         if (d == NULL) {
 422                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 423                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 424                 return -1;
 425         }
 426         while ((dir = readdir(d)) != NULL) {
 427                 if (!strncmp(dir->d_name, ".", 1) ||
 428                                 !strncmp(dir->d_name, "..", 2))
 429                         continue;
 430
 431                 strlcpy(socket_name, dir->d_name, sizeof(socket_name));
 432                 remaining = socket_name;
 433                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 434                 token = strsep(&remaining, ".");
 435                 if (remaining == NULL)
 436                         continue;
 437                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 438                         continue;
 439
 440                 /* remaining should contain only <channel_num> */
 441                 errno = 0;
 442                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 443                 if ((errno != 0) || (remaining[0] == '\0') ||
 444                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 445                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 446                                         "'%s' found it should be in the form of "
 447                                         "'<guest_name>.<channel_num>(decimal)'\n",
 448                                         dir->d_name);
 449                         continue;
 450                 }
 451                 if (channel_num >= RTE_MAX_LCORE) {
 452                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 453                                         "greater than max allowable: %d, skipping '%s%s'\n",
 454                                         channel_num, RTE_MAX_LCORE-1,
 455                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 456                         continue;
 457                 }
 458                 /* if channel has not been added previously */
 459                 if (channel_exists(vm_info, channel_num))
 460                         continue;
 461
 462                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 463                                 RTE_CACHE_LINE_SIZE);
 464                 if (chan_info == NULL) {
 465                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 466                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 467                         continue;
 468                 }
 469
 470                 snprintf(chan_info->channel_path,
 471                                 sizeof(chan_info->channel_path), "%s%s",
 472                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 473
 474                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 475                         rte_free(chan_info);
 476                         continue;
 477                 }
 478
 479                 num_channels_enabled++;
 480         }
 481         closedir(d);
 482         return num_channels_enabled;
 483 }
 484
 485 int
 486 add_channels(const char *vm_name, unsigned *channel_list,
 487                 unsigned len_channel_list)
 488 {
 489         struct virtual_machine_info *vm_info;
 490         struct channel_info *chan_info;
 491         char socket_path[PATH_MAX];
 492         unsigned i;
 493         int num_channels_enabled = 0;
 494
 495         vm_info = find_domain_by_name(vm_name);
 496         if (vm_info == NULL) {
 497                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 498                                 "not found\n", vm_name);
 499                 return 0;
 500         }
 501
 502         if (!virDomainIsActive(vm_info->domainPtr)) {
 503                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 504                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 505                 return 0;
 506         }
 507
 508         for (i = 0; i < len_channel_list; i++) {
 509
 510                 if (channel_list[i] >= RTE_MAX_LCORE) {
 511                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 512                                                         "0...%d\n", channel_list[i],
 513                                                         RTE_MAX_LCORE-1);
 514                         continue;
 515                 }
 516                 if (channel_exists(vm_info, channel_list[i])) {
 517                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 518                                         "'%s.%u'\n", vm_name, i);
 519                         continue;
 520                 }
 521
 522                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 523                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 524                 errno = 0;
 525                 if (access(socket_path, F_OK) < 0) {
 526                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 527                                         "%s\n", socket_path, strerror(errno));
 528                         continue;
 529                 }
 530                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 531                                 RTE_CACHE_LINE_SIZE);
 532                 if (chan_info == NULL) {
 533                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 534                                         "channel '%s'\n", socket_path);
 535                         continue;
 536                 }
 537                 snprintf(chan_info->channel_path,
 538                                 sizeof(chan_info->channel_path), "%s%s.%u",
 539                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 540                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 541                         rte_free(chan_info);
 542                         continue;
 543                 }
 544                 num_channels_enabled++;
 545
 546         }
 547         return num_channels_enabled;
 548 }
 549
 550 int
 551 add_host_channels(void)
 552 {
 553         struct channel_info *chan_info;
 554         char socket_path[PATH_MAX];
 555         int num_channels_enabled = 0;
 556         int ret;
 557         struct core_info *ci;
 558         struct channel_info *chan_infos[RTE_MAX_LCORE];
 559         int i;
 560
 561         for (i = 0; i < RTE_MAX_LCORE; i++)
 562                 chan_infos[i] = NULL;
 563
 564         ci = get_core_info();
 565         if (ci == NULL) {
 566                 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot allocate memory for core_info\n");
 567                 return 0;
 568         }
 569
 570         for (i = 0; i < ci->core_count; i++) {
 571                 if (ci->cd[i].global_enabled_cpus == 0)
 572                         continue;
 573
 574                 ret = fifo_path(socket_path, sizeof(socket_path), i);
 575                 if (ret < 0)
 576                         goto error;
 577
 578                 ret = mkfifo(socket_path, 0660);
 579                 RTE_LOG(DEBUG, CHANNEL_MANAGER, "TRY CREATE fifo '%s'\n",
 580                         socket_path);
 581                 if ((errno != EEXIST) && (ret < 0)) {
 582                         RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: "
 583                                         "%s\n", socket_path, strerror(errno));
 584                         goto error;
 585                 }
 586                 chan_info = rte_malloc(NULL, sizeof(*chan_info), 0);
 587                 if (chan_info == NULL) {
 588                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 589                                         "channel '%s'\n", socket_path);
 590                         goto error;
 591                 }
 592                 chan_infos[i] = chan_info;
 593                 strlcpy(chan_info->channel_path, socket_path,
 594                                 sizeof(chan_info->channel_path));
 595
 596                 if (setup_host_channel_info(&chan_info, i) < 0) {
 597                         rte_free(chan_info);
 598                         chan_infos[i] = NULL;
 599                         goto error;
 600                 }
 601                 num_channels_enabled++;
 602         }
 603
 604         return num_channels_enabled;
 605 error:
 606         /* Clean up the channels opened before we hit an error. */
 607         for (i = 0; i < ci->core_count; i++) {
 608                 if (chan_infos[i] != NULL) {
 609                         remove_channel_from_monitor(chan_infos[i]);
 610                         close(chan_infos[i]->fd);
 611                         rte_free(chan_infos[i]);
 612                 }
 613         }
 614         return 0;
 615 }
 616
 617 int
 618 remove_channel(struct channel_info **chan_info_dptr)
 619 {
 620         struct virtual_machine_info *vm_info;
 621         struct channel_info *chan_info = *chan_info_dptr;
 622
 623         close(chan_info->fd);
 624
 625         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 626
 627         rte_spinlock_lock(&(vm_info->config_spinlock));
 628         vm_info->channel_mask[chan_info->channel_num] = 0;
 629         vm_info->num_channels--;
 630         rte_spinlock_unlock(&(vm_info->config_spinlock));
 631
 632         rte_free(chan_info);
 633         return 0;
 634 }
 635
 636 int
 637 set_channel_status_all(const char *vm_name, enum channel_status status)
 638 {
 639         struct virtual_machine_info *vm_info;
 640         unsigned i;
 641         char mask[RTE_MAX_LCORE];
 642         int num_channels_changed = 0;
 643
 644         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 645                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 646                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 647                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 648         }
 649         vm_info = find_domain_by_name(vm_name);
 650         if (vm_info == NULL) {
 651                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 652                                 "not found\n", vm_name);
 653                 return 0;
 654         }
 655
 656         rte_spinlock_lock(&(vm_info->config_spinlock));
 657         memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
 658         for (i = 0; i < RTE_MAX_LCORE; i++) {
 659                 if (mask[i] != 1)
 660                         continue;
 661                 vm_info->channels[i]->status = status;
 662                 num_channels_changed++;
 663         }
 664         rte_spinlock_unlock(&(vm_info->config_spinlock));
 665         return num_channels_changed;
 666
 667 }
 668
 669 int
 670 set_channel_status(const char *vm_name, unsigned *channel_list,
 671                 unsigned len_channel_list, enum channel_status status)
 672 {
 673         struct virtual_machine_info *vm_info;
 674         unsigned i;
 675         int num_channels_changed = 0;
 676
 677         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 678                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 679                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 680                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 681         }
 682         vm_info = find_domain_by_name(vm_name);
 683         if (vm_info == NULL) {
 684                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 685                                 "not found\n", vm_name);
 686                 return 0;
 687         }
 688         for (i = 0; i < len_channel_list; i++) {
 689                 if (channel_exists(vm_info, channel_list[i])) {
 690                         rte_spinlock_lock(&(vm_info->config_spinlock));
 691                         vm_info->channels[channel_list[i]]->status = status;
 692                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 693                         num_channels_changed++;
 694                 }
 695         }
 696         return num_channels_changed;
 697 }
 698
 699 void
 700 get_all_vm(int *num_vm, int *num_vcpu)
 701 {
 702
 703         virNodeInfo node_info;
 704         virDomainPtr *domptr;
 705         int i, ii, numVcpus[MAX_VCPUS], n_vcpus;
 706         unsigned int jj;
 707         const char *vm_name;
 708         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 709                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 710         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 711
 712         if (!global_hypervisor_available)
 713                 return;
 714
 715         memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen);
 716         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 717                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 718                 return;
 719         }
 720
 721         /* Returns number of pcpus */
 722         global_n_host_cpus = (unsigned int)node_info.cpus;
 723
 724         /* Returns number of active domains */
 725         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 726                                         domain_flags);
 727         if (*num_vm <= 0) {
 728                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 729                 return;
 730         }
 731
 732         for (i = 0; i < *num_vm; i++) {
 733
 734                 /* Get Domain Names */
 735                 vm_name = virDomainGetName(domptr[i]);
 736                 lvm_info[i].vm_name = vm_name;
 737
 738                 /* Get Number of Vcpus */
 739                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 740
 741                 /* Get Number of VCpus & VcpuPinInfo */
 742                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 743                                 numVcpus[i], global_cpumaps,
 744                                 global_maplen, domain_flag);
 745
 746                 if ((int)n_vcpus > 0) {
 747                         *num_vcpu = n_vcpus;
 748                         lvm_info[i].num_cpus = n_vcpus;
 749                 }
 750
 751                 /* Save pcpu in use by libvirt VMs */
 752                 for (ii = 0; ii < n_vcpus; ii++) {
 753                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 754                                 if (VIR_CPU_USABLE(global_cpumaps,
 755                                                 global_maplen, ii, jj) > 0) {
 756                                         lvm_info[i].pcpus[ii] = jj;
 757                                 }
 758                         }
 759                 }
 760         }
 761 }
 762
 763 int
 764 get_info_vm(const char *vm_name, struct vm_info *info)
 765 {
 766         struct virtual_machine_info *vm_info;
 767         unsigned i, channel_num = 0;
 768         char mask[RTE_MAX_LCORE];
 769
 770         vm_info = find_domain_by_name(vm_name);
 771         if (vm_info == NULL) {
 772                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 773                 return -1;
 774         }
 775         info->status = CHANNEL_MGR_VM_ACTIVE;
 776         if (!virDomainIsActive(vm_info->domainPtr))
 777                 info->status = CHANNEL_MGR_VM_INACTIVE;
 778
 779         rte_spinlock_lock(&(vm_info->config_spinlock));
 780
 781         memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
 782         for (i = 0; i < RTE_MAX_LCORE; i++) {
 783                 if (mask[i] != 1)
 784                         continue;
 785                 info->channels[channel_num].channel_num = i;
 786                 memcpy(info->channels[channel_num].channel_path,
 787                                 vm_info->channels[i]->channel_path,
 788                                 UNIX_PATH_MAX);
 789                 info->channels[channel_num].status =
 790                                 vm_info->channels[i]->status;
 791                 info->channels[channel_num].fd =
 792                                 vm_info->channels[i]->fd;
 793                 channel_num++;
 794         }
 795
 796         info->allow_query = vm_info->allow_query;
 797         info->num_channels = channel_num;
 798         info->num_vcpus = vm_info->info.nrVirtCpu;
 799         rte_spinlock_unlock(&(vm_info->config_spinlock));
 800
 801         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 802         rte_spinlock_lock(&(vm_info->config_spinlock));
 803         for (i = 0; i < info->num_vcpus; i++) {
 804                 info->pcpu_map[i] = vm_info->pcpu_map[i];
 805         }
 806         rte_spinlock_unlock(&(vm_info->config_spinlock));
 807         return 0;
 808 }
 809
 810 int
 811 add_vm(const char *vm_name)
 812 {
 813         struct virtual_machine_info *new_domain;
 814         virDomainPtr dom_ptr;
 815         int i;
 816
 817         if (find_domain_by_name(vm_name) != NULL) {
 818                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 819                                 "already exists\n", vm_name);
 820                 return -1;
 821         }
 822
 823         if (global_vir_conn_ptr == NULL) {
 824                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 825                 return -1;
 826         }
 827         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 828         if (dom_ptr == NULL) {
 829                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 830                                 "VM '%s' not found\n", vm_name);
 831                 return -1;
 832         }
 833
 834         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 835                         RTE_CACHE_LINE_SIZE);
 836         if (new_domain == NULL) {
 837                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 838                                 "info\n");
 839                 return -1;
 840         }
 841         new_domain->domainPtr = dom_ptr;
 842         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 843                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 844                 rte_free(new_domain);
 845                 return -1;
 846         }
 847         if (new_domain->info.nrVirtCpu > RTE_MAX_LCORE) {
 848                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 849                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 850                                 RTE_MAX_LCORE);
 851                 rte_free(new_domain);
 852                 return -1;
 853         }
 854
 855         for (i = 0; i < RTE_MAX_LCORE; i++)
 856                 new_domain->pcpu_map[i] = 0;
 857
 858         if (update_pcpus_mask(new_domain) < 0) {
 859                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 860                 rte_free(new_domain);
 861                 return -1;
 862         }
 863         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 864         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 865         memset(new_domain->channel_mask, 0, RTE_MAX_LCORE);
 866         new_domain->num_channels = 0;
 867
 868         if (!virDomainIsActive(dom_ptr))
 869                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 870         else
 871                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 872
 873         new_domain->allow_query = 0;
 874         rte_spinlock_init(&(new_domain->config_spinlock));
 875         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 876         return 0;
 877 }
 878
 879 int
 880 remove_vm(const char *vm_name)
 881 {
 882         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 883
 884         if (vm_info == NULL) {
 885                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 886                                 "not found\n", vm_name);
 887                 return -1;
 888         }
 889         rte_spinlock_lock(&vm_info->config_spinlock);
 890         if (vm_info->num_channels != 0) {
 891                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 892                                 "%"PRId8" channels still active\n",
 893                                 vm_name, vm_info->num_channels);
 894                 rte_spinlock_unlock(&vm_info->config_spinlock);
 895                 return -1;
 896         }
 897         LIST_REMOVE(vm_info, vms_info);
 898         rte_spinlock_unlock(&vm_info->config_spinlock);
 899         rte_free(vm_info);
 900         return 0;
 901 }
 902
 903 int
 904 set_query_status(char *vm_name,
 905                 bool allow_query)
 906 {
 907         struct virtual_machine_info *vm_info;
 908
 909         vm_info = find_domain_by_name(vm_name);
 910         if (vm_info == NULL) {
 911                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 912                 return -1;
 913         }
 914         rte_spinlock_lock(&(vm_info->config_spinlock));
 915         vm_info->allow_query = allow_query ? 1 : 0;
 916         rte_spinlock_unlock(&(vm_info->config_spinlock));
 917         return 0;
 918 }
 919
 920 static void
 921 disconnect_hypervisor(void)
 922 {
 923         if (global_vir_conn_ptr != NULL) {
 924                 virConnectClose(global_vir_conn_ptr);
 925                 global_vir_conn_ptr = NULL;
 926         }
 927 }
 928
 929 static int
 930 connect_hypervisor(const char *path)
 931 {
 932         if (global_vir_conn_ptr != NULL) {
 933                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 934                                 "already established\n", path);
 935                 return -1;
 936         }
 937         global_vir_conn_ptr = virConnectOpen(path);
 938         if (global_vir_conn_ptr == NULL) {
 939                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 940                                 "Hypervisor '%s'\n", path);
 941                 return -1;
 942         }
 943         return 0;
 944 }
 945 int
 946 channel_manager_init(const char *path __rte_unused)
 947 {
 948         virNodeInfo info;
 949
 950         LIST_INIT(&vm_list_head);
 951         if (connect_hypervisor(path) < 0) {
 952                 global_n_host_cpus = 64;
 953                 global_hypervisor_available = 0;
 954                 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 955         } else {
 956                 global_hypervisor_available = 1;
 957
 958                 global_maplen = VIR_CPU_MAPLEN(RTE_MAX_LCORE);
 959
 960                 global_vircpuinfo = rte_zmalloc(NULL,
 961                                 sizeof(*global_vircpuinfo) *
 962                                 RTE_MAX_LCORE, RTE_CACHE_LINE_SIZE);
 963                 if (global_vircpuinfo == NULL) {
 964                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 965                         goto error;
 966                 }
 967                 global_cpumaps = rte_zmalloc(NULL,
 968                                 RTE_MAX_LCORE * global_maplen,
 969                                 RTE_CACHE_LINE_SIZE);
 970                 if (global_cpumaps == NULL)
 971                         goto error;
 972
 973                 if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 974                         RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 975                         goto error;
 976                 }
 977                 global_n_host_cpus = (unsigned int)info.cpus;
 978         }
 979
 980
 981
 982         if (global_n_host_cpus > RTE_MAX_LCORE) {
 983                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 984                                 "maximum of %u. No cores over %u should be used.\n",
 985                                 global_n_host_cpus, RTE_MAX_LCORE,
 986                                 RTE_MAX_LCORE - 1);
 987                 global_n_host_cpus = RTE_MAX_LCORE;
 988         }
 989
 990         return 0;
 991 error:
 992         if (global_hypervisor_available)
 993                 disconnect_hypervisor();
 994         return -1;
 995 }
 996
 997 void
 998 channel_manager_exit(void)
 999 {
1000         unsigned i;
1001         char mask[RTE_MAX_LCORE];
1002         struct virtual_machine_info *vm_info;
1003
1004         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
1005
1006                 rte_spinlock_lock(&(vm_info->config_spinlock));
1007
1008                 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
1009                 for (i = 0; i < RTE_MAX_LCORE; i++) {
1010                         if (mask[i] != 1)
1011                                 continue;
1012                         remove_channel_from_monitor(
1013                                         vm_info->channels[i]);
1014                         close(vm_info->channels[i]->fd);
1015                         rte_free(vm_info->channels[i]);
1016                 }
1017                 rte_spinlock_unlock(&(vm_info->config_spinlock));
1018
1019                 LIST_REMOVE(vm_info, vms_info);
1020                 rte_free(vm_info);
1021         }
1022
1023         if (global_hypervisor_available) {
1024                 /* Only needed if hypervisor available */
1025                 rte_free(global_cpumaps);
1026                 rte_free(global_vircpuinfo);
1027                 disconnect_hypervisor();
1028         }
1029 }