examples/vm_power_manager/channel_manager.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <fcntl.h>
   8 #include <unistd.h>
   9 #include <inttypes.h>
  10 #include <dirent.h>
  11 #include <errno.h>
  12
  13 #include <sys/queue.h>
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include <sys/socket.h>
  17 #include <sys/select.h>
  18
  19 #include <rte_string_fns.h>
  20 #include <rte_malloc.h>
  21 #include <rte_memory.h>
  22 #include <rte_mempool.h>
  23 #include <rte_log.h>
  24 #include <rte_atomic.h>
  25 #include <rte_spinlock.h>
  26
  27 #include <libvirt/libvirt.h>
  28
  29 #include "channel_manager.h"
  30 #include "channel_monitor.h"
  31 #include "power_manager.h"
  32
  33
  34 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  35
  36 struct libvirt_vm_info lvm_info[MAX_CLIENTS];
  37
  38 /* Global pointer to libvirt connection */
  39 static virConnectPtr global_vir_conn_ptr;
  40
  41 static unsigned char *global_cpumaps;
  42 static virVcpuInfo *global_vircpuinfo;
  43 static size_t global_maplen;
  44
  45 static unsigned int global_n_host_cpus;
  46 static bool global_hypervisor_available;
  47
  48 /*
  49  * Represents a single Virtual Machine
  50  */
  51 struct virtual_machine_info {
  52         char name[CHANNEL_MGR_MAX_NAME_LEN];
  53         uint16_t pcpu_map[RTE_MAX_LCORE];
  54         struct channel_info *channels[RTE_MAX_LCORE];
  55         char channel_mask[RTE_MAX_LCORE];
  56         uint8_t num_channels;
  57         enum vm_status status;
  58         virDomainPtr domainPtr;
  59         virDomainInfo info;
  60         rte_spinlock_t config_spinlock;
  61         int allow_query;
  62         LIST_ENTRY(virtual_machine_info) vms_info;
  63 };
  64
  65 LIST_HEAD(, virtual_machine_info) vm_list_head;
  66
  67 static struct virtual_machine_info *
  68 find_domain_by_name(const char *name)
  69 {
  70         struct virtual_machine_info *info;
  71         LIST_FOREACH(info, &vm_list_head, vms_info) {
  72                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
  73                         return info;
  74         }
  75         return NULL;
  76 }
  77
  78 static int
  79 update_pcpus_mask(struct virtual_machine_info *vm_info)
  80 {
  81         virVcpuInfoPtr cpuinfo;
  82         unsigned i, j;
  83         int n_vcpus;
  84
  85         memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen);
  86
  87         if (!virDomainIsActive(vm_info->domainPtr)) {
  88                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
  89                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
  90                                 VIR_DOMAIN_AFFECT_CONFIG);
  91                 if (n_vcpus < 0) {
  92                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
  93                                         "in-active VM '%s'\n", vm_info->name);
  94                         return -1;
  95                 }
  96                 goto update_pcpus;
  97         }
  98
  99         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
 100                         RTE_MAX_LCORE);
 101
 102         cpuinfo = global_vircpuinfo;
 103
 104         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 105                         RTE_MAX_LCORE, global_cpumaps, global_maplen);
 106         if (n_vcpus < 0) {
 107                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 108                                 "active VM '%s'\n", vm_info->name);
 109                 return -1;
 110         }
 111 update_pcpus:
 112         if (n_vcpus >= RTE_MAX_LCORE) {
 113                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 114                                 "0...%d\n", n_vcpus, RTE_MAX_LCORE-1);
 115                 return -1;
 116         }
 117         if (n_vcpus != vm_info->info.nrVirtCpu) {
 118                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 119                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 120                                 n_vcpus);
 121                 vm_info->info.nrVirtCpu = n_vcpus;
 122         }
 123         rte_spinlock_lock(&(vm_info->config_spinlock));
 124         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 125                 for (j = 0; j < global_n_host_cpus; j++) {
 126                         if (VIR_CPU_USABLE(global_cpumaps,
 127                                         global_maplen, i, j) <= 0)
 128                                 continue;
 129                         vm_info->pcpu_map[i] = j;
 130                 }
 131         }
 132         rte_spinlock_unlock(&(vm_info->config_spinlock));
 133         return 0;
 134 }
 135
 136 int
 137 set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu)
 138 {
 139         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 140         struct virtual_machine_info *vm_info;
 141
 142         if (vcpu >= RTE_MAX_LCORE) {
 143                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 144                                 vcpu, RTE_MAX_LCORE-1);
 145                 return -1;
 146         }
 147
 148         vm_info = find_domain_by_name(vm_name);
 149         if (vm_info == NULL) {
 150                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 151                 return -1;
 152         }
 153
 154         if (!virDomainIsActive(vm_info->domainPtr)) {
 155                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 156                                 " for VM '%s', VM is not active\n",
 157                                 vcpu, vm_info->name);
 158                 return -1;
 159         }
 160
 161         if (vcpu >= vm_info->info.nrVirtCpu) {
 162                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 163                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 164                 return -1;
 165         }
 166         memset(global_cpumaps, 0, RTE_MAX_LCORE * global_maplen);
 167
 168         VIR_USE_CPU(global_cpumaps, pcpu);
 169
 170         if (pcpu >= global_n_host_cpus) {
 171                 RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 172                                 "number of CPUs(%u)\n",
 173                                 pcpu, global_n_host_cpus);
 174                 return -1;
 175         }
 176
 177         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 178                         global_maplen, flags) < 0) {
 179                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 180                                 " for VM '%s'\n", vcpu,
 181                                 vm_info->name);
 182                 return -1;
 183         }
 184         rte_spinlock_lock(&(vm_info->config_spinlock));
 185         vm_info->pcpu_map[vcpu] = pcpu;
 186         rte_spinlock_unlock(&(vm_info->config_spinlock));
 187         return 0;
 188 }
 189
 190 uint16_t
 191 get_pcpu(struct channel_info *chan_info, unsigned int vcpu)
 192 {
 193         struct virtual_machine_info *vm_info =
 194                         (struct virtual_machine_info *)chan_info->priv_info;
 195
 196         if (global_hypervisor_available && (vm_info != NULL)) {
 197                 uint16_t pcpu;
 198                 rte_spinlock_lock(&(vm_info->config_spinlock));
 199                 pcpu = vm_info->pcpu_map[vcpu];
 200                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 201                 return pcpu;
 202         } else
 203                 return 0;
 204 }
 205
 206 static inline int
 207 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 208 {
 209         rte_spinlock_lock(&(vm_info->config_spinlock));
 210         if (vm_info->channel_mask[channel_num] == 1) {
 211                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 212                 return 1;
 213         }
 214         rte_spinlock_unlock(&(vm_info->config_spinlock));
 215         return 0;
 216 }
 217
 218
 219
 220 static int
 221 open_non_blocking_channel(struct channel_info *info)
 222 {
 223         int ret, flags;
 224         struct sockaddr_un sock_addr;
 225         fd_set soc_fd_set;
 226         struct timeval tv;
 227
 228         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 229         if (info->fd < 0) {
 230                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 231                                 strerror(errno),
 232                                 info->channel_path);
 233                 return -1;
 234         }
 235         sock_addr.sun_family = AF_UNIX;
 236         memcpy(&sock_addr.sun_path, info->channel_path,
 237                         strlen(info->channel_path)+1);
 238
 239         /* Get current flags */
 240         flags = fcntl(info->fd, F_GETFL, 0);
 241         if (flags < 0) {
 242                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 243                                 "'%s'\n", strerror(errno), info->channel_path);
 244                 return 1;
 245         }
 246         /* Set to Non Blocking */
 247         flags |= O_NONBLOCK;
 248         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 249                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 250                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 251                 return -1;
 252         }
 253         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 254                         sizeof(sock_addr));
 255         if (ret < 0) {
 256                 /* ECONNREFUSED error is given when VM is not active */
 257                 if (errno == ECONNREFUSED) {
 258                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 259                                         "activated its endpoint to channel %s\n",
 260                                         info->channel_path);
 261                         return -1;
 262                 }
 263                 /* Wait for tv_sec if in progress */
 264                 else if (errno == EINPROGRESS) {
 265                         tv.tv_sec = 2;
 266                         tv.tv_usec = 0;
 267                         FD_ZERO(&soc_fd_set);
 268                         FD_SET(info->fd, &soc_fd_set);
 269                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 270                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 271                                                 "'%s'\n", info->channel_path);
 272                                 return -1;
 273                         }
 274                 } else {
 275                         /* Any other error */
 276                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 277                                         " for '%s'\n", strerror(errno), info->channel_path);
 278                         return -1;
 279                 }
 280         }
 281         return 0;
 282 }
 283
 284 static int
 285 open_host_channel(struct channel_info *info)
 286 {
 287         int flags;
 288
 289         info->fd = open(info->channel_path, O_RDWR | O_RSYNC);
 290         if (info->fd < 0) {
 291                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n",
 292                                 strerror(errno),
 293                                 info->channel_path);
 294                 return -1;
 295         }
 296
 297         /* Get current flags */
 298         flags = fcntl(info->fd, F_GETFL, 0);
 299         if (flags < 0) {
 300                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 301                                 "'%s'\n", strerror(errno), info->channel_path);
 302                 return 1;
 303         }
 304         /* Set to Non Blocking */
 305         flags |= O_NONBLOCK;
 306         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 307                 RTE_LOG(WARNING, CHANNEL_MANAGER,
 308                                 "Error(%s) setting non-blocking "
 309                                 "socket for '%s'\n",
 310                                 strerror(errno), info->channel_path);
 311                 return -1;
 312         }
 313         return 0;
 314 }
 315
 316 static int
 317 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 318                 struct channel_info **chan_info_dptr, unsigned channel_num)
 319 {
 320         struct channel_info *chan_info = *chan_info_dptr;
 321         struct virtual_machine_info *vm_info = *vm_info_dptr;
 322
 323         chan_info->channel_num = channel_num;
 324         chan_info->priv_info = (void *)vm_info;
 325         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 326         chan_info->type = CHANNEL_TYPE_BINARY;
 327         if (open_non_blocking_channel(chan_info) < 0) {
 328                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 329                                 "'%s' for VM '%s'\n",
 330                                 chan_info->channel_path, vm_info->name);
 331                 return -1;
 332         }
 333         if (add_channel_to_monitor(&chan_info) < 0) {
 334                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 335                                 "'%s' to epoll ctl for VM '%s'\n",
 336                                 chan_info->channel_path, vm_info->name);
 337                 return -1;
 338
 339         }
 340         rte_spinlock_lock(&(vm_info->config_spinlock));
 341         vm_info->num_channels++;
 342         vm_info->channel_mask[channel_num] = 1;
 343         vm_info->channels[channel_num] = chan_info;
 344         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 345         rte_spinlock_unlock(&(vm_info->config_spinlock));
 346         return 0;
 347 }
 348
 349 static int
 350 fifo_path(char *dst, unsigned int len, unsigned int id)
 351 {
 352         int cnt;
 353
 354         cnt = snprintf(dst, len, "%s%s%d", CHANNEL_MGR_SOCKET_PATH,
 355                         CHANNEL_MGR_FIFO_PATTERN_NAME, id);
 356
 357         if ((cnt < 0) || (cnt > (int)len - 1)) {
 358                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not create proper "
 359                         "string for fifo path\n");
 360
 361                 return -1;
 362         }
 363
 364         return 0;
 365 }
 366
 367 static int
 368 setup_host_channel_info(struct channel_info **chan_info_dptr,
 369                 unsigned int channel_num)
 370 {
 371         struct channel_info *chan_info = *chan_info_dptr;
 372
 373         chan_info->channel_num = channel_num;
 374         chan_info->priv_info = (void *)NULL;
 375         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 376         chan_info->type = CHANNEL_TYPE_JSON;
 377
 378         if (open_host_channel(chan_info) < 0) {
 379                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: "
 380                                 "'%s'\n",
 381                                 chan_info->channel_path);
 382                 return -1;
 383         }
 384         if (add_channel_to_monitor(&chan_info) < 0) {
 385                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 386                                 "'%s' to epoll ctl\n",
 387                                 chan_info->channel_path);
 388                 return -1;
 389
 390         }
 391         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 392         return 0;
 393 }
 394
 395 int
 396 add_all_channels(const char *vm_name)
 397 {
 398         DIR *d;
 399         struct dirent *dir;
 400         struct virtual_machine_info *vm_info;
 401         struct channel_info *chan_info;
 402         char *token, *remaining, *tail_ptr;
 403         char socket_name[PATH_MAX];
 404         unsigned channel_num;
 405         int num_channels_enabled = 0;
 406
 407         /* verify VM exists */
 408         vm_info = find_domain_by_name(vm_name);
 409         if (vm_info == NULL) {
 410                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 411                                 " during channel discovery\n", vm_name);
 412                 return 0;
 413         }
 414         if (!virDomainIsActive(vm_info->domainPtr)) {
 415                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 416                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 417                 return 0;
 418         }
 419         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 420         if (d == NULL) {
 421                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 422                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 423                 return -1;
 424         }
 425         while ((dir = readdir(d)) != NULL) {
 426                 if (!strncmp(dir->d_name, ".", 1) ||
 427                                 !strncmp(dir->d_name, "..", 2))
 428                         continue;
 429
 430                 strlcpy(socket_name, dir->d_name, sizeof(socket_name));
 431                 remaining = socket_name;
 432                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 433                 token = strsep(&remaining, ".");
 434                 if (remaining == NULL)
 435                         continue;
 436                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 437                         continue;
 438
 439                 /* remaining should contain only <channel_num> */
 440                 errno = 0;
 441                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 442                 if ((errno != 0) || (remaining[0] == '\0') ||
 443                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 444                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 445                                         "'%s' found it should be in the form of "
 446                                         "'<guest_name>.<channel_num>(decimal)'\n",
 447                                         dir->d_name);
 448                         continue;
 449                 }
 450                 if (channel_num >= RTE_MAX_LCORE) {
 451                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 452                                         "greater than max allowable: %d, skipping '%s%s'\n",
 453                                         channel_num, RTE_MAX_LCORE-1,
 454                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 455                         continue;
 456                 }
 457                 if (rte_lcore_index(channel_num) == -1)
 458                         continue;
 459
 460                 /* if channel has not been added previously */
 461                 if (channel_exists(vm_info, channel_num))
 462                         continue;
 463
 464                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 465                                 RTE_CACHE_LINE_SIZE);
 466                 if (chan_info == NULL) {
 467                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 468                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 469                         continue;
 470                 }
 471
 472                 if ((size_t)snprintf(chan_info->channel_path,
 473                                 sizeof(chan_info->channel_path), "%s%s",
 474                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name)
 475                                         >= sizeof(chan_info->channel_path)) {
 476                         RTE_LOG(ERR, CHANNEL_MANAGER, "Pathname too long for channel '%s%s'\n",
 477                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 478                         rte_free(chan_info);
 479                         continue;
 480                 }
 481
 482                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 483                         rte_free(chan_info);
 484                         continue;
 485                 }
 486
 487                 num_channels_enabled++;
 488         }
 489         closedir(d);
 490         return num_channels_enabled;
 491 }
 492
 493 int
 494 add_channels(const char *vm_name, unsigned *channel_list,
 495                 unsigned len_channel_list)
 496 {
 497         struct virtual_machine_info *vm_info;
 498         struct channel_info *chan_info;
 499         char socket_path[PATH_MAX];
 500         unsigned i;
 501         int num_channels_enabled = 0;
 502
 503         vm_info = find_domain_by_name(vm_name);
 504         if (vm_info == NULL) {
 505                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 506                                 "not found\n", vm_name);
 507                 return 0;
 508         }
 509
 510         if (!virDomainIsActive(vm_info->domainPtr)) {
 511                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 512                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 513                 return 0;
 514         }
 515
 516         for (i = 0; i < len_channel_list; i++) {
 517                 if (rte_lcore_index(i) == -1)
 518                         continue;
 519
 520                 if (channel_list[i] >= RTE_MAX_LCORE) {
 521                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 522                                                         "0...%d\n", channel_list[i],
 523                                                         RTE_MAX_LCORE-1);
 524                         continue;
 525                 }
 526                 if (channel_exists(vm_info, channel_list[i])) {
 527                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 528                                         "'%s.%u'\n", vm_name, i);
 529                         continue;
 530                 }
 531
 532                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 533                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 534                 errno = 0;
 535                 if (access(socket_path, F_OK) < 0) {
 536                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 537                                         "%s\n", socket_path, strerror(errno));
 538                         continue;
 539                 }
 540                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 541                                 RTE_CACHE_LINE_SIZE);
 542                 if (chan_info == NULL) {
 543                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 544                                         "channel '%s'\n", socket_path);
 545                         continue;
 546                 }
 547                 snprintf(chan_info->channel_path,
 548                                 sizeof(chan_info->channel_path), "%s%s.%u",
 549                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 550                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 551                         rte_free(chan_info);
 552                         continue;
 553                 }
 554                 num_channels_enabled++;
 555
 556         }
 557         return num_channels_enabled;
 558 }
 559
 560 int
 561 add_host_channels(void)
 562 {
 563         struct channel_info *chan_info;
 564         char socket_path[PATH_MAX];
 565         int num_channels_enabled = 0;
 566         int ret;
 567         struct core_info *ci;
 568         struct channel_info *chan_infos[RTE_MAX_LCORE];
 569         int i;
 570
 571         for (i = 0; i < RTE_MAX_LCORE; i++)
 572                 chan_infos[i] = NULL;
 573
 574         ci = get_core_info();
 575         if (ci == NULL) {
 576                 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot allocate memory for core_info\n");
 577                 return 0;
 578         }
 579
 580         for (i = 0; i < ci->core_count; i++) {
 581                 if (rte_lcore_index(i) == -1)
 582                         continue;
 583
 584                 if (ci->cd[i].global_enabled_cpus == 0)
 585                         continue;
 586
 587                 ret = fifo_path(socket_path, sizeof(socket_path), i);
 588                 if (ret < 0)
 589                         goto error;
 590
 591                 ret = mkfifo(socket_path, 0660);
 592                 RTE_LOG(DEBUG, CHANNEL_MANAGER, "TRY CREATE fifo '%s'\n",
 593                         socket_path);
 594                 if ((errno != EEXIST) && (ret < 0)) {
 595                         RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: "
 596                                         "%s\n", socket_path, strerror(errno));
 597                         goto error;
 598                 }
 599                 chan_info = rte_malloc(NULL, sizeof(*chan_info), 0);
 600                 if (chan_info == NULL) {
 601                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 602                                         "channel '%s'\n", socket_path);
 603                         goto error;
 604                 }
 605                 chan_infos[i] = chan_info;
 606                 strlcpy(chan_info->channel_path, socket_path,
 607                                 sizeof(chan_info->channel_path));
 608
 609                 if (setup_host_channel_info(&chan_info, i) < 0) {
 610                         rte_free(chan_info);
 611                         chan_infos[i] = NULL;
 612                         goto error;
 613                 }
 614                 num_channels_enabled++;
 615         }
 616
 617         return num_channels_enabled;
 618 error:
 619         /* Clean up the channels opened before we hit an error. */
 620         for (i = 0; i < ci->core_count; i++) {
 621                 if (chan_infos[i] != NULL) {
 622                         remove_channel_from_monitor(chan_infos[i]);
 623                         close(chan_infos[i]->fd);
 624                         rte_free(chan_infos[i]);
 625                 }
 626         }
 627         return 0;
 628 }
 629
 630 int
 631 remove_channel(struct channel_info **chan_info_dptr)
 632 {
 633         struct virtual_machine_info *vm_info;
 634         struct channel_info *chan_info = *chan_info_dptr;
 635
 636         close(chan_info->fd);
 637
 638         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 639
 640         rte_spinlock_lock(&(vm_info->config_spinlock));
 641         vm_info->channel_mask[chan_info->channel_num] = 0;
 642         vm_info->num_channels--;
 643         rte_spinlock_unlock(&(vm_info->config_spinlock));
 644
 645         rte_free(chan_info);
 646         return 0;
 647 }
 648
 649 int
 650 set_channel_status_all(const char *vm_name, enum channel_status status)
 651 {
 652         struct virtual_machine_info *vm_info;
 653         unsigned i;
 654         char mask[RTE_MAX_LCORE];
 655         int num_channels_changed = 0;
 656
 657         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 658                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 659                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 660                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 661         }
 662         vm_info = find_domain_by_name(vm_name);
 663         if (vm_info == NULL) {
 664                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 665                                 "not found\n", vm_name);
 666                 return 0;
 667         }
 668
 669         rte_spinlock_lock(&(vm_info->config_spinlock));
 670         memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
 671         for (i = 0; i < RTE_MAX_LCORE; i++) {
 672                 if (mask[i] != 1)
 673                         continue;
 674                 vm_info->channels[i]->status = status;
 675                 num_channels_changed++;
 676         }
 677         rte_spinlock_unlock(&(vm_info->config_spinlock));
 678         return num_channels_changed;
 679
 680 }
 681
 682 int
 683 set_channel_status(const char *vm_name, unsigned *channel_list,
 684                 unsigned len_channel_list, enum channel_status status)
 685 {
 686         struct virtual_machine_info *vm_info;
 687         unsigned i;
 688         int num_channels_changed = 0;
 689
 690         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 691                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 692                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 693                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 694         }
 695         vm_info = find_domain_by_name(vm_name);
 696         if (vm_info == NULL) {
 697                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 698                                 "not found\n", vm_name);
 699                 return 0;
 700         }
 701         for (i = 0; i < len_channel_list; i++) {
 702                 if (channel_exists(vm_info, channel_list[i])) {
 703                         rte_spinlock_lock(&(vm_info->config_spinlock));
 704                         vm_info->channels[channel_list[i]]->status = status;
 705                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 706                         num_channels_changed++;
 707                 }
 708         }
 709         return num_channels_changed;
 710 }
 711
 712 void
 713 get_all_vm(int *num_vm, int *num_vcpu)
 714 {
 715
 716         virNodeInfo node_info;
 717         virDomainPtr *domptr;
 718         int i, ii, numVcpus[MAX_VCPUS], n_vcpus;
 719         unsigned int jj;
 720         const char *vm_name;
 721         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 722                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 723         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 724
 725         if (!global_hypervisor_available)
 726                 return;
 727
 728         memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen);
 729         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 730                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 731                 return;
 732         }
 733
 734         /* Returns number of pcpus */
 735         global_n_host_cpus = (unsigned int)node_info.cpus;
 736
 737         /* Returns number of active domains */
 738         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 739                                         domain_flags);
 740         if (*num_vm <= 0) {
 741                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 742                 return;
 743         }
 744
 745         for (i = 0; i < *num_vm; i++) {
 746
 747                 /* Get Domain Names */
 748                 vm_name = virDomainGetName(domptr[i]);
 749                 lvm_info[i].vm_name = vm_name;
 750
 751                 /* Get Number of Vcpus */
 752                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 753
 754                 /* Get Number of VCpus & VcpuPinInfo */
 755                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 756                                 numVcpus[i], global_cpumaps,
 757                                 global_maplen, domain_flag);
 758
 759                 if ((int)n_vcpus > 0) {
 760                         *num_vcpu = n_vcpus;
 761                         lvm_info[i].num_cpus = n_vcpus;
 762                 }
 763
 764                 /* Save pcpu in use by libvirt VMs */
 765                 for (ii = 0; ii < n_vcpus; ii++) {
 766                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 767                                 if (VIR_CPU_USABLE(global_cpumaps,
 768                                                 global_maplen, ii, jj) > 0) {
 769                                         lvm_info[i].pcpus[ii] = jj;
 770                                 }
 771                         }
 772                 }
 773         }
 774 }
 775
 776 int
 777 get_info_vm(const char *vm_name, struct vm_info *info)
 778 {
 779         struct virtual_machine_info *vm_info;
 780         unsigned i, channel_num = 0;
 781         char mask[RTE_MAX_LCORE];
 782
 783         vm_info = find_domain_by_name(vm_name);
 784         if (vm_info == NULL) {
 785                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 786                 return -1;
 787         }
 788         info->status = CHANNEL_MGR_VM_ACTIVE;
 789         if (!virDomainIsActive(vm_info->domainPtr))
 790                 info->status = CHANNEL_MGR_VM_INACTIVE;
 791
 792         rte_spinlock_lock(&(vm_info->config_spinlock));
 793
 794         memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
 795         for (i = 0; i < RTE_MAX_LCORE; i++) {
 796                 if (mask[i] != 1)
 797                         continue;
 798                 info->channels[channel_num].channel_num = i;
 799                 memcpy(info->channels[channel_num].channel_path,
 800                                 vm_info->channels[i]->channel_path,
 801                                 UNIX_PATH_MAX);
 802                 info->channels[channel_num].status =
 803                                 vm_info->channels[i]->status;
 804                 info->channels[channel_num].fd =
 805                                 vm_info->channels[i]->fd;
 806                 channel_num++;
 807         }
 808
 809         info->allow_query = vm_info->allow_query;
 810         info->num_channels = channel_num;
 811         info->num_vcpus = vm_info->info.nrVirtCpu;
 812         rte_spinlock_unlock(&(vm_info->config_spinlock));
 813
 814         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 815         rte_spinlock_lock(&(vm_info->config_spinlock));
 816         for (i = 0; i < info->num_vcpus; i++) {
 817                 info->pcpu_map[i] = vm_info->pcpu_map[i];
 818         }
 819         rte_spinlock_unlock(&(vm_info->config_spinlock));
 820         return 0;
 821 }
 822
 823 int
 824 add_vm(const char *vm_name)
 825 {
 826         struct virtual_machine_info *new_domain;
 827         virDomainPtr dom_ptr;
 828         int i;
 829
 830         if (find_domain_by_name(vm_name) != NULL) {
 831                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 832                                 "already exists\n", vm_name);
 833                 return -1;
 834         }
 835
 836         if (global_vir_conn_ptr == NULL) {
 837                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 838                 return -1;
 839         }
 840         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 841         if (dom_ptr == NULL) {
 842                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 843                                 "VM '%s' not found\n", vm_name);
 844                 return -1;
 845         }
 846
 847         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 848                         RTE_CACHE_LINE_SIZE);
 849         if (new_domain == NULL) {
 850                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 851                                 "info\n");
 852                 return -1;
 853         }
 854         new_domain->domainPtr = dom_ptr;
 855         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 856                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 857                 rte_free(new_domain);
 858                 return -1;
 859         }
 860         if (new_domain->info.nrVirtCpu > RTE_MAX_LCORE) {
 861                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 862                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 863                                 RTE_MAX_LCORE);
 864                 rte_free(new_domain);
 865                 return -1;
 866         }
 867
 868         for (i = 0; i < RTE_MAX_LCORE; i++)
 869                 new_domain->pcpu_map[i] = 0;
 870
 871         if (update_pcpus_mask(new_domain) < 0) {
 872                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 873                 rte_free(new_domain);
 874                 return -1;
 875         }
 876         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 877         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 878         memset(new_domain->channel_mask, 0, RTE_MAX_LCORE);
 879         new_domain->num_channels = 0;
 880
 881         if (!virDomainIsActive(dom_ptr))
 882                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 883         else
 884                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 885
 886         new_domain->allow_query = 0;
 887         rte_spinlock_init(&(new_domain->config_spinlock));
 888         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 889         return 0;
 890 }
 891
 892 int
 893 remove_vm(const char *vm_name)
 894 {
 895         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 896
 897         if (vm_info == NULL) {
 898                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 899                                 "not found\n", vm_name);
 900                 return -1;
 901         }
 902         rte_spinlock_lock(&vm_info->config_spinlock);
 903         if (vm_info->num_channels != 0) {
 904                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 905                                 "%"PRId8" channels still active\n",
 906                                 vm_name, vm_info->num_channels);
 907                 rte_spinlock_unlock(&vm_info->config_spinlock);
 908                 return -1;
 909         }
 910         LIST_REMOVE(vm_info, vms_info);
 911         rte_spinlock_unlock(&vm_info->config_spinlock);
 912         rte_free(vm_info);
 913         return 0;
 914 }
 915
 916 int
 917 set_query_status(char *vm_name,
 918                 bool allow_query)
 919 {
 920         struct virtual_machine_info *vm_info;
 921
 922         vm_info = find_domain_by_name(vm_name);
 923         if (vm_info == NULL) {
 924                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 925                 return -1;
 926         }
 927         rte_spinlock_lock(&(vm_info->config_spinlock));
 928         vm_info->allow_query = allow_query ? 1 : 0;
 929         rte_spinlock_unlock(&(vm_info->config_spinlock));
 930         return 0;
 931 }
 932
 933 static void
 934 disconnect_hypervisor(void)
 935 {
 936         if (global_vir_conn_ptr != NULL) {
 937                 virConnectClose(global_vir_conn_ptr);
 938                 global_vir_conn_ptr = NULL;
 939         }
 940 }
 941
 942 static int
 943 connect_hypervisor(const char *path)
 944 {
 945         if (global_vir_conn_ptr != NULL) {
 946                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 947                                 "already established\n", path);
 948                 return -1;
 949         }
 950         global_vir_conn_ptr = virConnectOpen(path);
 951         if (global_vir_conn_ptr == NULL) {
 952                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 953                                 "Hypervisor '%s'\n", path);
 954                 return -1;
 955         }
 956         return 0;
 957 }
 958 int
 959 channel_manager_init(const char *path __rte_unused)
 960 {
 961         virNodeInfo info;
 962
 963         LIST_INIT(&vm_list_head);
 964         if (connect_hypervisor(path) < 0) {
 965                 global_n_host_cpus = 64;
 966                 global_hypervisor_available = 0;
 967                 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 968         } else {
 969                 global_hypervisor_available = 1;
 970
 971                 global_maplen = VIR_CPU_MAPLEN(RTE_MAX_LCORE);
 972
 973                 global_vircpuinfo = rte_zmalloc(NULL,
 974                                 sizeof(*global_vircpuinfo) *
 975                                 RTE_MAX_LCORE, RTE_CACHE_LINE_SIZE);
 976                 if (global_vircpuinfo == NULL) {
 977                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 978                         goto error;
 979                 }
 980                 global_cpumaps = rte_zmalloc(NULL,
 981                                 RTE_MAX_LCORE * global_maplen,
 982                                 RTE_CACHE_LINE_SIZE);
 983                 if (global_cpumaps == NULL)
 984                         goto error;
 985
 986                 if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 987                         RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 988                         goto error;
 989                 }
 990                 global_n_host_cpus = (unsigned int)info.cpus;
 991         }
 992
 993
 994
 995         if (global_n_host_cpus > RTE_MAX_LCORE) {
 996                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 997                                 "maximum of %u. No cores over %u should be used.\n",
 998                                 global_n_host_cpus, RTE_MAX_LCORE,
 999                                 RTE_MAX_LCORE - 1);
1000                 global_n_host_cpus = RTE_MAX_LCORE;
1001         }
1002
1003         return 0;
1004 error:
1005         if (global_hypervisor_available)
1006                 disconnect_hypervisor();
1007         return -1;
1008 }
1009
1010 void
1011 channel_manager_exit(void)
1012 {
1013         unsigned i;
1014         char mask[RTE_MAX_LCORE];
1015         struct virtual_machine_info *vm_info;
1016
1017         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
1018
1019                 rte_spinlock_lock(&(vm_info->config_spinlock));
1020
1021                 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
1022                 for (i = 0; i < RTE_MAX_LCORE; i++) {
1023                         if (mask[i] != 1)
1024                                 continue;
1025                         remove_channel_from_monitor(
1026                                         vm_info->channels[i]);
1027                         close(vm_info->channels[i]->fd);
1028                         rte_free(vm_info->channels[i]);
1029                 }
1030                 rte_spinlock_unlock(&(vm_info->config_spinlock));
1031
1032                 LIST_REMOVE(vm_info, vms_info);
1033                 rte_free(vm_info);
1034         }
1035
1036         if (global_hypervisor_available) {
1037                 /* Only needed if hypervisor available */
1038                 rte_free(global_cpumaps);
1039                 rte_free(global_vircpuinfo);
1040                 disconnect_hypervisor();
1041         }
1042 }