examples/vm_power_manager/channel_manager.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <sys/un.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <inttypes.h>
  11 #include <dirent.h>
  12 #include <errno.h>
  13
  14 #include <sys/queue.h>
  15 #include <sys/types.h>
  16 #include <sys/stat.h>
  17 #include <sys/socket.h>
  18 #include <sys/select.h>
  19
  20 #include <rte_string_fns.h>
  21 #include <rte_malloc.h>
  22 #include <rte_memory.h>
  23 #include <rte_mempool.h>
  24 #include <rte_log.h>
  25 #include <rte_atomic.h>
  26 #include <rte_spinlock.h>
  27
  28 #include <libvirt/libvirt.h>
  29
  30 #include "channel_manager.h"
  31 #include "channel_commands.h"
  32 #include "channel_monitor.h"
  33 #include "power_manager.h"
  34
  35
  36 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  37
  38 struct libvirt_vm_info lvm_info[MAX_CLIENTS];
  39
  40 /* Global pointer to libvirt connection */
  41 static virConnectPtr global_vir_conn_ptr;
  42
  43 static unsigned char *global_cpumaps;
  44 static virVcpuInfo *global_vircpuinfo;
  45 static size_t global_maplen;
  46
  47 static unsigned int global_n_host_cpus;
  48 static bool global_hypervisor_available;
  49
  50 /*
  51  * Represents a single Virtual Machine
  52  */
  53 struct virtual_machine_info {
  54         char name[CHANNEL_MGR_MAX_NAME_LEN];
  55         uint16_t pcpu_map[RTE_MAX_LCORE];
  56         struct channel_info *channels[RTE_MAX_LCORE];
  57         char channel_mask[RTE_MAX_LCORE];
  58         uint8_t num_channels;
  59         enum vm_status status;
  60         virDomainPtr domainPtr;
  61         virDomainInfo info;
  62         rte_spinlock_t config_spinlock;
  63         int allow_query;
  64         LIST_ENTRY(virtual_machine_info) vms_info;
  65 };
  66
  67 LIST_HEAD(, virtual_machine_info) vm_list_head;
  68
  69 static struct virtual_machine_info *
  70 find_domain_by_name(const char *name)
  71 {
  72         struct virtual_machine_info *info;
  73         LIST_FOREACH(info, &vm_list_head, vms_info) {
  74                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
  75                         return info;
  76         }
  77         return NULL;
  78 }
  79
  80 static int
  81 update_pcpus_mask(struct virtual_machine_info *vm_info)
  82 {
  83         virVcpuInfoPtr cpuinfo;
  84         unsigned i, j;
  85         int n_vcpus;
  86
  87         memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen);
  88
  89         if (!virDomainIsActive(vm_info->domainPtr)) {
  90                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
  91                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
  92                                 VIR_DOMAIN_AFFECT_CONFIG);
  93                 if (n_vcpus < 0) {
  94                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
  95                                         "in-active VM '%s'\n", vm_info->name);
  96                         return -1;
  97                 }
  98                 goto update_pcpus;
  99         }
 100
 101         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
 102                         RTE_MAX_LCORE);
 103
 104         cpuinfo = global_vircpuinfo;
 105
 106         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 107                         RTE_MAX_LCORE, global_cpumaps, global_maplen);
 108         if (n_vcpus < 0) {
 109                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 110                                 "active VM '%s'\n", vm_info->name);
 111                 return -1;
 112         }
 113 update_pcpus:
 114         if (n_vcpus >= RTE_MAX_LCORE) {
 115                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 116                                 "0...%d\n", n_vcpus, RTE_MAX_LCORE-1);
 117                 return -1;
 118         }
 119         if (n_vcpus != vm_info->info.nrVirtCpu) {
 120                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 121                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 122                                 n_vcpus);
 123                 vm_info->info.nrVirtCpu = n_vcpus;
 124         }
 125         rte_spinlock_lock(&(vm_info->config_spinlock));
 126         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 127                 for (j = 0; j < global_n_host_cpus; j++) {
 128                         if (VIR_CPU_USABLE(global_cpumaps,
 129                                         global_maplen, i, j) <= 0)
 130                                 continue;
 131                         vm_info->pcpu_map[i] = j;
 132                 }
 133         }
 134         rte_spinlock_unlock(&(vm_info->config_spinlock));
 135         return 0;
 136 }
 137
 138 int
 139 set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu)
 140 {
 141         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 142         struct virtual_machine_info *vm_info;
 143
 144         if (vcpu >= RTE_MAX_LCORE) {
 145                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 146                                 vcpu, RTE_MAX_LCORE-1);
 147                 return -1;
 148         }
 149
 150         vm_info = find_domain_by_name(vm_name);
 151         if (vm_info == NULL) {
 152                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 153                 return -1;
 154         }
 155
 156         if (!virDomainIsActive(vm_info->domainPtr)) {
 157                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 158                                 " for VM '%s', VM is not active\n",
 159                                 vcpu, vm_info->name);
 160                 return -1;
 161         }
 162
 163         if (vcpu >= vm_info->info.nrVirtCpu) {
 164                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 165                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 166                 return -1;
 167         }
 168         memset(global_cpumaps, 0, RTE_MAX_LCORE * global_maplen);
 169
 170         VIR_USE_CPU(global_cpumaps, pcpu);
 171
 172         if (pcpu >= global_n_host_cpus) {
 173                 RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 174                                 "number of CPUs(%u)\n",
 175                                 pcpu, global_n_host_cpus);
 176                 return -1;
 177         }
 178
 179         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 180                         global_maplen, flags) < 0) {
 181                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 182                                 " for VM '%s'\n", vcpu,
 183                                 vm_info->name);
 184                 return -1;
 185         }
 186         rte_spinlock_lock(&(vm_info->config_spinlock));
 187         vm_info->pcpu_map[vcpu] = pcpu;
 188         rte_spinlock_unlock(&(vm_info->config_spinlock));
 189         return 0;
 190 }
 191
 192 uint16_t
 193 get_pcpu(struct channel_info *chan_info, unsigned int vcpu)
 194 {
 195         struct virtual_machine_info *vm_info =
 196                         (struct virtual_machine_info *)chan_info->priv_info;
 197
 198         if (global_hypervisor_available && (vm_info != NULL)) {
 199                 uint16_t pcpu;
 200                 rte_spinlock_lock(&(vm_info->config_spinlock));
 201                 pcpu = vm_info->pcpu_map[vcpu];
 202                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 203                 return pcpu;
 204         } else
 205                 return 0;
 206 }
 207
 208 static inline int
 209 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 210 {
 211         rte_spinlock_lock(&(vm_info->config_spinlock));
 212         if (vm_info->channel_mask[channel_num] == 1) {
 213                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 214                 return 1;
 215         }
 216         rte_spinlock_unlock(&(vm_info->config_spinlock));
 217         return 0;
 218 }
 219
 220
 221
 222 static int
 223 open_non_blocking_channel(struct channel_info *info)
 224 {
 225         int ret, flags;
 226         struct sockaddr_un sock_addr;
 227         fd_set soc_fd_set;
 228         struct timeval tv;
 229
 230         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 231         if (info->fd < 0) {
 232                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 233                                 strerror(errno),
 234                                 info->channel_path);
 235                 return -1;
 236         }
 237         sock_addr.sun_family = AF_UNIX;
 238         memcpy(&sock_addr.sun_path, info->channel_path,
 239                         strlen(info->channel_path)+1);
 240
 241         /* Get current flags */
 242         flags = fcntl(info->fd, F_GETFL, 0);
 243         if (flags < 0) {
 244                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 245                                 "'%s'\n", strerror(errno), info->channel_path);
 246                 return 1;
 247         }
 248         /* Set to Non Blocking */
 249         flags |= O_NONBLOCK;
 250         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 251                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 252                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 253                 return -1;
 254         }
 255         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 256                         sizeof(sock_addr));
 257         if (ret < 0) {
 258                 /* ECONNREFUSED error is given when VM is not active */
 259                 if (errno == ECONNREFUSED) {
 260                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 261                                         "activated its endpoint to channel %s\n",
 262                                         info->channel_path);
 263                         return -1;
 264                 }
 265                 /* Wait for tv_sec if in progress */
 266                 else if (errno == EINPROGRESS) {
 267                         tv.tv_sec = 2;
 268                         tv.tv_usec = 0;
 269                         FD_ZERO(&soc_fd_set);
 270                         FD_SET(info->fd, &soc_fd_set);
 271                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 272                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 273                                                 "'%s'\n", info->channel_path);
 274                                 return -1;
 275                         }
 276                 } else {
 277                         /* Any other error */
 278                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 279                                         " for '%s'\n", strerror(errno), info->channel_path);
 280                         return -1;
 281                 }
 282         }
 283         return 0;
 284 }
 285
 286 static int
 287 open_host_channel(struct channel_info *info)
 288 {
 289         int flags;
 290
 291         info->fd = open(info->channel_path, O_RDWR | O_RSYNC);
 292         if (info->fd < 0) {
 293                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n",
 294                                 strerror(errno),
 295                                 info->channel_path);
 296                 return -1;
 297         }
 298
 299         /* Get current flags */
 300         flags = fcntl(info->fd, F_GETFL, 0);
 301         if (flags < 0) {
 302                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 303                                 "'%s'\n", strerror(errno), info->channel_path);
 304                 return 1;
 305         }
 306         /* Set to Non Blocking */
 307         flags |= O_NONBLOCK;
 308         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 309                 RTE_LOG(WARNING, CHANNEL_MANAGER,
 310                                 "Error(%s) setting non-blocking "
 311                                 "socket for '%s'\n",
 312                                 strerror(errno), info->channel_path);
 313                 return -1;
 314         }
 315         return 0;
 316 }
 317
 318 static int
 319 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 320                 struct channel_info **chan_info_dptr, unsigned channel_num)
 321 {
 322         struct channel_info *chan_info = *chan_info_dptr;
 323         struct virtual_machine_info *vm_info = *vm_info_dptr;
 324
 325         chan_info->channel_num = channel_num;
 326         chan_info->priv_info = (void *)vm_info;
 327         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 328         chan_info->type = CHANNEL_TYPE_BINARY;
 329         if (open_non_blocking_channel(chan_info) < 0) {
 330                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 331                                 "'%s' for VM '%s'\n",
 332                                 chan_info->channel_path, vm_info->name);
 333                 return -1;
 334         }
 335         if (add_channel_to_monitor(&chan_info) < 0) {
 336                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 337                                 "'%s' to epoll ctl for VM '%s'\n",
 338                                 chan_info->channel_path, vm_info->name);
 339                 return -1;
 340
 341         }
 342         rte_spinlock_lock(&(vm_info->config_spinlock));
 343         vm_info->num_channels++;
 344         vm_info->channel_mask[channel_num] = 1;
 345         vm_info->channels[channel_num] = chan_info;
 346         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 347         rte_spinlock_unlock(&(vm_info->config_spinlock));
 348         return 0;
 349 }
 350
 351 static int
 352 fifo_path(char *dst, unsigned int len, unsigned int id)
 353 {
 354         int cnt;
 355
 356         cnt = snprintf(dst, len, "%s%s%d", CHANNEL_MGR_SOCKET_PATH,
 357                         CHANNEL_MGR_FIFO_PATTERN_NAME, id);
 358
 359         if ((cnt < 0) || (cnt > (int)len - 1)) {
 360                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not create proper "
 361                         "string for fifo path\n");
 362
 363                 return -1;
 364         }
 365
 366         return 0;
 367 }
 368
 369 static int
 370 setup_host_channel_info(struct channel_info **chan_info_dptr,
 371                 unsigned int channel_num)
 372 {
 373         struct channel_info *chan_info = *chan_info_dptr;
 374
 375         chan_info->channel_num = channel_num;
 376         chan_info->priv_info = (void *)NULL;
 377         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 378         chan_info->type = CHANNEL_TYPE_JSON;
 379
 380         if (open_host_channel(chan_info) < 0) {
 381                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: "
 382                                 "'%s'\n",
 383                                 chan_info->channel_path);
 384                 return -1;
 385         }
 386         if (add_channel_to_monitor(&chan_info) < 0) {
 387                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 388                                 "'%s' to epoll ctl\n",
 389                                 chan_info->channel_path);
 390                 return -1;
 391
 392         }
 393         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 394         return 0;
 395 }
 396
 397 int
 398 add_all_channels(const char *vm_name)
 399 {
 400         DIR *d;
 401         struct dirent *dir;
 402         struct virtual_machine_info *vm_info;
 403         struct channel_info *chan_info;
 404         char *token, *remaining, *tail_ptr;
 405         char socket_name[PATH_MAX];
 406         unsigned channel_num;
 407         int num_channels_enabled = 0;
 408
 409         /* verify VM exists */
 410         vm_info = find_domain_by_name(vm_name);
 411         if (vm_info == NULL) {
 412                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 413                                 " during channel discovery\n", vm_name);
 414                 return 0;
 415         }
 416         if (!virDomainIsActive(vm_info->domainPtr)) {
 417                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 418                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 419                 return 0;
 420         }
 421         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 422         if (d == NULL) {
 423                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 424                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 425                 return -1;
 426         }
 427         while ((dir = readdir(d)) != NULL) {
 428                 if (!strncmp(dir->d_name, ".", 1) ||
 429                                 !strncmp(dir->d_name, "..", 2))
 430                         continue;
 431
 432                 strlcpy(socket_name, dir->d_name, sizeof(socket_name));
 433                 remaining = socket_name;
 434                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 435                 token = strsep(&remaining, ".");
 436                 if (remaining == NULL)
 437                         continue;
 438                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 439                         continue;
 440
 441                 /* remaining should contain only <channel_num> */
 442                 errno = 0;
 443                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 444                 if ((errno != 0) || (remaining[0] == '\0') ||
 445                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 446                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 447                                         "'%s' found it should be in the form of "
 448                                         "'<guest_name>.<channel_num>(decimal)'\n",
 449                                         dir->d_name);
 450                         continue;
 451                 }
 452                 if (channel_num >= RTE_MAX_LCORE) {
 453                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 454                                         "greater than max allowable: %d, skipping '%s%s'\n",
 455                                         channel_num, RTE_MAX_LCORE-1,
 456                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 457                         continue;
 458                 }
 459                 /* if channel has not been added previously */
 460                 if (channel_exists(vm_info, channel_num))
 461                         continue;
 462
 463                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 464                                 RTE_CACHE_LINE_SIZE);
 465                 if (chan_info == NULL) {
 466                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 467                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 468                         continue;
 469                 }
 470
 471                 snprintf(chan_info->channel_path,
 472                                 sizeof(chan_info->channel_path), "%s%s",
 473                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 474
 475                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 476                         rte_free(chan_info);
 477                         continue;
 478                 }
 479
 480                 num_channels_enabled++;
 481         }
 482         closedir(d);
 483         return num_channels_enabled;
 484 }
 485
 486 int
 487 add_channels(const char *vm_name, unsigned *channel_list,
 488                 unsigned len_channel_list)
 489 {
 490         struct virtual_machine_info *vm_info;
 491         struct channel_info *chan_info;
 492         char socket_path[PATH_MAX];
 493         unsigned i;
 494         int num_channels_enabled = 0;
 495
 496         vm_info = find_domain_by_name(vm_name);
 497         if (vm_info == NULL) {
 498                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 499                                 "not found\n", vm_name);
 500                 return 0;
 501         }
 502
 503         if (!virDomainIsActive(vm_info->domainPtr)) {
 504                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 505                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 506                 return 0;
 507         }
 508
 509         for (i = 0; i < len_channel_list; i++) {
 510
 511                 if (channel_list[i] >= RTE_MAX_LCORE) {
 512                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 513                                                         "0...%d\n", channel_list[i],
 514                                                         RTE_MAX_LCORE-1);
 515                         continue;
 516                 }
 517                 if (channel_exists(vm_info, channel_list[i])) {
 518                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 519                                         "'%s.%u'\n", vm_name, i);
 520                         continue;
 521                 }
 522
 523                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 524                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 525                 errno = 0;
 526                 if (access(socket_path, F_OK) < 0) {
 527                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 528                                         "%s\n", socket_path, strerror(errno));
 529                         continue;
 530                 }
 531                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 532                                 RTE_CACHE_LINE_SIZE);
 533                 if (chan_info == NULL) {
 534                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 535                                         "channel '%s'\n", socket_path);
 536                         continue;
 537                 }
 538                 snprintf(chan_info->channel_path,
 539                                 sizeof(chan_info->channel_path), "%s%s.%u",
 540                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 541                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 542                         rte_free(chan_info);
 543                         continue;
 544                 }
 545                 num_channels_enabled++;
 546
 547         }
 548         return num_channels_enabled;
 549 }
 550
 551 int
 552 add_host_channels(void)
 553 {
 554         struct channel_info *chan_info;
 555         char socket_path[PATH_MAX];
 556         int num_channels_enabled = 0;
 557         int ret;
 558         struct core_info *ci;
 559         struct channel_info *chan_infos[RTE_MAX_LCORE];
 560         int i;
 561
 562         for (i = 0; i < RTE_MAX_LCORE; i++)
 563                 chan_infos[i] = NULL;
 564
 565         ci = get_core_info();
 566         if (ci == NULL) {
 567                 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot allocate memory for core_info\n");
 568                 return 0;
 569         }
 570
 571         for (i = 0; i < ci->core_count; i++) {
 572                 if (ci->cd[i].global_enabled_cpus == 0)
 573                         continue;
 574
 575                 ret = fifo_path(socket_path, sizeof(socket_path), i);
 576                 if (ret < 0)
 577                         goto error;
 578
 579                 ret = mkfifo(socket_path, 0660);
 580                 RTE_LOG(DEBUG, CHANNEL_MANAGER, "TRY CREATE fifo '%s'\n",
 581                         socket_path);
 582                 if ((errno != EEXIST) && (ret < 0)) {
 583                         RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: "
 584                                         "%s\n", socket_path, strerror(errno));
 585                         goto error;
 586                 }
 587                 chan_info = rte_malloc(NULL, sizeof(*chan_info), 0);
 588                 if (chan_info == NULL) {
 589                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 590                                         "channel '%s'\n", socket_path);
 591                         goto error;
 592                 }
 593                 chan_infos[i] = chan_info;
 594                 strlcpy(chan_info->channel_path, socket_path,
 595                                 sizeof(chan_info->channel_path));
 596
 597                 if (setup_host_channel_info(&chan_info, i) < 0) {
 598                         rte_free(chan_info);
 599                         chan_infos[i] = NULL;
 600                         goto error;
 601                 }
 602                 num_channels_enabled++;
 603         }
 604
 605         return num_channels_enabled;
 606 error:
 607         /* Clean up the channels opened before we hit an error. */
 608         for (i = 0; i < ci->core_count; i++) {
 609                 if (chan_infos[i] != NULL) {
 610                         remove_channel_from_monitor(chan_infos[i]);
 611                         close(chan_infos[i]->fd);
 612                         rte_free(chan_infos[i]);
 613                 }
 614         }
 615         return 0;
 616 }
 617
 618 int
 619 remove_channel(struct channel_info **chan_info_dptr)
 620 {
 621         struct virtual_machine_info *vm_info;
 622         struct channel_info *chan_info = *chan_info_dptr;
 623
 624         close(chan_info->fd);
 625
 626         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 627
 628         rte_spinlock_lock(&(vm_info->config_spinlock));
 629         vm_info->channel_mask[chan_info->channel_num] = 0;
 630         vm_info->num_channels--;
 631         rte_spinlock_unlock(&(vm_info->config_spinlock));
 632
 633         rte_free(chan_info);
 634         return 0;
 635 }
 636
 637 int
 638 set_channel_status_all(const char *vm_name, enum channel_status status)
 639 {
 640         struct virtual_machine_info *vm_info;
 641         unsigned i;
 642         char mask[RTE_MAX_LCORE];
 643         int num_channels_changed = 0;
 644
 645         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 646                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 647                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 648                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 649         }
 650         vm_info = find_domain_by_name(vm_name);
 651         if (vm_info == NULL) {
 652                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 653                                 "not found\n", vm_name);
 654                 return 0;
 655         }
 656
 657         rte_spinlock_lock(&(vm_info->config_spinlock));
 658         memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
 659         for (i = 0; i < RTE_MAX_LCORE; i++) {
 660                 if (mask[i] != 1)
 661                         continue;
 662                 vm_info->channels[i]->status = status;
 663                 num_channels_changed++;
 664         }
 665         rte_spinlock_unlock(&(vm_info->config_spinlock));
 666         return num_channels_changed;
 667
 668 }
 669
 670 int
 671 set_channel_status(const char *vm_name, unsigned *channel_list,
 672                 unsigned len_channel_list, enum channel_status status)
 673 {
 674         struct virtual_machine_info *vm_info;
 675         unsigned i;
 676         int num_channels_changed = 0;
 677
 678         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 679                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 680                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 681                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 682         }
 683         vm_info = find_domain_by_name(vm_name);
 684         if (vm_info == NULL) {
 685                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 686                                 "not found\n", vm_name);
 687                 return 0;
 688         }
 689         for (i = 0; i < len_channel_list; i++) {
 690                 if (channel_exists(vm_info, channel_list[i])) {
 691                         rte_spinlock_lock(&(vm_info->config_spinlock));
 692                         vm_info->channels[channel_list[i]]->status = status;
 693                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 694                         num_channels_changed++;
 695                 }
 696         }
 697         return num_channels_changed;
 698 }
 699
 700 void
 701 get_all_vm(int *num_vm, int *num_vcpu)
 702 {
 703
 704         virNodeInfo node_info;
 705         virDomainPtr *domptr;
 706         int i, ii, numVcpus[MAX_VCPUS], n_vcpus;
 707         unsigned int jj;
 708         const char *vm_name;
 709         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 710                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 711         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 712
 713         if (!global_hypervisor_available)
 714                 return;
 715
 716         memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen);
 717         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 718                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 719                 return;
 720         }
 721
 722         /* Returns number of pcpus */
 723         global_n_host_cpus = (unsigned int)node_info.cpus;
 724
 725         /* Returns number of active domains */
 726         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 727                                         domain_flags);
 728         if (*num_vm <= 0) {
 729                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 730                 return;
 731         }
 732
 733         for (i = 0; i < *num_vm; i++) {
 734
 735                 /* Get Domain Names */
 736                 vm_name = virDomainGetName(domptr[i]);
 737                 lvm_info[i].vm_name = vm_name;
 738
 739                 /* Get Number of Vcpus */
 740                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 741
 742                 /* Get Number of VCpus & VcpuPinInfo */
 743                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 744                                 numVcpus[i], global_cpumaps,
 745                                 global_maplen, domain_flag);
 746
 747                 if ((int)n_vcpus > 0) {
 748                         *num_vcpu = n_vcpus;
 749                         lvm_info[i].num_cpus = n_vcpus;
 750                 }
 751
 752                 /* Save pcpu in use by libvirt VMs */
 753                 for (ii = 0; ii < n_vcpus; ii++) {
 754                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 755                                 if (VIR_CPU_USABLE(global_cpumaps,
 756                                                 global_maplen, ii, jj) > 0) {
 757                                         lvm_info[i].pcpus[ii] = jj;
 758                                 }
 759                         }
 760                 }
 761         }
 762 }
 763
 764 int
 765 get_info_vm(const char *vm_name, struct vm_info *info)
 766 {
 767         struct virtual_machine_info *vm_info;
 768         unsigned i, channel_num = 0;
 769         char mask[RTE_MAX_LCORE];
 770
 771         vm_info = find_domain_by_name(vm_name);
 772         if (vm_info == NULL) {
 773                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 774                 return -1;
 775         }
 776         info->status = CHANNEL_MGR_VM_ACTIVE;
 777         if (!virDomainIsActive(vm_info->domainPtr))
 778                 info->status = CHANNEL_MGR_VM_INACTIVE;
 779
 780         rte_spinlock_lock(&(vm_info->config_spinlock));
 781
 782         memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
 783         for (i = 0; i < RTE_MAX_LCORE; i++) {
 784                 if (mask[i] != 1)
 785                         continue;
 786                 info->channels[channel_num].channel_num = i;
 787                 memcpy(info->channels[channel_num].channel_path,
 788                                 vm_info->channels[i]->channel_path,
 789                                 UNIX_PATH_MAX);
 790                 info->channels[channel_num].status =
 791                                 vm_info->channels[i]->status;
 792                 info->channels[channel_num].fd =
 793                                 vm_info->channels[i]->fd;
 794                 channel_num++;
 795         }
 796
 797         info->allow_query = vm_info->allow_query;
 798         info->num_channels = channel_num;
 799         info->num_vcpus = vm_info->info.nrVirtCpu;
 800         rte_spinlock_unlock(&(vm_info->config_spinlock));
 801
 802         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 803         rte_spinlock_lock(&(vm_info->config_spinlock));
 804         for (i = 0; i < info->num_vcpus; i++) {
 805                 info->pcpu_map[i] = vm_info->pcpu_map[i];
 806         }
 807         rte_spinlock_unlock(&(vm_info->config_spinlock));
 808         return 0;
 809 }
 810
 811 int
 812 add_vm(const char *vm_name)
 813 {
 814         struct virtual_machine_info *new_domain;
 815         virDomainPtr dom_ptr;
 816         int i;
 817
 818         if (find_domain_by_name(vm_name) != NULL) {
 819                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 820                                 "already exists\n", vm_name);
 821                 return -1;
 822         }
 823
 824         if (global_vir_conn_ptr == NULL) {
 825                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 826                 return -1;
 827         }
 828         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 829         if (dom_ptr == NULL) {
 830                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 831                                 "VM '%s' not found\n", vm_name);
 832                 return -1;
 833         }
 834
 835         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 836                         RTE_CACHE_LINE_SIZE);
 837         if (new_domain == NULL) {
 838                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 839                                 "info\n");
 840                 return -1;
 841         }
 842         new_domain->domainPtr = dom_ptr;
 843         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 844                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 845                 rte_free(new_domain);
 846                 return -1;
 847         }
 848         if (new_domain->info.nrVirtCpu > RTE_MAX_LCORE) {
 849                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 850                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 851                                 RTE_MAX_LCORE);
 852                 rte_free(new_domain);
 853                 return -1;
 854         }
 855
 856         for (i = 0; i < RTE_MAX_LCORE; i++)
 857                 new_domain->pcpu_map[i] = 0;
 858
 859         if (update_pcpus_mask(new_domain) < 0) {
 860                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 861                 rte_free(new_domain);
 862                 return -1;
 863         }
 864         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 865         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 866         memset(new_domain->channel_mask, 0, RTE_MAX_LCORE);
 867         new_domain->num_channels = 0;
 868
 869         if (!virDomainIsActive(dom_ptr))
 870                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 871         else
 872                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 873
 874         new_domain->allow_query = 0;
 875         rte_spinlock_init(&(new_domain->config_spinlock));
 876         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 877         return 0;
 878 }
 879
 880 int
 881 remove_vm(const char *vm_name)
 882 {
 883         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 884
 885         if (vm_info == NULL) {
 886                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 887                                 "not found\n", vm_name);
 888                 return -1;
 889         }
 890         rte_spinlock_lock(&vm_info->config_spinlock);
 891         if (vm_info->num_channels != 0) {
 892                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 893                                 "%"PRId8" channels still active\n",
 894                                 vm_name, vm_info->num_channels);
 895                 rte_spinlock_unlock(&vm_info->config_spinlock);
 896                 return -1;
 897         }
 898         LIST_REMOVE(vm_info, vms_info);
 899         rte_spinlock_unlock(&vm_info->config_spinlock);
 900         rte_free(vm_info);
 901         return 0;
 902 }
 903
 904 int
 905 set_query_status(char *vm_name,
 906                 bool allow_query)
 907 {
 908         struct virtual_machine_info *vm_info;
 909
 910         vm_info = find_domain_by_name(vm_name);
 911         if (vm_info == NULL) {
 912                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 913                 return -1;
 914         }
 915         rte_spinlock_lock(&(vm_info->config_spinlock));
 916         vm_info->allow_query = allow_query ? 1 : 0;
 917         rte_spinlock_unlock(&(vm_info->config_spinlock));
 918         return 0;
 919 }
 920
 921 static void
 922 disconnect_hypervisor(void)
 923 {
 924         if (global_vir_conn_ptr != NULL) {
 925                 virConnectClose(global_vir_conn_ptr);
 926                 global_vir_conn_ptr = NULL;
 927         }
 928 }
 929
 930 static int
 931 connect_hypervisor(const char *path)
 932 {
 933         if (global_vir_conn_ptr != NULL) {
 934                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 935                                 "already established\n", path);
 936                 return -1;
 937         }
 938         global_vir_conn_ptr = virConnectOpen(path);
 939         if (global_vir_conn_ptr == NULL) {
 940                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 941                                 "Hypervisor '%s'\n", path);
 942                 return -1;
 943         }
 944         return 0;
 945 }
 946 int
 947 channel_manager_init(const char *path __rte_unused)
 948 {
 949         virNodeInfo info;
 950
 951         LIST_INIT(&vm_list_head);
 952         if (connect_hypervisor(path) < 0) {
 953                 global_n_host_cpus = 64;
 954                 global_hypervisor_available = 0;
 955                 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 956         } else {
 957                 global_hypervisor_available = 1;
 958
 959                 global_maplen = VIR_CPU_MAPLEN(RTE_MAX_LCORE);
 960
 961                 global_vircpuinfo = rte_zmalloc(NULL,
 962                                 sizeof(*global_vircpuinfo) *
 963                                 RTE_MAX_LCORE, RTE_CACHE_LINE_SIZE);
 964                 if (global_vircpuinfo == NULL) {
 965                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 966                         goto error;
 967                 }
 968                 global_cpumaps = rte_zmalloc(NULL,
 969                                 RTE_MAX_LCORE * global_maplen,
 970                                 RTE_CACHE_LINE_SIZE);
 971                 if (global_cpumaps == NULL)
 972                         goto error;
 973
 974                 if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 975                         RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 976                         goto error;
 977                 }
 978                 global_n_host_cpus = (unsigned int)info.cpus;
 979         }
 980
 981
 982
 983         if (global_n_host_cpus > RTE_MAX_LCORE) {
 984                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 985                                 "maximum of %u. No cores over %u should be used.\n",
 986                                 global_n_host_cpus, RTE_MAX_LCORE,
 987                                 RTE_MAX_LCORE - 1);
 988                 global_n_host_cpus = RTE_MAX_LCORE;
 989         }
 990
 991         return 0;
 992 error:
 993         if (global_hypervisor_available)
 994                 disconnect_hypervisor();
 995         return -1;
 996 }
 997
 998 void
 999 channel_manager_exit(void)
1000 {
1001         unsigned i;
1002         char mask[RTE_MAX_LCORE];
1003         struct virtual_machine_info *vm_info;
1004
1005         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
1006
1007                 rte_spinlock_lock(&(vm_info->config_spinlock));
1008
1009                 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
1010                 for (i = 0; i < RTE_MAX_LCORE; i++) {
1011                         if (mask[i] != 1)
1012                                 continue;
1013                         remove_channel_from_monitor(
1014                                         vm_info->channels[i]);
1015                         close(vm_info->channels[i]->fd);
1016                         rte_free(vm_info->channels[i]);
1017                 }
1018                 rte_spinlock_unlock(&(vm_info->config_spinlock));
1019
1020                 LIST_REMOVE(vm_info, vms_info);
1021                 rte_free(vm_info);
1022         }
1023
1024         if (global_hypervisor_available) {
1025                 /* Only needed if hypervisor available */
1026                 rte_free(global_cpumaps);
1027                 rte_free(global_vircpuinfo);
1028                 disconnect_hypervisor();
1029         }
1030 }