examples/vm_power_manager/channel_manager.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <fcntl.h>
   8 #include <unistd.h>
   9 #include <inttypes.h>
  10 #include <dirent.h>
  11 #include <errno.h>
  12
  13 #include <sys/queue.h>
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include <sys/socket.h>
  17 #include <sys/select.h>
  18
  19 #include <rte_string_fns.h>
  20 #include <rte_malloc.h>
  21 #include <rte_memory.h>
  22 #include <rte_mempool.h>
  23 #include <rte_log.h>
  24 #include <rte_spinlock.h>
  25
  26 #include <libvirt/libvirt.h>
  27
  28 #include "channel_manager.h"
  29 #include "channel_monitor.h"
  30 #include "power_manager.h"
  31
  32
  33 #define RTE_LOGTYPE_CHANNEL_MANAGER RTE_LOGTYPE_USER1
  34
  35 struct libvirt_vm_info lvm_info[MAX_CLIENTS];
  36
  37 /* Global pointer to libvirt connection */
  38 static virConnectPtr global_vir_conn_ptr;
  39
  40 static unsigned char *global_cpumaps;
  41 static virVcpuInfo *global_vircpuinfo;
  42 static size_t global_maplen;
  43
  44 static unsigned int global_n_host_cpus;
  45 static bool global_hypervisor_available;
  46
  47 /*
  48  * Represents a single Virtual Machine
  49  */
  50 struct virtual_machine_info {
  51         char name[CHANNEL_MGR_MAX_NAME_LEN];
  52         uint16_t pcpu_map[RTE_MAX_LCORE];
  53         struct channel_info *channels[RTE_MAX_LCORE];
  54         char channel_mask[RTE_MAX_LCORE];
  55         uint8_t num_channels;
  56         enum vm_status status;
  57         virDomainPtr domainPtr;
  58         virDomainInfo info;
  59         rte_spinlock_t config_spinlock;
  60         int allow_query;
  61         LIST_ENTRY(virtual_machine_info) vms_info;
  62 };
  63
  64 LIST_HEAD(, virtual_machine_info) vm_list_head;
  65
  66 static struct virtual_machine_info *
  67 find_domain_by_name(const char *name)
  68 {
  69         struct virtual_machine_info *info;
  70         LIST_FOREACH(info, &vm_list_head, vms_info) {
  71                 if (!strncmp(info->name, name, CHANNEL_MGR_MAX_NAME_LEN-1))
  72                         return info;
  73         }
  74         return NULL;
  75 }
  76
  77 static int
  78 update_pcpus_mask(struct virtual_machine_info *vm_info)
  79 {
  80         virVcpuInfoPtr cpuinfo;
  81         unsigned i, j;
  82         int n_vcpus;
  83
  84         memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen);
  85
  86         if (!virDomainIsActive(vm_info->domainPtr)) {
  87                 n_vcpus = virDomainGetVcpuPinInfo(vm_info->domainPtr,
  88                                 vm_info->info.nrVirtCpu, global_cpumaps, global_maplen,
  89                                 VIR_DOMAIN_AFFECT_CONFIG);
  90                 if (n_vcpus < 0) {
  91                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
  92                                         "in-active VM '%s'\n", vm_info->name);
  93                         return -1;
  94                 }
  95                 goto update_pcpus;
  96         }
  97
  98         memset(global_vircpuinfo, 0, sizeof(*global_vircpuinfo)*
  99                         RTE_MAX_LCORE);
 100
 101         cpuinfo = global_vircpuinfo;
 102
 103         n_vcpus = virDomainGetVcpus(vm_info->domainPtr, cpuinfo,
 104                         RTE_MAX_LCORE, global_cpumaps, global_maplen);
 105         if (n_vcpus < 0) {
 106                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting vCPU info for "
 107                                 "active VM '%s'\n", vm_info->name);
 108                 return -1;
 109         }
 110 update_pcpus:
 111         if (n_vcpus >= RTE_MAX_LCORE) {
 112                 RTE_LOG(ERR, CHANNEL_MANAGER, "Number of vCPUS(%u) is out of range "
 113                                 "0...%d\n", n_vcpus, RTE_MAX_LCORE-1);
 114                 return -1;
 115         }
 116         if (n_vcpus != vm_info->info.nrVirtCpu) {
 117                 RTE_LOG(INFO, CHANNEL_MANAGER, "Updating the number of vCPUs for VM '%s"
 118                                 " from %d -> %d\n", vm_info->name, vm_info->info.nrVirtCpu,
 119                                 n_vcpus);
 120                 vm_info->info.nrVirtCpu = n_vcpus;
 121         }
 122         rte_spinlock_lock(&(vm_info->config_spinlock));
 123         for (i = 0; i < vm_info->info.nrVirtCpu; i++) {
 124                 for (j = 0; j < global_n_host_cpus; j++) {
 125                         if (VIR_CPU_USABLE(global_cpumaps,
 126                                         global_maplen, i, j) <= 0)
 127                                 continue;
 128                         vm_info->pcpu_map[i] = j;
 129                 }
 130         }
 131         rte_spinlock_unlock(&(vm_info->config_spinlock));
 132         return 0;
 133 }
 134
 135 int
 136 set_pcpu(char *vm_name, unsigned int vcpu, unsigned int pcpu)
 137 {
 138         int flags = VIR_DOMAIN_AFFECT_LIVE|VIR_DOMAIN_AFFECT_CONFIG;
 139         struct virtual_machine_info *vm_info;
 140
 141         if (vcpu >= RTE_MAX_LCORE) {
 142                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds max allowable(%d)\n",
 143                                 vcpu, RTE_MAX_LCORE-1);
 144                 return -1;
 145         }
 146
 147         vm_info = find_domain_by_name(vm_name);
 148         if (vm_info == NULL) {
 149                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 150                 return -1;
 151         }
 152
 153         if (!virDomainIsActive(vm_info->domainPtr)) {
 154                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 155                                 " for VM '%s', VM is not active\n",
 156                                 vcpu, vm_info->name);
 157                 return -1;
 158         }
 159
 160         if (vcpu >= vm_info->info.nrVirtCpu) {
 161                 RTE_LOG(ERR, CHANNEL_MANAGER, "vCPU(%u) exceeds the assigned number of "
 162                                 "vCPUs(%u)\n", vcpu, vm_info->info.nrVirtCpu);
 163                 return -1;
 164         }
 165         memset(global_cpumaps, 0, RTE_MAX_LCORE * global_maplen);
 166
 167         VIR_USE_CPU(global_cpumaps, pcpu);
 168
 169         if (pcpu >= global_n_host_cpus) {
 170                 RTE_LOG(ERR, CHANNEL_MANAGER, "CPU(%u) exceeds the available "
 171                                 "number of CPUs(%u)\n",
 172                                 pcpu, global_n_host_cpus);
 173                 return -1;
 174         }
 175
 176         if (virDomainPinVcpuFlags(vm_info->domainPtr, vcpu, global_cpumaps,
 177                         global_maplen, flags) < 0) {
 178                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to set vCPU(%u) to pCPU "
 179                                 " for VM '%s'\n", vcpu,
 180                                 vm_info->name);
 181                 return -1;
 182         }
 183         rte_spinlock_lock(&(vm_info->config_spinlock));
 184         vm_info->pcpu_map[vcpu] = pcpu;
 185         rte_spinlock_unlock(&(vm_info->config_spinlock));
 186         return 0;
 187 }
 188
 189 uint16_t
 190 get_pcpu(struct channel_info *chan_info, unsigned int vcpu)
 191 {
 192         struct virtual_machine_info *vm_info =
 193                         (struct virtual_machine_info *)chan_info->priv_info;
 194
 195         if (global_hypervisor_available && (vm_info != NULL)) {
 196                 uint16_t pcpu;
 197                 rte_spinlock_lock(&(vm_info->config_spinlock));
 198                 pcpu = vm_info->pcpu_map[vcpu];
 199                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 200                 return pcpu;
 201         } else
 202                 return 0;
 203 }
 204
 205 static inline int
 206 channel_exists(struct virtual_machine_info *vm_info, unsigned channel_num)
 207 {
 208         rte_spinlock_lock(&(vm_info->config_spinlock));
 209         if (vm_info->channel_mask[channel_num] == 1) {
 210                 rte_spinlock_unlock(&(vm_info->config_spinlock));
 211                 return 1;
 212         }
 213         rte_spinlock_unlock(&(vm_info->config_spinlock));
 214         return 0;
 215 }
 216
 217
 218
 219 static int
 220 open_non_blocking_channel(struct channel_info *info)
 221 {
 222         int ret, flags;
 223         struct sockaddr_un sock_addr;
 224         fd_set soc_fd_set;
 225         struct timeval tv;
 226
 227         info->fd = socket(AF_UNIX, SOCK_STREAM, 0);
 228         if (info->fd < 0) {
 229                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) creating socket for '%s'\n",
 230                                 strerror(errno),
 231                                 info->channel_path);
 232                 return -1;
 233         }
 234         sock_addr.sun_family = AF_UNIX;
 235         memcpy(&sock_addr.sun_path, info->channel_path,
 236                         strlen(info->channel_path)+1);
 237
 238         /* Get current flags */
 239         flags = fcntl(info->fd, F_GETFL, 0);
 240         if (flags < 0) {
 241                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 242                                 "'%s'\n", strerror(errno), info->channel_path);
 243                 return 1;
 244         }
 245         /* Set to Non Blocking */
 246         flags |= O_NONBLOCK;
 247         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 248                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) setting non-blocking "
 249                                 "socket for '%s'\n", strerror(errno), info->channel_path);
 250                 return -1;
 251         }
 252         ret = connect(info->fd, (struct sockaddr *)&sock_addr,
 253                         sizeof(sock_addr));
 254         if (ret < 0) {
 255                 /* ECONNREFUSED error is given when VM is not active */
 256                 if (errno == ECONNREFUSED) {
 257                         RTE_LOG(WARNING, CHANNEL_MANAGER, "VM is not active or has not "
 258                                         "activated its endpoint to channel %s\n",
 259                                         info->channel_path);
 260                         return -1;
 261                 }
 262                 /* Wait for tv_sec if in progress */
 263                 else if (errno == EINPROGRESS) {
 264                         tv.tv_sec = 2;
 265                         tv.tv_usec = 0;
 266                         FD_ZERO(&soc_fd_set);
 267                         FD_SET(info->fd, &soc_fd_set);
 268                         if (select(info->fd+1, NULL, &soc_fd_set, NULL, &tv) > 0) {
 269                                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Timeout or error on channel "
 270                                                 "'%s'\n", info->channel_path);
 271                                 return -1;
 272                         }
 273                 } else {
 274                         /* Any other error */
 275                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) connecting socket"
 276                                         " for '%s'\n", strerror(errno), info->channel_path);
 277                         return -1;
 278                 }
 279         }
 280         return 0;
 281 }
 282
 283 static int
 284 open_host_channel(struct channel_info *info)
 285 {
 286         int flags;
 287
 288         info->fd = open(info->channel_path, O_RDWR | O_RSYNC);
 289         if (info->fd < 0) {
 290                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n",
 291                                 strerror(errno),
 292                                 info->channel_path);
 293                 return -1;
 294         }
 295
 296         /* Get current flags */
 297         flags = fcntl(info->fd, F_GETFL, 0);
 298         if (flags < 0) {
 299                 RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
 300                                 "'%s'\n", strerror(errno), info->channel_path);
 301                 return 1;
 302         }
 303         /* Set to Non Blocking */
 304         flags |= O_NONBLOCK;
 305         if (fcntl(info->fd, F_SETFL, flags) < 0) {
 306                 RTE_LOG(WARNING, CHANNEL_MANAGER,
 307                                 "Error(%s) setting non-blocking "
 308                                 "socket for '%s'\n",
 309                                 strerror(errno), info->channel_path);
 310                 return -1;
 311         }
 312         return 0;
 313 }
 314
 315 static int
 316 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 317                 struct channel_info **chan_info_dptr, unsigned channel_num)
 318 {
 319         struct channel_info *chan_info = *chan_info_dptr;
 320         struct virtual_machine_info *vm_info = *vm_info_dptr;
 321
 322         chan_info->channel_num = channel_num;
 323         chan_info->priv_info = (void *)vm_info;
 324         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 325         chan_info->type = CHANNEL_TYPE_BINARY;
 326         if (open_non_blocking_channel(chan_info) < 0) {
 327                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 328                                 "'%s' for VM '%s'\n",
 329                                 chan_info->channel_path, vm_info->name);
 330                 return -1;
 331         }
 332         if (add_channel_to_monitor(&chan_info) < 0) {
 333                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 334                                 "'%s' to epoll ctl for VM '%s'\n",
 335                                 chan_info->channel_path, vm_info->name);
 336                 return -1;
 337
 338         }
 339         rte_spinlock_lock(&(vm_info->config_spinlock));
 340         vm_info->num_channels++;
 341         vm_info->channel_mask[channel_num] = 1;
 342         vm_info->channels[channel_num] = chan_info;
 343         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 344         rte_spinlock_unlock(&(vm_info->config_spinlock));
 345         return 0;
 346 }
 347
 348 static int
 349 fifo_path(char *dst, unsigned int len, unsigned int id)
 350 {
 351         int cnt;
 352
 353         cnt = snprintf(dst, len, "%s%s%d", CHANNEL_MGR_SOCKET_PATH,
 354                         CHANNEL_MGR_FIFO_PATTERN_NAME, id);
 355
 356         if ((cnt < 0) || (cnt > (int)len - 1)) {
 357                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not create proper "
 358                         "string for fifo path\n");
 359
 360                 return -1;
 361         }
 362
 363         return 0;
 364 }
 365
 366 static int
 367 setup_host_channel_info(struct channel_info **chan_info_dptr,
 368                 unsigned int channel_num)
 369 {
 370         struct channel_info *chan_info = *chan_info_dptr;
 371
 372         chan_info->channel_num = channel_num;
 373         chan_info->priv_info = (void *)NULL;
 374         chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
 375         chan_info->type = CHANNEL_TYPE_JSON;
 376
 377         if (open_host_channel(chan_info) < 0) {
 378                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: "
 379                                 "'%s'\n",
 380                                 chan_info->channel_path);
 381                 return -1;
 382         }
 383         if (add_channel_to_monitor(&chan_info) < 0) {
 384                 RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
 385                                 "'%s' to epoll ctl\n",
 386                                 chan_info->channel_path);
 387                 return -1;
 388
 389         }
 390         chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
 391         return 0;
 392 }
 393
 394 int
 395 add_all_channels(const char *vm_name)
 396 {
 397         DIR *d;
 398         struct dirent *dir;
 399         struct virtual_machine_info *vm_info;
 400         struct channel_info *chan_info;
 401         char *token, *remaining, *tail_ptr;
 402         char socket_name[PATH_MAX];
 403         unsigned channel_num;
 404         int num_channels_enabled = 0;
 405
 406         /* verify VM exists */
 407         vm_info = find_domain_by_name(vm_name);
 408         if (vm_info == NULL) {
 409                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' not found"
 410                                 " during channel discovery\n", vm_name);
 411                 return 0;
 412         }
 413         if (!virDomainIsActive(vm_info->domainPtr)) {
 414                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 415                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 416                 return 0;
 417         }
 418         d = opendir(CHANNEL_MGR_SOCKET_PATH);
 419         if (d == NULL) {
 420                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error opening directory '%s': %s\n",
 421                                 CHANNEL_MGR_SOCKET_PATH, strerror(errno));
 422                 return -1;
 423         }
 424         while ((dir = readdir(d)) != NULL) {
 425                 if (!strncmp(dir->d_name, ".", 1) ||
 426                                 !strncmp(dir->d_name, "..", 2))
 427                         continue;
 428
 429                 strlcpy(socket_name, dir->d_name, sizeof(socket_name));
 430                 remaining = socket_name;
 431                 /* Extract vm_name from "<vm_name>.<channel_num>" */
 432                 token = strsep(&remaining, ".");
 433                 if (remaining == NULL)
 434                         continue;
 435                 if (strncmp(vm_name, token, CHANNEL_MGR_MAX_NAME_LEN))
 436                         continue;
 437
 438                 /* remaining should contain only <channel_num> */
 439                 errno = 0;
 440                 channel_num = (unsigned)strtol(remaining, &tail_ptr, 0);
 441                 if ((errno != 0) || (remaining[0] == '\0') ||
 442                                 tail_ptr == NULL || (*tail_ptr != '\0')) {
 443                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Malformed channel name"
 444                                         "'%s' found it should be in the form of "
 445                                         "'<guest_name>.<channel_num>(decimal)'\n",
 446                                         dir->d_name);
 447                         continue;
 448                 }
 449                 if (channel_num >= RTE_MAX_LCORE) {
 450                         RTE_LOG(WARNING, CHANNEL_MANAGER, "Channel number(%u) is "
 451                                         "greater than max allowable: %d, skipping '%s%s'\n",
 452                                         channel_num, RTE_MAX_LCORE-1,
 453                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 454                         continue;
 455                 }
 456                 /* if channel has not been added previously */
 457                 if (channel_exists(vm_info, channel_num))
 458                         continue;
 459
 460                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 461                                 RTE_CACHE_LINE_SIZE);
 462                 if (chan_info == NULL) {
 463                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 464                                 "channel '%s%s'\n", CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 465                         continue;
 466                 }
 467
 468                 if ((size_t)snprintf(chan_info->channel_path,
 469                                 sizeof(chan_info->channel_path), "%s%s",
 470                                 CHANNEL_MGR_SOCKET_PATH, dir->d_name)
 471                                         >= sizeof(chan_info->channel_path)) {
 472                         RTE_LOG(ERR, CHANNEL_MANAGER, "Pathname too long for channel '%s%s'\n",
 473                                         CHANNEL_MGR_SOCKET_PATH, dir->d_name);
 474                         rte_free(chan_info);
 475                         continue;
 476                 }
 477
 478                 if (setup_channel_info(&vm_info, &chan_info, channel_num) < 0) {
 479                         rte_free(chan_info);
 480                         continue;
 481                 }
 482
 483                 num_channels_enabled++;
 484         }
 485         closedir(d);
 486         return num_channels_enabled;
 487 }
 488
 489 int
 490 add_channels(const char *vm_name, unsigned *channel_list,
 491                 unsigned len_channel_list)
 492 {
 493         struct virtual_machine_info *vm_info;
 494         struct channel_info *chan_info;
 495         char socket_path[PATH_MAX];
 496         unsigned i;
 497         int num_channels_enabled = 0;
 498
 499         vm_info = find_domain_by_name(vm_name);
 500         if (vm_info == NULL) {
 501                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 502                                 "not found\n", vm_name);
 503                 return 0;
 504         }
 505
 506         if (!virDomainIsActive(vm_info->domainPtr)) {
 507                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM: '%s' is not active\n", vm_name);
 508                 vm_info->status = CHANNEL_MGR_VM_INACTIVE;
 509                 return 0;
 510         }
 511
 512         for (i = 0; i < len_channel_list; i++) {
 513                 if (channel_list[i] >= RTE_MAX_LCORE) {
 514                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel(%u) is out of range "
 515                                                         "0...%d\n", channel_list[i],
 516                                                         RTE_MAX_LCORE-1);
 517                         continue;
 518                 }
 519                 if (channel_exists(vm_info, channel_list[i])) {
 520                         RTE_LOG(INFO, CHANNEL_MANAGER, "Channel already exists, skipping  "
 521                                         "'%s.%u'\n", vm_name, i);
 522                         continue;
 523                 }
 524
 525                 snprintf(socket_path, sizeof(socket_path), "%s%s.%u",
 526                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 527                 errno = 0;
 528                 if (access(socket_path, F_OK) < 0) {
 529                         RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
 530                                         "%s\n", socket_path, strerror(errno));
 531                         continue;
 532                 }
 533                 chan_info = rte_malloc(NULL, sizeof(*chan_info),
 534                                 RTE_CACHE_LINE_SIZE);
 535                 if (chan_info == NULL) {
 536                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 537                                         "channel '%s'\n", socket_path);
 538                         continue;
 539                 }
 540                 snprintf(chan_info->channel_path,
 541                                 sizeof(chan_info->channel_path), "%s%s.%u",
 542                                 CHANNEL_MGR_SOCKET_PATH, vm_name, channel_list[i]);
 543                 if (setup_channel_info(&vm_info, &chan_info, channel_list[i]) < 0) {
 544                         rte_free(chan_info);
 545                         continue;
 546                 }
 547                 num_channels_enabled++;
 548
 549         }
 550         return num_channels_enabled;
 551 }
 552
 553 int
 554 add_host_channels(void)
 555 {
 556         struct channel_info *chan_info;
 557         char socket_path[PATH_MAX];
 558         int num_channels_enabled = 0;
 559         int ret;
 560         struct core_info *ci;
 561         struct channel_info *chan_infos[RTE_MAX_LCORE];
 562         int i;
 563
 564         for (i = 0; i < RTE_MAX_LCORE; i++)
 565                 chan_infos[i] = NULL;
 566
 567         ci = get_core_info();
 568         if (ci == NULL) {
 569                 RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot allocate memory for core_info\n");
 570                 return 0;
 571         }
 572
 573         for (i = 0; i < ci->core_count; i++) {
 574                 if (rte_lcore_index(i) == -1)
 575                         continue;
 576
 577                 if (ci->cd[i].global_enabled_cpus == 0)
 578                         continue;
 579
 580                 ret = fifo_path(socket_path, sizeof(socket_path), i);
 581                 if (ret < 0)
 582                         goto error;
 583
 584                 ret = mkfifo(socket_path, 0660);
 585                 RTE_LOG(DEBUG, CHANNEL_MANAGER, "TRY CREATE fifo '%s'\n",
 586                         socket_path);
 587                 if ((errno != EEXIST) && (ret < 0)) {
 588                         RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: "
 589                                         "%s\n", socket_path, strerror(errno));
 590                         goto error;
 591                 }
 592                 chan_info = rte_malloc(NULL, sizeof(*chan_info), 0);
 593                 if (chan_info == NULL) {
 594                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
 595                                         "channel '%s'\n", socket_path);
 596                         goto error;
 597                 }
 598                 chan_infos[i] = chan_info;
 599                 strlcpy(chan_info->channel_path, socket_path,
 600                                 sizeof(chan_info->channel_path));
 601
 602                 if (setup_host_channel_info(&chan_info, i) < 0) {
 603                         rte_free(chan_info);
 604                         chan_infos[i] = NULL;
 605                         goto error;
 606                 }
 607                 num_channels_enabled++;
 608         }
 609
 610         return num_channels_enabled;
 611 error:
 612         /* Clean up the channels opened before we hit an error. */
 613         for (i = 0; i < ci->core_count; i++) {
 614                 if (chan_infos[i] != NULL) {
 615                         remove_channel_from_monitor(chan_infos[i]);
 616                         close(chan_infos[i]->fd);
 617                         rte_free(chan_infos[i]);
 618                 }
 619         }
 620         return 0;
 621 }
 622
 623 int
 624 remove_channel(struct channel_info **chan_info_dptr)
 625 {
 626         struct virtual_machine_info *vm_info;
 627         struct channel_info *chan_info = *chan_info_dptr;
 628
 629         close(chan_info->fd);
 630
 631         vm_info = (struct virtual_machine_info *)chan_info->priv_info;
 632
 633         rte_spinlock_lock(&(vm_info->config_spinlock));
 634         vm_info->channel_mask[chan_info->channel_num] = 0;
 635         vm_info->num_channels--;
 636         rte_spinlock_unlock(&(vm_info->config_spinlock));
 637
 638         rte_free(chan_info);
 639         return 0;
 640 }
 641
 642 int
 643 set_channel_status_all(const char *vm_name, enum channel_status status)
 644 {
 645         struct virtual_machine_info *vm_info;
 646         unsigned i;
 647         char mask[RTE_MAX_LCORE];
 648         int num_channels_changed = 0;
 649
 650         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 651                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 652                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 653                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 654         }
 655         vm_info = find_domain_by_name(vm_name);
 656         if (vm_info == NULL) {
 657                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to disable channels: VM '%s' "
 658                                 "not found\n", vm_name);
 659                 return 0;
 660         }
 661
 662         rte_spinlock_lock(&(vm_info->config_spinlock));
 663         memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
 664         for (i = 0; i < RTE_MAX_LCORE; i++) {
 665                 if (mask[i] != 1)
 666                         continue;
 667                 vm_info->channels[i]->status = status;
 668                 num_channels_changed++;
 669         }
 670         rte_spinlock_unlock(&(vm_info->config_spinlock));
 671         return num_channels_changed;
 672
 673 }
 674
 675 int
 676 set_channel_status(const char *vm_name, unsigned *channel_list,
 677                 unsigned len_channel_list, enum channel_status status)
 678 {
 679         struct virtual_machine_info *vm_info;
 680         unsigned i;
 681         int num_channels_changed = 0;
 682
 683         if (!(status == CHANNEL_MGR_CHANNEL_CONNECTED ||
 684                         status == CHANNEL_MGR_CHANNEL_DISABLED)) {
 685                 RTE_LOG(ERR, CHANNEL_MANAGER, "Channels can only be enabled or "
 686                                 "disabled: Unable to change status for VM '%s'\n", vm_name);
 687         }
 688         vm_info = find_domain_by_name(vm_name);
 689         if (vm_info == NULL) {
 690                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add channels: VM '%s' "
 691                                 "not found\n", vm_name);
 692                 return 0;
 693         }
 694         for (i = 0; i < len_channel_list; i++) {
 695                 if (channel_exists(vm_info, channel_list[i])) {
 696                         rte_spinlock_lock(&(vm_info->config_spinlock));
 697                         vm_info->channels[channel_list[i]]->status = status;
 698                         rte_spinlock_unlock(&(vm_info->config_spinlock));
 699                         num_channels_changed++;
 700                 }
 701         }
 702         return num_channels_changed;
 703 }
 704
 705 void
 706 get_all_vm(int *num_vm, int *num_vcpu)
 707 {
 708
 709         virNodeInfo node_info;
 710         virDomainPtr *domptr;
 711         int i, ii, numVcpus[MAX_VCPUS], n_vcpus;
 712         unsigned int jj;
 713         const char *vm_name;
 714         unsigned int domain_flags = VIR_CONNECT_LIST_DOMAINS_RUNNING |
 715                                 VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 716         unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 717
 718         if (!global_hypervisor_available)
 719                 return;
 720
 721         memset(global_cpumaps, 0, RTE_MAX_LCORE*global_maplen);
 722         if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
 723                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 724                 return;
 725         }
 726
 727         /* Returns number of pcpus */
 728         global_n_host_cpus = (unsigned int)node_info.cpus;
 729
 730         /* Returns number of active domains */
 731         *num_vm = virConnectListAllDomains(global_vir_conn_ptr, &domptr,
 732                                         domain_flags);
 733         if (*num_vm <= 0) {
 734                 RTE_LOG(ERR, CHANNEL_MANAGER, "No Active Domains Running\n");
 735                 return;
 736         }
 737
 738         for (i = 0; i < *num_vm; i++) {
 739
 740                 /* Get Domain Names */
 741                 vm_name = virDomainGetName(domptr[i]);
 742                 lvm_info[i].vm_name = vm_name;
 743
 744                 /* Get Number of Vcpus */
 745                 numVcpus[i] = virDomainGetVcpusFlags(domptr[i], domain_flag);
 746
 747                 /* Get Number of VCpus & VcpuPinInfo */
 748                 n_vcpus = virDomainGetVcpuPinInfo(domptr[i],
 749                                 numVcpus[i], global_cpumaps,
 750                                 global_maplen, domain_flag);
 751
 752                 if ((int)n_vcpus > 0) {
 753                         *num_vcpu = n_vcpus;
 754                         lvm_info[i].num_cpus = n_vcpus;
 755                 }
 756
 757                 /* Save pcpu in use by libvirt VMs */
 758                 for (ii = 0; ii < n_vcpus; ii++) {
 759                         for (jj = 0; jj < global_n_host_cpus; jj++) {
 760                                 if (VIR_CPU_USABLE(global_cpumaps,
 761                                                 global_maplen, ii, jj) > 0) {
 762                                         lvm_info[i].pcpus[ii] = jj;
 763                                 }
 764                         }
 765                 }
 766         }
 767 }
 768
 769 int
 770 get_info_vm(const char *vm_name, struct vm_info *info)
 771 {
 772         struct virtual_machine_info *vm_info;
 773         unsigned i, channel_num = 0;
 774         char mask[RTE_MAX_LCORE];
 775
 776         vm_info = find_domain_by_name(vm_name);
 777         if (vm_info == NULL) {
 778                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 779                 return -1;
 780         }
 781         info->status = CHANNEL_MGR_VM_ACTIVE;
 782         if (!virDomainIsActive(vm_info->domainPtr))
 783                 info->status = CHANNEL_MGR_VM_INACTIVE;
 784
 785         rte_spinlock_lock(&(vm_info->config_spinlock));
 786
 787         memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
 788         for (i = 0; i < RTE_MAX_LCORE; i++) {
 789                 if (mask[i] != 1)
 790                         continue;
 791                 info->channels[channel_num].channel_num = i;
 792                 memcpy(info->channels[channel_num].channel_path,
 793                                 vm_info->channels[i]->channel_path,
 794                                 UNIX_PATH_MAX);
 795                 info->channels[channel_num].status =
 796                                 vm_info->channels[i]->status;
 797                 info->channels[channel_num].fd =
 798                                 vm_info->channels[i]->fd;
 799                 channel_num++;
 800         }
 801
 802         info->allow_query = vm_info->allow_query;
 803         info->num_channels = channel_num;
 804         info->num_vcpus = vm_info->info.nrVirtCpu;
 805         rte_spinlock_unlock(&(vm_info->config_spinlock));
 806
 807         memcpy(info->name, vm_info->name, sizeof(vm_info->name));
 808         rte_spinlock_lock(&(vm_info->config_spinlock));
 809         for (i = 0; i < info->num_vcpus; i++) {
 810                 info->pcpu_map[i] = vm_info->pcpu_map[i];
 811         }
 812         rte_spinlock_unlock(&(vm_info->config_spinlock));
 813         return 0;
 814 }
 815
 816 int
 817 add_vm(const char *vm_name)
 818 {
 819         struct virtual_machine_info *new_domain;
 820         virDomainPtr dom_ptr;
 821         int i;
 822
 823         if (find_domain_by_name(vm_name) != NULL) {
 824                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to add VM: VM '%s' "
 825                                 "already exists\n", vm_name);
 826                 return -1;
 827         }
 828
 829         if (global_vir_conn_ptr == NULL) {
 830                 RTE_LOG(ERR, CHANNEL_MANAGER, "No connection to hypervisor exists\n");
 831                 return -1;
 832         }
 833         dom_ptr = virDomainLookupByName(global_vir_conn_ptr, vm_name);
 834         if (dom_ptr == NULL) {
 835                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error on VM lookup with libvirt: "
 836                                 "VM '%s' not found\n", vm_name);
 837                 return -1;
 838         }
 839
 840         new_domain = rte_malloc("virtual_machine_info", sizeof(*new_domain),
 841                         RTE_CACHE_LINE_SIZE);
 842         if (new_domain == NULL) {
 843                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to allocate memory for VM "
 844                                 "info\n");
 845                 return -1;
 846         }
 847         new_domain->domainPtr = dom_ptr;
 848         if (virDomainGetInfo(new_domain->domainPtr, &new_domain->info) != 0) {
 849                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to get libvirt VM info\n");
 850                 rte_free(new_domain);
 851                 return -1;
 852         }
 853         if (new_domain->info.nrVirtCpu > RTE_MAX_LCORE) {
 854                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error the number of virtual CPUs(%u) is "
 855                                 "greater than allowable(%d)\n", new_domain->info.nrVirtCpu,
 856                                 RTE_MAX_LCORE);
 857                 rte_free(new_domain);
 858                 return -1;
 859         }
 860
 861         for (i = 0; i < RTE_MAX_LCORE; i++)
 862                 new_domain->pcpu_map[i] = 0;
 863
 864         if (update_pcpus_mask(new_domain) < 0) {
 865                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error getting physical CPU pinning\n");
 866                 rte_free(new_domain);
 867                 return -1;
 868         }
 869         strncpy(new_domain->name, vm_name, sizeof(new_domain->name));
 870         new_domain->name[sizeof(new_domain->name) - 1] = '\0';
 871         memset(new_domain->channel_mask, 0, RTE_MAX_LCORE);
 872         new_domain->num_channels = 0;
 873
 874         if (!virDomainIsActive(dom_ptr))
 875                 new_domain->status = CHANNEL_MGR_VM_INACTIVE;
 876         else
 877                 new_domain->status = CHANNEL_MGR_VM_ACTIVE;
 878
 879         new_domain->allow_query = 0;
 880         rte_spinlock_init(&(new_domain->config_spinlock));
 881         LIST_INSERT_HEAD(&vm_list_head, new_domain, vms_info);
 882         return 0;
 883 }
 884
 885 int
 886 remove_vm(const char *vm_name)
 887 {
 888         struct virtual_machine_info *vm_info = find_domain_by_name(vm_name);
 889
 890         if (vm_info == NULL) {
 891                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM: VM '%s' "
 892                                 "not found\n", vm_name);
 893                 return -1;
 894         }
 895         rte_spinlock_lock(&vm_info->config_spinlock);
 896         if (vm_info->num_channels != 0) {
 897                 RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to remove VM '%s', there are "
 898                                 "%"PRId8" channels still active\n",
 899                                 vm_name, vm_info->num_channels);
 900                 rte_spinlock_unlock(&vm_info->config_spinlock);
 901                 return -1;
 902         }
 903         LIST_REMOVE(vm_info, vms_info);
 904         rte_spinlock_unlock(&vm_info->config_spinlock);
 905         rte_free(vm_info);
 906         return 0;
 907 }
 908
 909 int
 910 set_query_status(char *vm_name,
 911                 bool allow_query)
 912 {
 913         struct virtual_machine_info *vm_info;
 914
 915         vm_info = find_domain_by_name(vm_name);
 916         if (vm_info == NULL) {
 917                 RTE_LOG(ERR, CHANNEL_MANAGER, "VM '%s' not found\n", vm_name);
 918                 return -1;
 919         }
 920         rte_spinlock_lock(&(vm_info->config_spinlock));
 921         vm_info->allow_query = allow_query ? 1 : 0;
 922         rte_spinlock_unlock(&(vm_info->config_spinlock));
 923         return 0;
 924 }
 925
 926 static void
 927 disconnect_hypervisor(void)
 928 {
 929         if (global_vir_conn_ptr != NULL) {
 930                 virConnectClose(global_vir_conn_ptr);
 931                 global_vir_conn_ptr = NULL;
 932         }
 933 }
 934
 935 static int
 936 connect_hypervisor(const char *path)
 937 {
 938         if (global_vir_conn_ptr != NULL) {
 939                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error connecting to %s, connection "
 940                                 "already established\n", path);
 941                 return -1;
 942         }
 943         global_vir_conn_ptr = virConnectOpen(path);
 944         if (global_vir_conn_ptr == NULL) {
 945                 RTE_LOG(ERR, CHANNEL_MANAGER, "Error failed to open connection to "
 946                                 "Hypervisor '%s'\n", path);
 947                 return -1;
 948         }
 949         return 0;
 950 }
 951 int
 952 channel_manager_init(const char *path __rte_unused)
 953 {
 954         virNodeInfo info;
 955
 956         LIST_INIT(&vm_list_head);
 957         if (connect_hypervisor(path) < 0) {
 958                 global_n_host_cpus = 64;
 959                 global_hypervisor_available = 0;
 960                 RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
 961         } else {
 962                 global_hypervisor_available = 1;
 963
 964                 global_maplen = VIR_CPU_MAPLEN(RTE_MAX_LCORE);
 965
 966                 global_vircpuinfo = rte_zmalloc(NULL,
 967                                 sizeof(*global_vircpuinfo) *
 968                                 RTE_MAX_LCORE, RTE_CACHE_LINE_SIZE);
 969                 if (global_vircpuinfo == NULL) {
 970                         RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
 971                         goto error;
 972                 }
 973                 global_cpumaps = rte_zmalloc(NULL,
 974                                 RTE_MAX_LCORE * global_maplen,
 975                                 RTE_CACHE_LINE_SIZE);
 976                 if (global_cpumaps == NULL)
 977                         goto error;
 978
 979                 if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
 980                         RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
 981                         goto error;
 982                 }
 983                 global_n_host_cpus = (unsigned int)info.cpus;
 984         }
 985
 986
 987
 988         if (global_n_host_cpus > RTE_MAX_LCORE) {
 989                 RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
 990                                 "maximum of %u. No cores over %u should be used.\n",
 991                                 global_n_host_cpus, RTE_MAX_LCORE,
 992                                 RTE_MAX_LCORE - 1);
 993                 global_n_host_cpus = RTE_MAX_LCORE;
 994         }
 995
 996         return 0;
 997 error:
 998         if (global_hypervisor_available)
 999                 disconnect_hypervisor();
1000         return -1;
1001 }
1002
1003 void
1004 channel_manager_exit(void)
1005 {
1006         unsigned i;
1007         char mask[RTE_MAX_LCORE];
1008         struct virtual_machine_info *vm_info;
1009
1010         LIST_FOREACH(vm_info, &vm_list_head, vms_info) {
1011
1012                 rte_spinlock_lock(&(vm_info->config_spinlock));
1013
1014                 memcpy(mask, (char *)vm_info->channel_mask, RTE_MAX_LCORE);
1015                 for (i = 0; i < RTE_MAX_LCORE; i++) {
1016                         if (mask[i] != 1)
1017                                 continue;
1018                         remove_channel_from_monitor(
1019                                         vm_info->channels[i]);
1020                         close(vm_info->channels[i]->fd);
1021                         rte_free(vm_info->channels[i]);
1022                 }
1023                 rte_spinlock_unlock(&(vm_info->config_spinlock));
1024
1025                 LIST_REMOVE(vm_info, vms_info);
1026                 rte_free(vm_info);
1027         }
1028
1029         if (global_hypervisor_available) {
1030                 /* Only needed if hypervisor available */
1031                 rte_free(global_cpumaps);
1032                 rte_free(global_vircpuinfo);
1033                 disconnect_hypervisor();
1034         }
1035 }