1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
13 #include <sys/types.h>
14 #include <sys/epoll.h>
15 #include <sys/queue.h>
17 #include <sys/socket.h>
18 #include <sys/select.h>
22 #pragma message "Jansson dev libs unavailable, not including JSON parsing"
24 #include <rte_string_fns.h>
26 #include <rte_memory.h>
27 #include <rte_malloc.h>
28 #include <rte_atomic.h>
29 #include <rte_cycles.h>
30 #include <rte_ethdev.h>
31 #include <rte_pmd_i40e.h>
33 #include <libvirt/libvirt.h>
34 #include "channel_monitor.h"
35 #include "channel_commands.h"
36 #include "channel_manager.h"
37 #include "power_manager.h"
38 #include "oob_monitor.h"
40 #define RTE_LOGTYPE_CHANNEL_MONITOR RTE_LOGTYPE_USER1
42 #define MAX_EVENTS 256
44 uint64_t vsi_pkt_count_prev[384];
45 uint64_t rdtsc_prev[384];
46 #define MAX_JSON_STRING_LEN 1024
47 char json_data[MAX_JSON_STRING_LEN];
49 double time_period_ms = 1;
50 static volatile unsigned run_loop = 1;
51 static int global_event_fd;
52 static unsigned int policy_is_set;
53 static struct epoll_event *global_events_list;
54 static struct policy policies[MAX_CLIENTS];
59 struct rte_ether_addr addr;
64 str_to_ether_addr(const char *a, struct rte_ether_addr *ether_addr)
68 unsigned long o[RTE_ETHER_ADDR_LEN];
73 o[i] = strtoul(a, &end, 16);
74 if (errno != 0 || end == a || (end[0] != ':' && end[0] != 0))
77 } while (++i != RTE_DIM(o) / sizeof(o[0]) && end[0] != 0);
79 /* Junk at the end of line */
83 /* Support the format XX:XX:XX:XX:XX:XX */
84 if (i == RTE_ETHER_ADDR_LEN) {
88 ether_addr->addr_bytes[i] = (uint8_t)o[i];
90 /* Support the format XXXX:XXXX:XXXX */
91 } else if (i == RTE_ETHER_ADDR_LEN / 2) {
93 if (o[i] > UINT16_MAX)
95 ether_addr->addr_bytes[i * 2] =
97 ether_addr->addr_bytes[i * 2 + 1] =
98 (uint8_t)(o[i] & 0xff);
108 set_policy_mac(struct channel_packet *pkt, int idx, char *mac)
113 /* Use port MAC address as the vfid */
114 ret = str_to_ether_addr(mac, &pfid.addr);
117 RTE_LOG(ERR, CHANNEL_MONITOR,
118 "Invalid mac address received in JSON\n");
123 printf("Received MAC Address: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 ":"
124 "%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n",
125 pfid.addr.addr_bytes[0], pfid.addr.addr_bytes[1],
126 pfid.addr.addr_bytes[2], pfid.addr.addr_bytes[3],
127 pfid.addr.addr_bytes[4], pfid.addr.addr_bytes[5]);
129 pkt->vfid[idx] = pfid.pfid;
135 parse_json_to_pkt(json_t *element, struct channel_packet *pkt)
141 memset(pkt, 0, sizeof(struct channel_packet));
143 pkt->nb_mac_to_monitor = 0;
144 pkt->t_boost_status.tbEnabled = false;
146 pkt->policy_to_use = TIME;
147 pkt->command = PKT_POLICY;
148 pkt->core_type = CORE_TYPE_PHYSICAL;
150 json_object_foreach(element, key, value) {
151 if (!strcmp(key, "policy")) {
152 /* Recurse in to get the contents of profile */
153 ret = parse_json_to_pkt(value, pkt);
156 } else if (!strcmp(key, "instruction")) {
157 /* Recurse in to get the contents of instruction */
158 ret = parse_json_to_pkt(value, pkt);
161 } else if (!strcmp(key, "name")) {
162 strlcpy(pkt->vm_name, json_string_value(value),
163 sizeof(pkt->vm_name));
164 } else if (!strcmp(key, "command")) {
166 strlcpy(command, json_string_value(value), 32);
167 if (!strcmp(command, "power")) {
168 pkt->command = CPU_POWER;
169 } else if (!strcmp(command, "create")) {
170 pkt->command = PKT_POLICY;
171 } else if (!strcmp(command, "destroy")) {
172 pkt->command = PKT_POLICY_REMOVE;
174 RTE_LOG(ERR, CHANNEL_MONITOR,
175 "Invalid command received in JSON\n");
178 } else if (!strcmp(key, "policy_type")) {
180 strlcpy(command, json_string_value(value), 32);
181 if (!strcmp(command, "TIME")) {
182 pkt->policy_to_use = TIME;
183 } else if (!strcmp(command, "TRAFFIC")) {
184 pkt->policy_to_use = TRAFFIC;
185 } else if (!strcmp(command, "WORKLOAD")) {
186 pkt->policy_to_use = WORKLOAD;
187 } else if (!strcmp(command, "BRANCH_RATIO")) {
188 pkt->policy_to_use = BRANCH_RATIO;
190 RTE_LOG(ERR, CHANNEL_MONITOR,
191 "Wrong policy_type received in JSON\n");
194 } else if (!strcmp(key, "workload")) {
196 strlcpy(command, json_string_value(value), 32);
197 if (!strcmp(command, "HIGH")) {
198 pkt->workload = HIGH;
199 } else if (!strcmp(command, "MEDIUM")) {
200 pkt->workload = MEDIUM;
201 } else if (!strcmp(command, "LOW")) {
204 RTE_LOG(ERR, CHANNEL_MONITOR,
205 "Wrong workload received in JSON\n");
208 } else if (!strcmp(key, "busy_hours")) {
210 size_t size = json_array_size(value);
212 for (i = 0; i < size; i++) {
213 int hour = (int)json_integer_value(
214 json_array_get(value, i));
215 pkt->timer_policy.busy_hours[i] = hour;
217 } else if (!strcmp(key, "quiet_hours")) {
219 size_t size = json_array_size(value);
221 for (i = 0; i < size; i++) {
222 int hour = (int)json_integer_value(
223 json_array_get(value, i));
224 pkt->timer_policy.quiet_hours[i] = hour;
226 } else if (!strcmp(key, "core_list")) {
228 size_t size = json_array_size(value);
230 for (i = 0; i < size; i++) {
231 int core = (int)json_integer_value(
232 json_array_get(value, i));
233 pkt->vcpu_to_control[i] = core;
235 pkt->num_vcpu = size;
236 } else if (!strcmp(key, "mac_list")) {
238 size_t size = json_array_size(value);
240 for (i = 0; i < size; i++) {
243 json_string_value(json_array_get(value, i)),
245 set_policy_mac(pkt, i, mac);
247 pkt->nb_mac_to_monitor = size;
248 } else if (!strcmp(key, "avg_packet_thresh")) {
249 pkt->traffic_policy.avg_max_packet_thresh =
250 (uint32_t)json_integer_value(value);
251 } else if (!strcmp(key, "max_packet_thresh")) {
252 pkt->traffic_policy.max_max_packet_thresh =
253 (uint32_t)json_integer_value(value);
254 } else if (!strcmp(key, "unit")) {
256 strlcpy(unit, json_string_value(value), 32);
257 if (!strcmp(unit, "SCALE_UP")) {
258 pkt->unit = CPU_POWER_SCALE_UP;
259 } else if (!strcmp(unit, "SCALE_DOWN")) {
260 pkt->unit = CPU_POWER_SCALE_DOWN;
261 } else if (!strcmp(unit, "SCALE_MAX")) {
262 pkt->unit = CPU_POWER_SCALE_MAX;
263 } else if (!strcmp(unit, "SCALE_MIN")) {
264 pkt->unit = CPU_POWER_SCALE_MIN;
265 } else if (!strcmp(unit, "ENABLE_TURBO")) {
266 pkt->unit = CPU_POWER_ENABLE_TURBO;
267 } else if (!strcmp(unit, "DISABLE_TURBO")) {
268 pkt->unit = CPU_POWER_DISABLE_TURBO;
270 RTE_LOG(ERR, CHANNEL_MONITOR,
271 "Invalid command received in JSON\n");
274 } else if (!strcmp(key, "resource_id")) {
275 pkt->resource_id = (uint32_t)json_integer_value(value);
277 RTE_LOG(ERR, CHANNEL_MONITOR,
278 "Unknown key received in JSON string: %s\n",
286 void channel_monitor_exit(void)
289 rte_free(global_events_list);
293 core_share(int pNo, int z, int x, int t)
295 if (policies[pNo].core_share[z].pcpu == lvm_info[x].pcpus[t]) {
296 if (strcmp(policies[pNo].pkt.vm_name,
297 lvm_info[x].vm_name) != 0) {
298 policies[pNo].core_share[z].status = 1;
299 power_manager_scale_core_max(
300 policies[pNo].core_share[z].pcpu);
306 core_share_status(int pNo)
309 int noVms = 0, noVcpus = 0, z, x, t;
311 get_all_vm(&noVms, &noVcpus);
313 /* Reset Core Share Status. */
314 for (z = 0; z < noVcpus; z++)
315 policies[pNo].core_share[z].status = 0;
317 /* Foreach vcpu in a policy. */
318 for (z = 0; z < policies[pNo].pkt.num_vcpu; z++) {
319 /* Foreach VM on the platform. */
320 for (x = 0; x < noVms; x++) {
321 /* Foreach vcpu of VMs on platform. */
322 for (t = 0; t < lvm_info[x].num_cpus; t++)
323 core_share(pNo, z, x, t);
330 pcpu_monitor(struct policy *pol, struct core_info *ci, int pcpu, int count)
334 if (pol->pkt.policy_to_use == BRANCH_RATIO) {
335 ci->cd[pcpu].oob_enabled = 1;
336 ret = add_core_to_monitor(pcpu);
338 RTE_LOG(INFO, CHANNEL_MONITOR,
339 "Monitoring pcpu %d OOB for %s\n",
340 pcpu, pol->pkt.vm_name);
342 RTE_LOG(ERR, CHANNEL_MONITOR,
343 "Error monitoring pcpu %d OOB for %s\n",
344 pcpu, pol->pkt.vm_name);
347 pol->core_share[count].pcpu = pcpu;
348 RTE_LOG(INFO, CHANNEL_MONITOR,
349 "Monitoring pcpu %d for %s\n",
350 pcpu, pol->pkt.vm_name);
356 get_pcpu_to_control(struct policy *pol)
359 /* Convert vcpu to pcpu. */
362 struct core_info *ci;
364 ci = get_core_info();
366 RTE_LOG(DEBUG, CHANNEL_MONITOR,
367 "Looking for pcpu for %s\n", pol->pkt.vm_name);
370 * So now that we're handling virtual and physical cores, we need to
371 * differenciate between them when adding them to the branch monitor.
372 * Virtual cores need to be converted to physical cores.
374 if (pol->pkt.core_type == CORE_TYPE_VIRTUAL) {
376 * If the cores in the policy are virtual, we need to map them
377 * to physical core. We look up the vm info and use that for
380 get_info_vm(pol->pkt.vm_name, &info);
381 for (count = 0; count < pol->pkt.num_vcpu; count++) {
382 pcpu = info.pcpu_map[pol->pkt.vcpu_to_control[count]];
383 pcpu_monitor(pol, ci, pcpu, count);
387 * If the cores in the policy are physical, we just use
388 * those core id's directly.
390 for (count = 0; count < pol->pkt.num_vcpu; count++) {
391 pcpu = pol->pkt.vcpu_to_control[count];
392 pcpu_monitor(pol, ci, pcpu, count);
398 get_pfid(struct policy *pol)
403 for (i = 0; i < pol->pkt.nb_mac_to_monitor; i++) {
405 RTE_ETH_FOREACH_DEV(x) {
406 ret = rte_pmd_i40e_query_vfid_by_mac(x,
407 (struct rte_ether_addr *)&(pol->pkt.vfid[i]));
408 if (ret != -EINVAL) {
413 if (ret == -EINVAL || ret == -ENOTSUP || ret == ENODEV) {
414 RTE_LOG(INFO, CHANNEL_MONITOR,
415 "Error with Policy. MAC not found on "
426 update_policy(struct channel_packet *pkt)
429 unsigned int updated = 0;
433 RTE_LOG(INFO, CHANNEL_MONITOR,
434 "Applying policy for %s\n", pkt->vm_name);
436 for (i = 0; i < MAX_CLIENTS; i++) {
437 if (strcmp(policies[i].pkt.vm_name, pkt->vm_name) == 0) {
438 /* Copy the contents of *pkt into the policy.pkt */
439 policies[i].pkt = *pkt;
440 get_pcpu_to_control(&policies[i]);
441 /* Check Eth dev only for Traffic policy */
442 if (policies[i].pkt.policy_to_use == TRAFFIC) {
443 if (get_pfid(&policies[i]) < 0) {
448 core_share_status(i);
449 policies[i].enabled = 1;
454 for (i = 0; i < MAX_CLIENTS; i++) {
455 if (policies[i].enabled == 0) {
456 policies[i].pkt = *pkt;
457 get_pcpu_to_control(&policies[i]);
458 /* Check Eth dev only for Traffic policy */
459 if (policies[i].pkt.policy_to_use == TRAFFIC) {
460 if (get_pfid(&policies[i]) < 0) {
465 core_share_status(i);
466 policies[i].enabled = 1;
475 remove_policy(struct channel_packet *pkt __rte_unused)
480 * Disabling the policy is simply a case of setting
483 for (i = 0; i < MAX_CLIENTS; i++) {
484 if (strcmp(policies[i].pkt.vm_name, pkt->vm_name) == 0) {
485 policies[i].enabled = 0;
493 get_pkt_diff(struct policy *pol)
496 uint64_t vsi_pkt_count,
498 vsi_pkt_count_prev_total = 0;
499 double rdtsc_curr, rdtsc_diff, diff;
501 struct rte_eth_stats vf_stats;
503 for (x = 0; x < pol->pkt.nb_mac_to_monitor; x++) {
506 if (rte_pmd_i40e_get_vf_stats(x, pol->pfid[x], &vf_stats) == 0)
507 vsi_pkt_count = vf_stats.ipackets;
511 vsi_pkt_total += vsi_pkt_count;
513 vsi_pkt_count_prev_total += vsi_pkt_count_prev[pol->pfid[x]];
514 vsi_pkt_count_prev[pol->pfid[x]] = vsi_pkt_count;
517 rdtsc_curr = rte_rdtsc_precise();
518 rdtsc_diff = rdtsc_curr - rdtsc_prev[pol->pfid[x-1]];
519 rdtsc_prev[pol->pfid[x-1]] = rdtsc_curr;
521 diff = (vsi_pkt_total - vsi_pkt_count_prev_total) *
522 ((double)rte_get_tsc_hz() / rdtsc_diff);
528 apply_traffic_profile(struct policy *pol)
534 diff = get_pkt_diff(pol);
536 if (diff >= (pol->pkt.traffic_policy.max_max_packet_thresh)) {
537 for (count = 0; count < pol->pkt.num_vcpu; count++) {
538 if (pol->core_share[count].status != 1)
539 power_manager_scale_core_max(
540 pol->core_share[count].pcpu);
542 } else if (diff >= (pol->pkt.traffic_policy.avg_max_packet_thresh)) {
543 for (count = 0; count < pol->pkt.num_vcpu; count++) {
544 if (pol->core_share[count].status != 1)
545 power_manager_scale_core_med(
546 pol->core_share[count].pcpu);
548 } else if (diff < (pol->pkt.traffic_policy.avg_max_packet_thresh)) {
549 for (count = 0; count < pol->pkt.num_vcpu; count++) {
550 if (pol->core_share[count].status != 1)
551 power_manager_scale_core_min(
552 pol->core_share[count].pcpu);
558 apply_time_profile(struct policy *pol)
564 char time_string[40];
566 /* Obtain the time of day, and convert it to a tm struct. */
567 gettimeofday(&tv, NULL);
568 ptm = localtime(&tv.tv_sec);
569 /* Format the date and time, down to a single second. */
570 strftime(time_string, sizeof(time_string), "%Y-%m-%d %H:%M:%S", ptm);
572 for (x = 0; x < HOURS; x++) {
574 if (ptm->tm_hour == pol->pkt.timer_policy.busy_hours[x]) {
575 for (count = 0; count < pol->pkt.num_vcpu; count++) {
576 if (pol->core_share[count].status != 1) {
577 power_manager_scale_core_max(
578 pol->core_share[count].pcpu);
582 } else if (ptm->tm_hour ==
583 pol->pkt.timer_policy.quiet_hours[x]) {
584 for (count = 0; count < pol->pkt.num_vcpu; count++) {
585 if (pol->core_share[count].status != 1) {
586 power_manager_scale_core_min(
587 pol->core_share[count].pcpu);
591 } else if (ptm->tm_hour ==
592 pol->pkt.timer_policy.hours_to_use_traffic_profile[x]) {
593 apply_traffic_profile(pol);
600 apply_workload_profile(struct policy *pol)
605 if (pol->pkt.workload == HIGH) {
606 for (count = 0; count < pol->pkt.num_vcpu; count++) {
607 if (pol->core_share[count].status != 1)
608 power_manager_scale_core_max(
609 pol->core_share[count].pcpu);
611 } else if (pol->pkt.workload == MEDIUM) {
612 for (count = 0; count < pol->pkt.num_vcpu; count++) {
613 if (pol->core_share[count].status != 1)
614 power_manager_scale_core_med(
615 pol->core_share[count].pcpu);
617 } else if (pol->pkt.workload == LOW) {
618 for (count = 0; count < pol->pkt.num_vcpu; count++) {
619 if (pol->core_share[count].status != 1)
620 power_manager_scale_core_min(
621 pol->core_share[count].pcpu);
627 apply_policy(struct policy *pol)
630 struct channel_packet *pkt = &pol->pkt;
632 /*Check policy to use*/
633 if (pkt->policy_to_use == TRAFFIC)
634 apply_traffic_profile(pol);
635 else if (pkt->policy_to_use == TIME)
636 apply_time_profile(pol);
637 else if (pkt->policy_to_use == WORKLOAD)
638 apply_workload_profile(pol);
642 process_request(struct channel_packet *pkt, struct channel_info *chan_info)
646 if (chan_info == NULL)
649 if (rte_atomic32_cmpset(&(chan_info->status), CHANNEL_MGR_CHANNEL_CONNECTED,
650 CHANNEL_MGR_CHANNEL_PROCESSING) == 0)
653 if (pkt->command == CPU_POWER) {
654 unsigned int core_num;
656 if (pkt->core_type == CORE_TYPE_VIRTUAL)
657 core_num = get_pcpu(chan_info, pkt->resource_id);
659 core_num = pkt->resource_id;
661 RTE_LOG(DEBUG, CHANNEL_MONITOR, "Processing requested cmd for cpu:%d\n",
665 case(CPU_POWER_SCALE_MIN):
666 power_manager_scale_core_min(core_num);
668 case(CPU_POWER_SCALE_MAX):
669 power_manager_scale_core_max(core_num);
671 case(CPU_POWER_SCALE_DOWN):
672 power_manager_scale_core_down(core_num);
674 case(CPU_POWER_SCALE_UP):
675 power_manager_scale_core_up(core_num);
677 case(CPU_POWER_ENABLE_TURBO):
678 power_manager_enable_turbo_core(core_num);
680 case(CPU_POWER_DISABLE_TURBO):
681 power_manager_disable_turbo_core(core_num);
688 if (pkt->command == PKT_POLICY) {
689 RTE_LOG(INFO, CHANNEL_MONITOR, "Processing policy request %s\n",
695 if (pkt->command == PKT_POLICY_REMOVE) {
696 ret = remove_policy(pkt);
698 RTE_LOG(INFO, CHANNEL_MONITOR,
699 "Removed policy %s\n", pkt->vm_name);
701 RTE_LOG(INFO, CHANNEL_MONITOR,
702 "Policy %s does not exist\n", pkt->vm_name);
706 * Return is not checked as channel status may have been set to DISABLED
707 * from management thread
709 rte_atomic32_cmpset(&(chan_info->status), CHANNEL_MGR_CHANNEL_PROCESSING,
710 CHANNEL_MGR_CHANNEL_CONNECTED);
716 add_channel_to_monitor(struct channel_info **chan_info)
718 struct channel_info *info = *chan_info;
719 struct epoll_event event;
721 event.events = EPOLLIN;
722 event.data.ptr = info;
723 if (epoll_ctl(global_event_fd, EPOLL_CTL_ADD, info->fd, &event) < 0) {
724 RTE_LOG(ERR, CHANNEL_MONITOR, "Unable to add channel '%s' "
725 "to epoll\n", info->channel_path);
728 RTE_LOG(ERR, CHANNEL_MONITOR, "Added channel '%s' "
729 "to monitor\n", info->channel_path);
734 remove_channel_from_monitor(struct channel_info *chan_info)
736 if (epoll_ctl(global_event_fd, EPOLL_CTL_DEL,
737 chan_info->fd, NULL) < 0) {
738 RTE_LOG(ERR, CHANNEL_MONITOR, "Unable to remove channel '%s' "
739 "from epoll\n", chan_info->channel_path);
746 channel_monitor_init(void)
748 global_event_fd = epoll_create1(0);
749 if (global_event_fd == 0) {
750 RTE_LOG(ERR, CHANNEL_MONITOR,
751 "Error creating epoll context with error %s\n",
755 global_events_list = rte_malloc("epoll_events",
756 sizeof(*global_events_list)
757 * MAX_EVENTS, RTE_CACHE_LINE_SIZE);
758 if (global_events_list == NULL) {
759 RTE_LOG(ERR, CHANNEL_MONITOR, "Unable to rte_malloc for "
767 read_binary_packet(struct channel_info *chan_info)
769 struct channel_packet pkt;
771 int buffer_len = sizeof(pkt);
772 int n_bytes, err = 0;
774 while (buffer_len > 0) {
775 n_bytes = read(chan_info->fd,
777 if (n_bytes == buffer_len)
781 RTE_LOG(DEBUG, CHANNEL_MONITOR,
783 "channel '%s' read: %s\n",
784 chan_info->channel_path,
786 remove_channel(&chan_info);
789 buffer = (char *)buffer + n_bytes;
790 buffer_len -= n_bytes;
793 process_request(&pkt, chan_info);
798 read_json_packet(struct channel_info *chan_info)
800 struct channel_packet pkt;
805 /* read opening brace to closing brace */
810 n_bytes = read(chan_info->fd, &json_data[idx], 1);
813 if (json_data[idx] == '{')
815 if (json_data[idx] == '}')
817 if ((indent > 0) || (idx > 0))
821 if (idx >= MAX_JSON_STRING_LEN-1)
823 } while (indent > 0);
825 json_data[idx] = '\0';
827 if (strlen(json_data) == 0)
830 printf("got [%s]\n", json_data);
832 root = json_loads(json_data, 0, &error);
836 * Because our data is now in the json
837 * object, we can overwrite the pkt
838 * with a channel_packet struct, using
839 * parse_json_to_pkt()
841 ret = parse_json_to_pkt(root, &pkt);
844 RTE_LOG(ERR, CHANNEL_MONITOR,
845 "Error validating JSON profile data\n");
848 process_request(&pkt, chan_info);
850 RTE_LOG(ERR, CHANNEL_MONITOR,
851 "JSON error on line %d: %s\n",
852 error.line, error.text);
854 } while (n_bytes > 0);
859 run_channel_monitor(void)
864 n_events = epoll_wait(global_event_fd, global_events_list,
868 for (i = 0; i < n_events; i++) {
869 struct channel_info *chan_info = (struct channel_info *)
870 global_events_list[i].data.ptr;
871 if ((global_events_list[i].events & EPOLLERR) ||
872 (global_events_list[i].events & EPOLLHUP)) {
873 RTE_LOG(INFO, CHANNEL_MONITOR,
874 "Remote closed connection for "
876 chan_info->channel_path);
877 remove_channel(&chan_info);
880 if (global_events_list[i].events & EPOLLIN) {
882 switch (chan_info->type) {
883 case CHANNEL_TYPE_BINARY:
884 read_binary_packet(chan_info);
887 case CHANNEL_TYPE_JSON:
888 read_json_packet(chan_info);
896 rte_delay_us(time_period_ms*1000);
900 for (j = 0; j < MAX_CLIENTS; j++) {
901 if (policies[j].enabled == 1)
902 apply_policy(&policies[j]);