5 # * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
6 # * All rights reserved.
8 # * Redistribution and use in source and binary forms, with or without
9 # * modification, are permitted provided that the following conditions
12 # * * Redistributions of source code must retain the above copyright
13 # * notice, this list of conditions and the following disclaimer.
14 # * * Redistributions in binary form must reproduce the above copyright
15 # * notice, this list of conditions and the following disclaimer in
16 # * the documentation and/or other materials provided with the
18 # * * Neither the name of Intel Corporation nor the names of its
19 # * contributors may be used to endorse or promote products derived
20 # * from this software without specific prior written permission.
22 # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 # * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 # * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 # * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 # * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 # * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 # * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 # * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 # * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #####################################################################
36 # This script is designed to modify the call to the QEMU emulator
37 # to support userspace vhost when starting a guest machine through
38 # libvirt with vhost enabled. The steps to enable this are as follows
39 # and should be run as root:
41 # 1. Place this script in a libvirtd's binary search PATH ($PATH)
42 # A good location would be in the same directory that the QEMU
45 # 2. Ensure that the script has the same owner/group and file
46 # permissions as the QEMU binary
48 # 3. Update the VM xml file using "virsh edit VM.xml"
50 # 3.a) Set the VM to use the launch script
52 # Set the emulator path contained in the
53 # <emulator><emulator/> tags
55 # e.g replace <emulator>/usr/bin/qemu-kvm<emulator/>
56 # with <emulator>/usr/bin/qemu-wrap.py<emulator/>
58 # 3.b) Set the VM's device's to use vhost-net offload
60 # <interface type="network">
61 # <model type="virtio"/>
62 # <driver name="vhost"/>
65 # 4. Enable libvirt to access our userpace device file by adding it to
66 # controllers cgroup for libvirtd using the following steps
68 # 4.a) In /etc/libvirt/qemu.conf add/edit the following lines:
69 # 1) cgroup_controllers = [ ... "devices", ... ]
70 # 2) clear_emulator_capabilities = 0
73 # 5) cgroup_device_acl = [
74 # "/dev/null", "/dev/full", "/dev/zero",
75 # "/dev/random", "/dev/urandom",
76 # "/dev/ptmx", "/dev/kvm", "/dev/kqemu",
77 # "/dev/rtc", "/dev/hpet", "/dev/net/tun",
78 # "/dev/<devbase-name>",
82 # 4.b) Disable SELinux or set to permissive mode
84 # 4.c) Mount cgroup device controller
86 # "mount -t cgroup none /dev/cgroup -o devices"
88 # 4.d) Set hugetlbfs_mount variable - ( Optional )
89 # VMs using userspace vhost must use hugepage backed
90 # memory. This can be enabled in the libvirt XML
91 # config by adding a memory backing section to the
96 # This memory backing section should be added after the
97 # <memory> and <currentMemory> sections. This will add
98 # flags "-mem-prealloc -mem-path <path>" to the QEMU
99 # command line. The hugetlbfs_mount variable can be used
100 # to override the default <path> passed through by libvirt.
102 # if "-mem-prealloc" or "-mem-path <path>" are not passed
103 # through and a vhost device is detected then these options will
104 # be automatically added by this script. This script will detect
105 # the system hugetlbfs mount point to be used for <path>. The
106 # default <path> for this script can be overidden by the
107 # hugetlbfs_dir variable in the configuration section of this script.
110 # 4.e) Restart the libvirtd system process
111 # e.g. on Fedora "systemctl restart libvirtd.service"
114 # 4.f) Edit the Configuration Parameters section of this script
115 # to point to the correct emulator location and set any
118 # The script modifies the libvirtd Qemu call by modifying/adding
119 # options based on the configuration parameters below.
121 # emul_path and us_vhost_path must be set
122 # All other parameters are optional
123 #####################################################################
126 #############################################
127 # Configuration Parameters
128 #############################################
130 emul_path = "/usr/local/bin/qemu-system-x86_64"
132 #Path to userspace vhost device file
133 # This filename should match the --dev-basename parameters of
134 # the command used to launch the userspace vhost sample application e.g.
135 # if the sample app lauch command is:
136 # ./build/vhost-switch ..... --dev-basename usvhost
137 # then this variable should be set to:
138 # us_vhost_path = "/dev/usvhost"
139 us_vhost_path = "/dev/usvhost"
141 #List of additional user defined emulation options. These options will
142 #be added to all Qemu calls
145 #List of additional user defined emulation options for vhost only.
146 #These options will only be added to vhost enabled guests
147 emul_opts_user_vhost = []
149 #For all VHOST enabled VMs, the VM memory is preallocated from hugetlbfs
150 # Set this variable to one to enable this option for all VMs
153 #Instead of autodetecting, override the hugetlbfs directory by setting
157 #############################################
160 #############################################
161 # ****** Do Not Modify Below this Line ******
162 #############################################
164 import sys, os, subprocess
169 #List of open userspace vhost file descriptors
172 #additional virtio device flags when using userspace vhost
173 vhost_flags = [ "csum=off",
180 #String of the path to the Qemu process pid
181 qemu_pid = "/tmp/%d-qemu.pid" % os.getpid()
183 #############################################
184 # Signal haldler to kill Qemu subprocess
185 #############################################
186 def kill_qemu_process(signum, stack):
187 pidfile = open(qemu_pid, 'r')
188 pid = int(pidfile.read())
189 os.killpg(pid, signal.SIGTERM)
193 #############################################
194 # Find the system hugefile mount point.
196 # if multiple hugetlbfs mount points exist
197 # then the first one found will be used
198 #############################################
199 def find_huge_mount():
201 if (len(hugetlbfs_dir)):
206 if (os.access("/proc/mounts", os.F_OK)):
207 f = open("/proc/mounts", "r")
210 line_split = line.split(" ")
211 if line_split[2] == 'hugetlbfs':
212 huge_mount = line_split[1]
216 print "/proc/mounts not found"
220 if len(huge_mount) == 0:
221 print "Failed to find hugetlbfs mount point"
227 #############################################
228 # Get a userspace Vhost file descriptor
229 #############################################
232 if (os.access(us_vhost_path, os.F_OK)):
233 fd = os.open( us_vhost_path, os.O_RDWR)
235 print ("US-Vhost file %s not found" %us_vhost_path)
241 #############################################
242 # Check for vhostfd. if found then replace
243 # with our own vhost fd and append any vhost
245 #############################################
246 def modify_netdev_arg(arg):
252 netdev_opts = arg.split(",")
254 for opt in netdev_opts:
255 #check if vhost is used
256 if "vhost" == opt[:5]:
261 #if using vhost append vhost options
262 if vhost_in_use == 1:
263 #append vhost on option
264 new_opts.append('vhost=on')
265 #append vhostfd ption
266 new_fd = get_vhost_fd()
267 new_opts.append('vhostfd=' + str(new_fd))
268 fd_list.append(new_fd)
270 #concatenate all options
280 #############################################
282 #############################################
288 num_cmd_args = len(sys.argv)
294 #parse the parameters
295 while (num < num_cmd_args):
298 #Check netdev +1 parameter for vhostfd
300 num_vhost_devs = len(fd_list)
305 mod_arg = modify_netdev_arg(arg)
306 new_args.append(mod_arg)
308 #append vhost flags if this is a vhost device
309 # and -device is the next arg
310 # i.e -device -opt1,-opt2,...,-opt3,%vhost
311 if (num_vhost_devs < len(fd_list)):
317 new_arg = sys.argv[num]
318 for flag in vhost_flags:
319 new_arg = ''.join([new_arg,',',flag])
320 new_args.append(new_arg)
323 elif arg == '-mem-prealloc':
326 elif arg == '-mem-path':
335 #Set Qemu binary location
339 #Add prealloc mem options if using vhost and not already added
340 if ((len(fd_list) > 0) and (mem_prealloc_set == 0)):
341 emul_call += "-mem-prealloc "
343 #Add mempath mem options if using vhost and not already added
344 if ((len(fd_list) > 0) and (mem_path_set == 0)):
345 #Detect and add hugetlbfs mount point
346 mp = find_huge_mount()
347 mp = "".join(["-mem-path ", mp])
352 for opt in emul_opts_user:
356 #Add add user vhost only options
358 for opt in emul_opts_user_vhost:
362 #Add updated libvirt options
363 iter_args = iter(new_args)
364 #skip 1st arg i.e. call to this script
366 for arg in iter_args:
370 emul_call += "-pidfile %s " % qemu_pid
372 process = subprocess.Popen(emul_call, shell=True, preexec_fn=os.setsid)
374 for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT]:
375 signal.signal(sig, kill_qemu_process)
382 #Cleanup temporary files
383 if os.access(qemu_pid, os.F_OK):
386 if __name__ == "__main__":