5 # Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
8 # Redistribution and use in source and binary forms, with or without
9 # modification, are permitted provided that the following conditions
12 # * Redistributions of source code must retain the above copyright
13 # notice, this list of conditions and the following disclaimer.
14 # * Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in
16 # the documentation and/or other materials provided with the
18 # * Neither the name of Intel Corporation nor the names of its
19 # contributors may be used to endorse or promote products derived
20 # from this software without specific prior written permission.
22 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 # The main logic behind running autotests in parallel
36 import multiprocessing, subprocess, sys, pexpect, re, time, os, StringIO, csv
39 def wait_prompt(child):
42 result = child.expect(["RTE>>", pexpect.TIMEOUT, pexpect.EOF],
52 # each result tuple in results list consists of:
53 # result value (0 or -1)
56 # total test run time (double)
58 # test report (if not available, should be None)
60 # this function needs to be outside AutotestRunner class
61 # because otherwise Pool won't work (or rather it will require
62 # quite a bit of effort to make it work).
63 def run_test_group(cmdline, test_group):
66 start_time = time.time()
71 # prepare logging of init
72 startuplog = StringIO.StringIO()
74 print >>startuplog, "\n%s %s\n" % ("="*20, test_group["Prefix"])
75 print >>startuplog, "\ncmdline=%s" % cmdline
77 child = pexpect.spawn(cmdline, logfile=startuplog)
79 # wait for target to boot
80 if not wait_prompt(child):
83 results.append((-1, "Fail [No prompt]", "Start %s" % test_group["Prefix"],
84 time.time() - start_time, startuplog.getvalue(), None))
86 # mark all tests as failed
87 for test in test_group["Tests"]:
88 results.append((-1, "Fail [No prompt]", test["Name"],
89 time.time() - start_time, "", None))
94 results.append((-1, "Fail [Can't run]", "Start %s" % test_group["Prefix"],
95 time.time() - start_time, startuplog.getvalue(), None))
97 # mark all tests as failed
98 for t in test_group["Tests"]:
99 results.append((-1, "Fail [Can't run]", t["Name"],
100 time.time() - start_time, "", None))
104 # startup was successful
105 results.append((0, "Success", "Start %s" % test_group["Prefix"],
106 time.time() - start_time, startuplog.getvalue(), None))
108 # parse the binary for available test commands
109 binary = cmdline.split()[0]
110 symbols = subprocess.check_output(['nm', binary]).decode('utf-8')
111 avail_cmds = re.findall('test_register_(\w+)', symbols)
113 # run all tests in test group
114 for test in test_group["Tests"]:
116 # create log buffer for each test
117 # in multiprocessing environment, the logging would be
118 # interleaved and will create a mess, hence the buffering
119 logfile = StringIO.StringIO()
120 child.logfile = logfile
124 # make a note when the test started
125 start_time = time.time()
128 # print test name to log buffer
129 print >>logfile, "\n%s %s\n" % ("-"*20, test["Name"])
131 # run test function associated with the test
132 if test["Command"] in avail_cmds:
133 result = test["Func"](child, test["Command"])
135 result = (0, "Skipped [Not Available]")
137 # make a note when the test was finished
138 end_time = time.time()
140 # append test data to the result tuple
141 result += (test["Name"], end_time - start_time,
144 # call report function, if any defined, and supply it with
145 # target and complete log for test run
147 report = test["Report"](self.target, log)
149 # append report to results tuple
155 # make a note when the test crashed
156 end_time = time.time()
158 # mark test as failed
159 result = (-1, "Fail [Crash]", test["Name"],
160 end_time - start_time, logfile.getvalue(), None)
162 # append the results to the results list
163 results.append(result)
165 # regardless of whether test has crashed, try quitting it
167 child.sendline("quit")
169 # if the test crashed, just do nothing instead
174 # return test results
181 # class representing an instance of autotests run
182 class AutotestRunner:
184 parallel_test_groups = []
185 non_parallel_test_groups = []
197 def __init__(self, cmdline, target, blacklist, whitelist):
198 self.cmdline = cmdline
200 self.blacklist = blacklist
201 self.whitelist = whitelist
204 logfile = "%s.log" % target
205 csvfile = "%s.csv" % target
207 self.logfile = open(logfile, "w")
208 csvfile = open(csvfile, "w")
209 self.csvwriter = csv.writer(csvfile)
211 # prepare results table
212 self.csvwriter.writerow(["test_name","test_result","result_str"])
216 # set up cmdline string
217 def __get_cmdline(self, test):
218 cmdline = self.cmdline
220 # append memory limitations for each test
221 # otherwise tests won't run in parallel
222 if not "i686" in self.target:
223 cmdline += " --socket-mem=%s"% test["Memory"]
225 # affinitize startup so that tests don't fail on i686
226 cmdline = "taskset 1 " + cmdline
227 cmdline += " -m " + str(sum(map(int,test["Memory"].split(","))))
229 # set group prefix for autotest group
230 # otherwise they won't run in parallel
231 cmdline += " --file-prefix=%s"% test["Prefix"]
237 def add_parallel_test_group(self,test_group):
238 self.parallel_test_groups.append(test_group)
240 def add_non_parallel_test_group(self,test_group):
241 self.non_parallel_test_groups.append(test_group)
244 def __process_results(self, results):
245 # this iterates over individual test results
246 for i, result in enumerate(results):
248 # increase total number of tests that were run
249 # do not include "start" test
253 # unpack result tuple
254 test_result, result_str, test_name, \
255 test_time, log, report = result
258 cur_time = time.time()
259 total_time = int(cur_time - self.start)
261 # print results, test run time and total time since start
262 print ("%s:" % test_name).ljust(30),
263 print result_str.ljust(29),
264 print "[%02dm %02ds]" % (test_time / 60, test_time % 60),
266 # don't print out total time every line, it's the same anyway
267 if i == len(results) - 1:
268 print "[%02dm %02ds]" % (total_time / 60, total_time % 60)
272 # if test failed and it wasn't a "start" test
273 if test_result < 0 and not i == 0:
277 self.log_buffers.append(log)
279 # create report if it exists
282 f = open("%s_%s_report.rst" % (self.target,test_name), "w")
284 print "Report for %s could not be created!" % test_name
289 # write test result to CSV file
291 self.csvwriter.writerow([test_name, test_result, result_str])
296 # this function iterates over test groups and removes each
297 # test that is not in whitelist/blacklist
298 def __filter_groups(self, test_groups):
299 groups_to_remove = []
301 # filter out tests from parallel test groups
302 for i, test_group in enumerate(test_groups):
304 # iterate over a copy so that we could safely delete individual tests
305 for test in test_group["Tests"][:]:
306 test_id = test["Command"]
308 # dump tests are specified in full e.g. "Dump_mempool"
309 if "_autotest" in test_id:
310 test_id = test_id[:-len("_autotest")]
312 # filter out blacklisted/whitelisted tests
313 if self.blacklist and test_id in self.blacklist:
314 test_group["Tests"].remove(test)
316 if self.whitelist and test_id not in self.whitelist:
317 test_group["Tests"].remove(test)
320 # modify or remove original group
321 if len(test_group["Tests"]) > 0:
322 test_groups[i] = test_group
324 # remember which groups should be deleted
325 # put the numbers backwards so that we start
326 # deleting from the end, not from the beginning
327 groups_to_remove.insert(0, i)
329 # remove test groups that need to be removed
330 for i in groups_to_remove:
337 # iterate over test groups and run tests associated with them
338 def run_all_tests(self):
340 self.parallel_test_groups = \
341 self.__filter_groups(self.parallel_test_groups)
342 self.non_parallel_test_groups = \
343 self.__filter_groups(self.non_parallel_test_groups)
345 # create a pool of worker threads
346 pool = multiprocessing.Pool(processes=1)
350 # whatever happens, try to save as much logs as possible
353 # create table header
355 print "Test name".ljust(30),
356 print "Test result".ljust(29),
357 print "Test".center(9),
358 print "Total".center(9)
361 # make a note of tests start time
362 self.start = time.time()
364 # assign worker threads to run test groups
365 for test_group in self.parallel_test_groups:
366 result = pool.apply_async(run_test_group,
367 [self.__get_cmdline(test_group), test_group])
368 results.append(result)
370 # iterate while we have group execution results to get
371 while len(results) > 0:
373 # iterate over a copy to be able to safely delete results
374 # this iterates over a list of group results
375 for group_result in results[:]:
377 # if the thread hasn't finished yet, continue
378 if not group_result.ready():
381 res = group_result.get()
383 self.__process_results(res)
385 # remove result from results list once we're done with it
386 results.remove(group_result)
388 # run non_parallel tests. they are run one by one, synchronously
389 for test_group in self.non_parallel_test_groups:
390 group_result = run_test_group(self.__get_cmdline(test_group), test_group)
392 self.__process_results(group_result)
395 cur_time = time.time()
396 total_time = int(cur_time - self.start)
400 print "Total run time: %02dm %02ds" % (total_time / 60, total_time % 60)
402 print "Number of failed tests: %s" % str(self.fails)
404 # write summary to logfile
405 self.logfile.write("Summary\n")
406 self.logfile.write("Target: ".ljust(15) + "%s\n" % self.target)
407 self.logfile.write("Tests: ".ljust(15) + "%i\n" % self.n_tests)
408 self.logfile.write("Failed tests: ".ljust(15) + "%i\n" % self.fails)
410 print "Exception occured"
414 # drop logs from all executions to a logfile
415 for buf in self.log_buffers:
416 self.logfile.write(buf.replace("\r",""))