5 # Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
8 # Redistribution and use in source and binary forms, with or without
9 # modification, are permitted provided that the following conditions
12 # * Redistributions of source code must retain the above copyright
13 # notice, this list of conditions and the following disclaimer.
14 # * Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in
16 # the documentation and/or other materials provided with the
18 # * Neither the name of Intel Corporation nor the names of its
19 # contributors may be used to endorse or promote products derived
20 # from this software without specific prior written permission.
22 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 # Script that uses qemu controlled by python-pexpect to check that
36 # all autotests are working in the baremetal environment.
38 import sys, pexpect, time, os, re
40 directory = sys.argv[2]
42 log_file = "%s.txt"%(target)
44 if "baremetal" in target:
45 cmdline = "qemu-system-x86_64 -cdrom %s.iso -boot d "%(sys.argv[1])
46 cmdline += "-m 2000 -smp 4 -nographic -net nic,model=e1000"
47 platform = "QEMU x86_64"
49 cmdline = "%s -c f -n 4"%(sys.argv[1])
51 platform = open("/root/rte_platform_model.txt").read()
57 report_hdr=""".. <COPYRIGHT_TAG>
66 def __init__(self, title, function, command=None, timeout=10, genreport=None):
68 self.function = function
69 self.command = command
70 self.timeout = timeout
71 self.genreport = genreport
74 """This class contains all methods needed to launch several
75 automatic tests, archive test results, log, and generate a nice
76 test report in restructured text"""
86 def __init__(self, pexpectchild, filename, mode):
87 "Init the Autotest class"
88 self.mainlog = file(filename, mode)
89 self.child = pexpectchild
90 pexpectchild.logfile = self
91 def register(self, filename, title, subtest_list):
92 "Register a test with a list of subtests"
94 test["filename"] = filename
96 test["subtest_list"] = subtest_list
97 self.test_list.append(test)
100 "start the tests, and fill the internal report_list field"
101 for t in self.test_list:
103 report["date"] = time.asctime()
104 report["title"] = t["title"]
105 report["filename"] = t["filename"]
106 report["subreport_list"] = []
108 report["success"] = 0
109 report["subreport_list"] = []
110 for st in t["subtest_list"]:
111 if test_whitelist is not None and st.title not in test_whitelist:
113 if test_blacklist is not None and st.title in test_blacklist:
117 subreport["title"] = st.title
118 subreport["func"] = st.function
119 subreport["command"] = st.command
120 subreport["timeout"] = st.timeout
121 subreport["genreport"] = st.genreport
124 print "%s (%s): "%(subreport["title"], subreport["command"]),
127 res = subreport["func"](self.child,
128 command = subreport["command"],
129 timeout = subreport["timeout"])
130 t = int(time.time() - start)
132 subreport["time"] = "%dmn%d"%(t/60, t%60)
133 subreport["result"] = res[0] # 0 or -1
134 subreport["result_str"] = res[1] # cause of fail
135 subreport["logs"] = self.reportbuf
136 print "%s [%s]"%(subreport["result_str"], subreport["time"])
137 if subreport["result"] == 0:
138 report["success"] += 1
141 report["subreport_list"].append(subreport)
142 self.report_list.append(report)
144 def gen_report(self):
145 for report in self.report_list:
146 # main report header and stats
148 reportlog = file(report["filename"], "w")
149 reportlog.write(report_hdr)
150 reportlog.write(report["title"] + "\n")
151 reportlog.write(re.sub(".", "=", report["title"]) + "\n\n")
152 reportlog.write("Autogenerated test report:\n\n" )
153 reportlog.write("- date: **%s**\n"%(report["date"]))
154 reportlog.write("- target: **%s**\n"%(target))
155 reportlog.write("- success: **%d**\n"%(report["success"]))
156 reportlog.write("- fails: **%d**\n"%(report["fails"]))
157 reportlog.write("- platform: **%s**\n\n"%(platform))
160 reportlog.write(".. csv-table:: Test results summary\n")
161 reportlog.write(' :header: "Name", "Result"\n\n')
162 for subreport in report["subreport_list"]:
163 if subreport["result"] == 0:
167 reportlog.write(' "%s", "%s"\n'%(subreport["title"], res_str))
168 reportlog.write('\n')
171 for subreport in report["subreport_list"]:
173 reportlog.write(subreport["title"] + "\n")
174 reportlog.write(re.sub(".", "-", subreport["title"]) + "\n\n")
176 reportlog.write("::\n \n ")
177 s = subreport["logs"].replace("\n", "\n ")
180 reportlog.write("\n\n")
181 reportlog.write("**" + subreport["result_str"] + "**\n\n")
183 if subreport["genreport"] != None:
184 s = subreport["genreport"]()
189 # displayed on console
191 print "-------------------------"
193 if report["fails"] == 0:
196 print "%s test(s) failed"%(report["fails"])
198 # file API, to store logs from pexpect
199 def write(self, buf):
201 s = s.replace("\r", "")
202 self.mainlog.write(s)
210 # Try to match prompt: return 0 on success, else return -1
211 def wait_prompt(child):
213 index = child.expect(["RTE>>", pexpect.TIMEOUT], timeout = 1)
217 print "Cannot find prompt"
220 # Try to match prompt after boot: return 0 on success, else return -1
221 def wait_boot(child):
222 index = child.expect(["RTE>>", pexpect.TIMEOUT],
226 if (wait_prompt(child) == -1):
227 print "Target did not boot, failed"
233 if wait_boot(child) != 0:
234 return -1, "Cannot find prompt"
235 child.sendline("quit")
238 # Default function to launch an autotest that does not need to
239 # interact with the user. Basically, this function calls the autotest
240 # function through command line interface, then check that it displays
241 # "Test OK" or "Test Failed".
242 def default_autotest(child, command, timeout=10):
243 if wait_prompt(child) != 0:
244 return -1, "Failed: cannot find prompt"
245 child.sendline(command)
246 index = child.expect(["Test OK", "Test Failed",
247 pexpect.TIMEOUT], timeout = timeout)
251 return -1, "Failed [Timeout]"
255 def boot_autotest(child, **kargs):
256 if wait_boot(child) != 0:
257 return -1, "Cannot find prompt"
260 # Test memory dump. We need to check that at least one memory zone is
262 def memory_autotest(child, command, **kargs):
263 if wait_prompt(child) != 0:
264 return -1, "Failed: cannot find prompt"
265 child.sendline(command)
266 regexp = "phys:0x[0-9a-f]*, len:0x([0-9a-f]*), virt:0x[0-9a-f]*, socket_id:[0-9]*"
267 index = child.expect([regexp, pexpect.TIMEOUT], timeout = 180)
269 return -1, "Failed: timeout"
270 size = int(child.match.groups()[0], 16)
272 return -1, "Failed: bad size"
273 index = child.expect(["Test OK", "Test Failed",
274 pexpect.TIMEOUT], timeout = 10)
276 return -1, "Failed: C code returned an error"
278 return -1, "Failed: timeout"
281 # Test some libc functions including scanf. This requires a
282 # interaction with the user (simulated in expect), so we cannot use
283 # default_autotest() here.
284 def string_autotest(child, command, **kargs):
285 if wait_prompt(child) != 0:
286 return -1, "Failed: cannot find prompt"
287 child.sendline(command)
288 index = child.expect(["Now, test scanf, enter this number",
289 pexpect.TIMEOUT], timeout = 10)
291 return -1, "Failed: timeout"
292 child.sendline("123456")
293 index = child.expect(["number=123456", pexpect.TIMEOUT], timeout = 10)
295 return -1, "Failed: timeout (2)"
296 index = child.expect(["Test OK", "Test Failed",
297 pexpect.TIMEOUT], timeout = 10)
299 return -1, "Failed: C code returned an error"
302 # Test spinlock. This requires to check the order of displayed lines:
303 # we cannot use default_autotest() here.
304 def spinlock_autotest(child, command, **kargs):
307 if wait_prompt(child) != 0:
308 return -1, "Failed: cannot find prompt"
309 child.sendline(command)
311 index = child.expect(["Test OK",
313 "Hello from core ([0-9]*) !",
314 "Hello from within recursive locks from ([0-9]*) !",
315 pexpect.TIMEOUT], timeout = 20)
320 # message, check ordering
322 if int(child.match.groups()[0]) < i:
323 return -1, "Failed: bad order"
324 i = int(child.match.groups()[0])
326 if int(child.match.groups()[0]) < ir:
327 return -1, "Failed: bad order"
328 ir = int(child.match.groups()[0])
332 return -1, "Failed: timeout or error"
337 # Test rwlock. This requires to check the order of displayed lines:
338 # we cannot use default_autotest() here.
339 def rwlock_autotest(child, command, **kargs):
341 if wait_prompt(child) != 0:
342 return -1, "Failed: cannot find prompt"
343 child.sendline(command)
345 index = child.expect(["Test OK",
347 "Hello from core ([0-9]*) !",
348 "Global write lock taken on master core ([0-9]*)",
349 pexpect.TIMEOUT], timeout = 10)
353 return -1, "Failed: a message is missing"
356 # message, check ordering
358 if int(child.match.groups()[0]) < i:
359 return -1, "Failed: bad order"
360 i = int(child.match.groups()[0])
362 # must be the last message, check ordering
368 return -1, "Failed: timeout or error"
372 # Test logs. This requires to check the order of displayed lines:
373 # we cannot use default_autotest() here.
374 def logs_autotest(child, command, **kargs):
376 if wait_prompt(child) != 0:
377 return -1, "Failed: cannot find prompt"
378 child.sendline(command)
381 "TESTAPP1: this is a debug level message",
382 "TESTAPP1: this is a info level message",
383 "TESTAPP1: this is a warning level message",
384 "TESTAPP2: this is a info level message",
385 "TESTAPP2: this is a warning level message",
386 "TESTAPP1: this is a debug level message",
387 "TESTAPP1: this is a debug level message",
388 "TESTAPP1: this is a info level message",
389 "TESTAPP1: this is a warning level message",
390 "TESTAPP2: this is a info level message",
391 "TESTAPP2: this is a warning level message",
392 "TESTAPP1: this is a debug level message",
395 for log_msg in log_list:
396 index = child.expect([log_msg,
399 pexpect.TIMEOUT], timeout = 10)
403 return -1, "Failed: timeout or error"
405 index = child.expect(["Test OK",
407 pexpect.TIMEOUT], timeout = 10)
411 # Test timers. This requires to check the order of displayed lines:
412 # we cannot use default_autotest() here.
413 def timer_autotest(child, command, **kargs):
415 if wait_prompt(child) != 0:
416 return -1, "Failed: cannot find prompt"
417 child.sendline(command)
419 index = child.expect(["Start timer stress tests \(30 seconds\)",
421 pexpect.TIMEOUT], timeout = 10)
425 return -1, "Failed: timeout or error"
427 index = child.expect(["Start timer basic tests \(30 seconds\)",
429 pexpect.TIMEOUT], timeout = 40)
433 return -1, "Failed: timeout or error (2)"
435 prev_lcore_timer1 = -1
443 index = child.expect(["TESTTIMER: ([0-9]*): callback id=([0-9]*) count=([0-9]*) on core ([0-9]*)",
446 pexpect.TIMEOUT], timeout = 10)
452 return -1, "Failed: timeout or error (3)"
455 t = int(child.match.groups()[0])
456 id = int(child.match.groups()[1])
457 cnt = int(child.match.groups()[2])
458 lcore = int(child.match.groups()[3])
460 return -1, "Failed: cannot parse output"
462 # timer0 always expires on the same core when cnt < 20
466 elif lcore != lcore_tim0 and cnt < 20:
467 return -1, "Failed: lcore != lcore_tim0 (%d, %d)"%(lcore, lcore_tim0)
469 return -1, "Failed: tim0 cnt > 21"
471 # timer1 each time expires on a different core
473 if lcore == lcore_tim1:
474 return -1, "Failed: lcore == lcore_tim1 (%d, %d)"%(lcore, lcore_tim1)
477 return -1, "Failed: tim1 cnt > 30"
479 # timer0 always expires on the same core
483 elif lcore != lcore_tim2:
484 return -1, "Failed: lcore != lcore_tim2 (%d, %d)"%(lcore, lcore_tim2)
486 return -1, "Failed: tim2 cnt > 30"
488 # timer0 always expires on the same core
492 elif lcore != lcore_tim3:
493 return -1, "Failed: lcore_tim3 changed (%d -> %d)"%(lcore, lcore_tim3)
495 return -1, "Failed: tim3 cnt > 30"
497 # must be 2 different cores
498 if lcore_tim0 == lcore_tim3:
499 return -1, "Failed: lcore_tim0 (%d) == lcore_tim3 (%d)"%(lcore_tim0, lcore_tim3)
504 def ring_autotest(child, command, timeout=10):
505 if wait_prompt(child) != 0:
506 return -1, "Failed: cannot find prompt"
507 child.sendline(command)
508 index = child.expect(["Test OK", "Test Failed",
509 pexpect.TIMEOUT], timeout = timeout)
513 child.sendline("set_watermark test 100")
514 child.sendline("set_quota test 16")
515 child.sendline("dump_ring test")
516 index = child.expect([" watermark=100",
517 pexpect.TIMEOUT], timeout = 1)
519 return -1, "Failed: bad watermark"
521 index = child.expect([" bulk_default=16",
522 pexpect.TIMEOUT], timeout = 1)
524 return -1, "Failed: bad quota"
528 def ring_genreport():
529 s = "Performance curves\n"
530 s += "------------------\n\n"
531 sdk = os.getenv("RTE_SDK")
532 script = os.path.join(sdk, "app/test/graph_ring.py")
533 title ='"Autotest %s %s"'%(target, time.asctime())
534 filename = target + ".txt"
535 os.system("/usr/bin/python %s %s %s"%(script, filename, title))
536 for f in os.listdir("."):
537 if not f.startswith("ring"):
539 if not f.endswith(".svg"):
541 # skip single producer/consumer
547 s += ".. figure:: ../../images/autotests/%s/%s\n"%(target, f)
548 s += " :width: 50%\n\n"
552 def mempool_genreport():
553 s = "Performance curves\n"
554 s += "------------------\n\n"
555 sdk = os.getenv("RTE_SDK")
556 script = os.path.join(sdk, "app/test/graph_mempool.py")
557 title ='"Autotest %s %s"'%(target, time.asctime())
558 filename = target + ".txt"
559 os.system("/usr/bin/python %s %s %s"%(script, filename, title))
560 for f in os.listdir("."):
561 if not f.startswith("mempool"):
563 if not f.endswith(".svg"):
565 # skip when n_keep = 128
569 s += ".. figure:: ../../images/autotests/%s/%s\n"%(target, f)
570 s += " :width: 50%\n\n"
578 if len(sys.argv) > 4:
579 testlist=sys.argv[4].split(',')
580 if testlist[0].startswith('-'):
581 testlist[0]=testlist[0].lstrip('-')
582 test_blacklist=testlist
584 test_whitelist=testlist
586 child = pexpect.spawn(cmdline)
587 autotest = AutoTest(child, log_file,'w')
589 # timeout for memcpy and hash test
590 if "baremetal" in target:
595 autotest.register("eal_report.rst", "EAL-%s"%(target),
596 [ SubTest("Boot", boot_autotest, "boot_autotest"),
597 SubTest("EAL Flags", default_autotest, "eal_flags_autotest"),
598 SubTest("Version", default_autotest, "version_autotest"),
599 SubTest("PCI", default_autotest, "pci_autotest"),
600 SubTest("Memory", memory_autotest, "memory_autotest"),
601 SubTest("Lcore launch", default_autotest, "per_lcore_autotest"),
602 SubTest("Spinlock", spinlock_autotest, "spinlock_autotest"),
603 SubTest("Rwlock", rwlock_autotest, "rwlock_autotest"),
604 SubTest("Atomic", default_autotest, "atomic_autotest"),
605 SubTest("Byte order", default_autotest, "byteorder_autotest"),
606 SubTest("Prefetch", default_autotest, "prefetch_autotest"),
607 SubTest("Debug", default_autotest, "debug_autotest"),
608 SubTest("Cycles", default_autotest, "cycles_autotest"),
609 SubTest("Logs", logs_autotest, "logs_autotest"),
610 SubTest("Memzone", default_autotest, "memzone_autotest"),
611 SubTest("Cpu flags", default_autotest, "cpuflags_autotest"),
612 SubTest("Memcpy", default_autotest, "memcpy_autotest", timeout),
613 SubTest("String Functions", default_autotest, "string_autotest"),
614 SubTest("Alarm", default_autotest, "alarm_autotest", 30),
615 SubTest("Interrupt", default_autotest, "interrupt_autotest"),
618 autotest.register("ring_report.rst", "Ring-%s"%(target),
619 [ SubTest("Ring", ring_autotest, "ring_autotest", 30*60,
623 if "baremetal" in target:
628 autotest.register("mempool_report.rst", "Mempool-%s"%(target),
629 [ SubTest("Mempool", default_autotest, "mempool_autotest",
630 timeout, mempool_genreport)
632 autotest.register("mbuf_report.rst", "Mbuf-%s"%(target),
633 [ SubTest("Mbuf", default_autotest, "mbuf_autotest", timeout=120)
635 autotest.register("timer_report.rst", "Timer-%s"%(target),
636 [ SubTest("Timer", timer_autotest, "timer_autotest")
638 autotest.register("malloc_report.rst", "Malloc-%s"%(target),
639 [ SubTest("Malloc", default_autotest, "malloc_autotest")
642 # only do the hash autotest if supported by the platform
643 if not (platform.startswith("Intel(R) Core(TM)2 Quad CPU") or
644 platform.startswith("QEMU")):
645 autotest.register("hash_report.rst", "Hash-%s"%(target),
646 [ SubTest("Hash", default_autotest, "hash_autotest", timeout)
649 autotest.register("lpm_report.rst", "LPM-%s"%(target),
650 [ SubTest("Lpm", default_autotest, "lpm_autotest", timeout)
652 autotest.register("eal2_report.rst", "EAL2-%s"%(target),
653 [ SubTest("TailQ", default_autotest, "tailq_autotest"),
654 SubTest("Errno", default_autotest, "errno_autotest"),
655 SubTest("Multiprocess", default_autotest, "multiprocess_autotest")
659 autotest.gen_report()