5 # Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
8 # Redistribution and use in source and binary forms, with or without
9 # modification, are permitted provided that the following conditions
12 # * Redistributions of source code must retain the above copyright
13 # notice, this list of conditions and the following disclaimer.
14 # * Redistributions in binary form must reproduce the above copyright
15 # notice, this list of conditions and the following disclaimer in
16 # the documentation and/or other materials provided with the
18 # * Neither the name of Intel Corporation nor the names of its
19 # contributors may be used to endorse or promote products derived
20 # from this software without specific prior written permission.
22 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 # version: DPDK.L.1.2.3-3
36 # Script that uses qemu controlled by python-pexpect to check that
37 # all autotests are working in the baremetal environment.
39 import sys, pexpect, time, os, re
41 directory = sys.argv[2]
43 log_file = "%s.txt"%(target)
45 if "baremetal" in target:
46 cmdline = "qemu-system-x86_64 -cdrom %s.iso -boot d "%(sys.argv[1])
47 cmdline += "-m 2000 -smp 4 -nographic -net nic,model=e1000"
48 platform = "QEMU x86_64"
50 cmdline = "%s -c f -n 4"%(sys.argv[1])
52 platform = open("/root/rte_platform_model.txt").read()
58 report_hdr=""".. <COPYRIGHT_TAG>
67 def __init__(self, title, function, command=None, timeout=10, genreport=None):
69 self.function = function
70 self.command = command
71 self.timeout = timeout
72 self.genreport = genreport
75 """This class contains all methods needed to launch several
76 automatic tests, archive test results, log, and generate a nice
77 test report in restructured text"""
87 def __init__(self, pexpectchild, filename, mode):
88 "Init the Autotest class"
89 self.mainlog = file(filename, mode)
90 self.child = pexpectchild
91 pexpectchild.logfile = self
92 def register(self, filename, title, subtest_list):
93 "Register a test with a list of subtests"
95 test["filename"] = filename
97 test["subtest_list"] = subtest_list
98 self.test_list.append(test)
101 "start the tests, and fill the internal report_list field"
102 for t in self.test_list:
104 report["date"] = time.asctime()
105 report["title"] = t["title"]
106 report["filename"] = t["filename"]
107 report["subreport_list"] = []
109 report["success"] = 0
110 report["subreport_list"] = []
111 for st in t["subtest_list"]:
112 if test_whitelist is not None and st.title not in test_whitelist:
114 if test_blacklist is not None and st.title in test_blacklist:
118 subreport["title"] = st.title
119 subreport["func"] = st.function
120 subreport["command"] = st.command
121 subreport["timeout"] = st.timeout
122 subreport["genreport"] = st.genreport
125 print "%s (%s): "%(subreport["title"], subreport["command"]),
128 res = subreport["func"](self.child,
129 command = subreport["command"],
130 timeout = subreport["timeout"])
131 t = int(time.time() - start)
133 subreport["time"] = "%dmn%d"%(t/60, t%60)
134 subreport["result"] = res[0] # 0 or -1
135 subreport["result_str"] = res[1] # cause of fail
136 subreport["logs"] = self.reportbuf
137 print "%s [%s]"%(subreport["result_str"], subreport["time"])
138 if subreport["result"] == 0:
139 report["success"] += 1
142 report["subreport_list"].append(subreport)
143 self.report_list.append(report)
145 def gen_report(self):
146 for report in self.report_list:
147 # main report header and stats
149 reportlog = file(report["filename"], "w")
150 reportlog.write(report_hdr)
151 reportlog.write(report["title"] + "\n")
152 reportlog.write(re.sub(".", "=", report["title"]) + "\n\n")
153 reportlog.write("Autogenerated test report:\n\n" )
154 reportlog.write("- date: **%s**\n"%(report["date"]))
155 reportlog.write("- target: **%s**\n"%(target))
156 reportlog.write("- success: **%d**\n"%(report["success"]))
157 reportlog.write("- fails: **%d**\n"%(report["fails"]))
158 reportlog.write("- platform: **%s**\n\n"%(platform))
161 reportlog.write(".. csv-table:: Test results summary\n")
162 reportlog.write(' :header: "Name", "Result"\n\n')
163 for subreport in report["subreport_list"]:
164 if subreport["result"] == 0:
168 reportlog.write(' "%s", "%s"\n'%(subreport["title"], res_str))
169 reportlog.write('\n')
172 for subreport in report["subreport_list"]:
174 reportlog.write(subreport["title"] + "\n")
175 reportlog.write(re.sub(".", "-", subreport["title"]) + "\n\n")
177 reportlog.write("::\n \n ")
178 s = subreport["logs"].replace("\n", "\n ")
181 reportlog.write("\n\n")
182 reportlog.write("**" + subreport["result_str"] + "**\n\n")
184 if subreport["genreport"] != None:
185 s = subreport["genreport"]()
190 # displayed on console
192 print "-------------------------"
194 if report["fails"] == 0:
197 print "%s test(s) failed"%(report["fails"])
199 # file API, to store logs from pexpect
200 def write(self, buf):
202 s = s.replace("\r", "")
203 self.mainlog.write(s)
211 # Try to match prompt: return 0 on success, else return -1
212 def wait_prompt(child):
214 index = child.expect(["RTE>>", pexpect.TIMEOUT], timeout = 1)
218 print "Cannot find prompt"
221 # Try to match prompt after boot: return 0 on success, else return -1
222 def wait_boot(child):
223 index = child.expect(["RTE>>", pexpect.TIMEOUT],
227 if (wait_prompt(child) == -1):
228 print "Target did not boot, failed"
234 if wait_boot(child) != 0:
235 return -1, "Cannot find prompt"
236 child.sendline("quit")
239 # Default function to launch an autotest that does not need to
240 # interact with the user. Basically, this function calls the autotest
241 # function through command line interface, then check that it displays
242 # "Test OK" or "Test Failed".
243 def default_autotest(child, command, timeout=10):
244 if wait_prompt(child) != 0:
245 return -1, "Failed: cannot find prompt"
246 child.sendline(command)
247 index = child.expect(["Test OK", "Test Failed",
248 pexpect.TIMEOUT], timeout = timeout)
252 return -1, "Failed [Timeout]"
256 def boot_autotest(child, **kargs):
257 if wait_boot(child) != 0:
258 return -1, "Cannot find prompt"
261 # Test memory dump. We need to check that at least one memory zone is
263 def memory_autotest(child, command, **kargs):
264 if wait_prompt(child) != 0:
265 return -1, "Failed: cannot find prompt"
266 child.sendline(command)
267 regexp = "phys:0x[0-9a-f]*, len:0x([0-9a-f]*), virt:0x[0-9a-f]*, socket_id:[0-9]*"
268 index = child.expect([regexp, pexpect.TIMEOUT], timeout = 180)
270 return -1, "Failed: timeout"
271 size = int(child.match.groups()[0], 16)
273 return -1, "Failed: bad size"
274 index = child.expect(["Test OK", "Test Failed",
275 pexpect.TIMEOUT], timeout = 10)
277 return -1, "Failed: C code returned an error"
279 return -1, "Failed: timeout"
282 # Test some libc functions including scanf. This requires a
283 # interaction with the user (simulated in expect), so we cannot use
284 # default_autotest() here.
285 def string_autotest(child, command, **kargs):
286 if wait_prompt(child) != 0:
287 return -1, "Failed: cannot find prompt"
288 child.sendline(command)
289 index = child.expect(["Now, test scanf, enter this number",
290 pexpect.TIMEOUT], timeout = 10)
292 return -1, "Failed: timeout"
293 child.sendline("123456")
294 index = child.expect(["number=123456", pexpect.TIMEOUT], timeout = 10)
296 return -1, "Failed: timeout (2)"
297 index = child.expect(["Test OK", "Test Failed",
298 pexpect.TIMEOUT], timeout = 10)
300 return -1, "Failed: C code returned an error"
303 # Test spinlock. This requires to check the order of displayed lines:
304 # we cannot use default_autotest() here.
305 def spinlock_autotest(child, command, **kargs):
308 if wait_prompt(child) != 0:
309 return -1, "Failed: cannot find prompt"
310 child.sendline(command)
312 index = child.expect(["Test OK",
314 "Hello from core ([0-9]*) !",
315 "Hello from within recursive locks from ([0-9]*) !",
316 pexpect.TIMEOUT], timeout = 20)
321 # message, check ordering
323 if int(child.match.groups()[0]) < i:
324 return -1, "Failed: bad order"
325 i = int(child.match.groups()[0])
327 if int(child.match.groups()[0]) < ir:
328 return -1, "Failed: bad order"
329 ir = int(child.match.groups()[0])
333 return -1, "Failed: timeout or error"
338 # Test rwlock. This requires to check the order of displayed lines:
339 # we cannot use default_autotest() here.
340 def rwlock_autotest(child, command, **kargs):
342 if wait_prompt(child) != 0:
343 return -1, "Failed: cannot find prompt"
344 child.sendline(command)
346 index = child.expect(["Test OK",
348 "Hello from core ([0-9]*) !",
349 "Global write lock taken on master core ([0-9]*)",
350 pexpect.TIMEOUT], timeout = 10)
354 return -1, "Failed: a message is missing"
357 # message, check ordering
359 if int(child.match.groups()[0]) < i:
360 return -1, "Failed: bad order"
361 i = int(child.match.groups()[0])
363 # must be the last message, check ordering
369 return -1, "Failed: timeout or error"
373 # Test logs. This requires to check the order of displayed lines:
374 # we cannot use default_autotest() here.
375 def logs_autotest(child, command, **kargs):
377 if wait_prompt(child) != 0:
378 return -1, "Failed: cannot find prompt"
379 child.sendline(command)
382 "TESTAPP1: this is a debug level message",
383 "TESTAPP1: this is a info level message",
384 "TESTAPP1: this is a warning level message",
385 "TESTAPP2: this is a info level message",
386 "TESTAPP2: this is a warning level message",
387 "TESTAPP1: this is a debug level message",
388 "TESTAPP1: this is a debug level message",
389 "TESTAPP1: this is a info level message",
390 "TESTAPP1: this is a warning level message",
391 "TESTAPP2: this is a info level message",
392 "TESTAPP2: this is a warning level message",
393 "TESTAPP1: this is a debug level message",
396 for log_msg in log_list:
397 index = child.expect([log_msg,
400 pexpect.TIMEOUT], timeout = 10)
404 return -1, "Failed: timeout or error"
406 index = child.expect(["Test OK",
408 pexpect.TIMEOUT], timeout = 10)
412 # Test timers. This requires to check the order of displayed lines:
413 # we cannot use default_autotest() here.
414 def timer_autotest(child, command, **kargs):
416 if wait_prompt(child) != 0:
417 return -1, "Failed: cannot find prompt"
418 child.sendline(command)
420 index = child.expect(["Start timer stress tests \(30 seconds\)",
422 pexpect.TIMEOUT], timeout = 10)
426 return -1, "Failed: timeout or error"
428 index = child.expect(["Start timer basic tests \(30 seconds\)",
430 pexpect.TIMEOUT], timeout = 40)
434 return -1, "Failed: timeout or error (2)"
436 prev_lcore_timer1 = -1
444 index = child.expect(["TESTTIMER: ([0-9]*): callback id=([0-9]*) count=([0-9]*) on core ([0-9]*)",
447 pexpect.TIMEOUT], timeout = 10)
453 return -1, "Failed: timeout or error (3)"
456 t = int(child.match.groups()[0])
457 id = int(child.match.groups()[1])
458 cnt = int(child.match.groups()[2])
459 lcore = int(child.match.groups()[3])
461 return -1, "Failed: cannot parse output"
463 # timer0 always expires on the same core when cnt < 20
467 elif lcore != lcore_tim0 and cnt < 20:
468 return -1, "Failed: lcore != lcore_tim0 (%d, %d)"%(lcore, lcore_tim0)
470 return -1, "Failed: tim0 cnt > 21"
472 # timer1 each time expires on a different core
474 if lcore == lcore_tim1:
475 return -1, "Failed: lcore == lcore_tim1 (%d, %d)"%(lcore, lcore_tim1)
478 return -1, "Failed: tim1 cnt > 30"
480 # timer0 always expires on the same core
484 elif lcore != lcore_tim2:
485 return -1, "Failed: lcore != lcore_tim2 (%d, %d)"%(lcore, lcore_tim2)
487 return -1, "Failed: tim2 cnt > 30"
489 # timer0 always expires on the same core
493 elif lcore != lcore_tim3:
494 return -1, "Failed: lcore_tim3 changed (%d -> %d)"%(lcore, lcore_tim3)
496 return -1, "Failed: tim3 cnt > 30"
498 # must be 2 different cores
499 if lcore_tim0 == lcore_tim3:
500 return -1, "Failed: lcore_tim0 (%d) == lcore_tim3 (%d)"%(lcore_tim0, lcore_tim3)
505 def ring_autotest(child, command, timeout=10):
506 if wait_prompt(child) != 0:
507 return -1, "Failed: cannot find prompt"
508 child.sendline(command)
509 index = child.expect(["Test OK", "Test Failed",
510 pexpect.TIMEOUT], timeout = timeout)
514 child.sendline("set_watermark test 100")
515 child.sendline("set_quota test 16")
516 child.sendline("dump_ring test")
517 index = child.expect([" watermark=100",
518 pexpect.TIMEOUT], timeout = 1)
520 return -1, "Failed: bad watermark"
522 index = child.expect([" bulk_default=16",
523 pexpect.TIMEOUT], timeout = 1)
525 return -1, "Failed: bad quota"
529 def ring_genreport():
530 s = "Performance curves\n"
531 s += "------------------\n\n"
532 sdk = os.getenv("RTE_SDK")
533 script = os.path.join(sdk, "app/test/graph_ring.py")
534 title ='"Autotest %s %s"'%(target, time.asctime())
535 filename = target + ".txt"
536 os.system("/usr/bin/python %s %s %s"%(script, filename, title))
537 for f in os.listdir("."):
538 if not f.startswith("ring"):
540 if not f.endswith(".svg"):
542 # skip single producer/consumer
548 s += ".. figure:: ../../images/autotests/%s/%s\n"%(target, f)
549 s += " :width: 50%\n\n"
553 def mempool_genreport():
554 s = "Performance curves\n"
555 s += "------------------\n\n"
556 sdk = os.getenv("RTE_SDK")
557 script = os.path.join(sdk, "app/test/graph_mempool.py")
558 title ='"Autotest %s %s"'%(target, time.asctime())
559 filename = target + ".txt"
560 os.system("/usr/bin/python %s %s %s"%(script, filename, title))
561 for f in os.listdir("."):
562 if not f.startswith("mempool"):
564 if not f.endswith(".svg"):
566 # skip when n_keep = 128
570 s += ".. figure:: ../../images/autotests/%s/%s\n"%(target, f)
571 s += " :width: 50%\n\n"
579 if len(sys.argv) > 4:
580 testlist=sys.argv[4].split(',')
581 if testlist[0].startswith('-'):
582 testlist[0]=testlist[0].lstrip('-')
583 test_blacklist=testlist
585 test_whitelist=testlist
587 child = pexpect.spawn(cmdline)
588 autotest = AutoTest(child, log_file,'w')
590 # timeout for memcpy and hash test
591 if "baremetal" in target:
596 autotest.register("eal_report.rst", "EAL-%s"%(target),
597 [ SubTest("Boot", boot_autotest, "boot_autotest"),
598 SubTest("EAL Flags", default_autotest, "eal_flags_autotest"),
599 SubTest("Version", default_autotest, "version_autotest"),
600 SubTest("PCI", default_autotest, "pci_autotest"),
601 SubTest("Memory", memory_autotest, "memory_autotest"),
602 SubTest("Lcore launch", default_autotest, "per_lcore_autotest"),
603 SubTest("Spinlock", spinlock_autotest, "spinlock_autotest"),
604 SubTest("Rwlock", rwlock_autotest, "rwlock_autotest"),
605 SubTest("Atomic", default_autotest, "atomic_autotest"),
606 SubTest("Byte order", default_autotest, "byteorder_autotest"),
607 SubTest("Prefetch", default_autotest, "prefetch_autotest"),
608 SubTest("Debug", default_autotest, "debug_autotest"),
609 SubTest("Cycles", default_autotest, "cycles_autotest"),
610 SubTest("Logs", logs_autotest, "logs_autotest"),
611 SubTest("Memzone", default_autotest, "memzone_autotest"),
612 SubTest("Cpu flags", default_autotest, "cpuflags_autotest"),
613 SubTest("Memcpy", default_autotest, "memcpy_autotest", timeout),
614 SubTest("String Functions", default_autotest, "string_autotest"),
615 SubTest("Alarm", default_autotest, "alarm_autotest", 30),
616 SubTest("Interrupt", default_autotest, "interrupt_autotest"),
619 autotest.register("ring_report.rst", "Ring-%s"%(target),
620 [ SubTest("Ring", ring_autotest, "ring_autotest", 30*60,
624 if "baremetal" in target:
629 autotest.register("mempool_report.rst", "Mempool-%s"%(target),
630 [ SubTest("Mempool", default_autotest, "mempool_autotest",
631 timeout, mempool_genreport)
633 autotest.register("mbuf_report.rst", "Mbuf-%s"%(target),
634 [ SubTest("Mbuf", default_autotest, "mbuf_autotest", timeout=120)
636 autotest.register("timer_report.rst", "Timer-%s"%(target),
637 [ SubTest("Timer", timer_autotest, "timer_autotest")
639 autotest.register("malloc_report.rst", "Malloc-%s"%(target),
640 [ SubTest("Malloc", default_autotest, "malloc_autotest")
643 # only do the hash autotest if supported by the platform
644 if not (platform.startswith("Intel(R) Core(TM)2 Quad CPU") or
645 platform.startswith("QEMU")):
646 autotest.register("hash_report.rst", "Hash-%s"%(target),
647 [ SubTest("Hash", default_autotest, "hash_autotest", timeout)
650 autotest.register("lpm_report.rst", "LPM-%s"%(target),
651 [ SubTest("Lpm", default_autotest, "lpm_autotest", timeout)
653 autotest.register("eal2_report.rst", "EAL2-%s"%(target),
654 [ SubTest("TailQ", default_autotest, "tailq_autotest"),
655 SubTest("Errno", default_autotest, "errno_autotest"),
656 SubTest("Multiprocess", default_autotest, "multiprocess_autotest")
660 autotest.gen_report()