direct/test-framework/unit_tests/test_runner.py

   1 #!/usr/bin/env python3
   2 """
   3 test_runner.py
   4 This file runs all of the yaffs unit tests and aggrates the outputs.
   5
   6 To run this file you can use:
   7
   8 $ python3 test_runner.py
   9
  10 or
  11
  12 $ ./test_runner.py
  13
  14 To add a new test to this test runner, add the test dir path to the
  15 test_list below. Inside that folder there need to be a makefile with
  16 a "test" target that compiles and runs the test script. I.e. "make test"
  17 is called in every test directory.
  18 """
  19
  20
  21 import subprocess, sys
  22
  23 test_list = ["is_yaffs_working_tests",
  24         "quick_tests",
  25         "64_and_32_bit_time/64_bit",
  26         "64_and_32_bit_time/32_bit",
  27         ]
  28
  29
  30 TEST_FAILED = -1
  31 TEST_PASSED = 1
  32
  33 def run(makefile_paths):
  34         failed_tests =[]
  35         for path in makefile_paths:
  36                 print("\nrunning test {}".format(path))
  37                 is_successful, test_output = run_makefile_test(path)
  38                 if is_successful != TEST_PASSED:
  39                         print('\033[41m' +'test {} failed'.format(path)+'\033[0m')
  40                         print(test_output)
  41                         failed_tests.append( (path, is_successful) )
  42                 else:
  43                         print('\033[42m' +"test passed"+'\033[0m')
  44         return failed_tests
  45
  46 def run_makefile_test(path):
  47         try:
  48                 subprocess.check_output("make -j -C {} test".format(path), shell=True)
  49         except subprocess.CalledProcessError as e:
  50                 return (TEST_FAILED, e.output.decode('UTF-8'))
  51
  52         return (TEST_PASSED, "test passed")
  53
  54 def clean_tests(makefile_paths):
  55     cmds = [("make -j -C {} clean".format(path), path) for path in makefile_paths]
  56
  57     failed = 0
  58     passed = 0
  59
  60     for cmd, is_successful, cmd_text, debug_info in run_cmds(cmds):
  61         if not is_successful:
  62             print("\033[41mtest failed to clean\033[0m {}".format(debug_info[0]))
  63             failed += 1
  64         else :
  65             print("\033[42mtest cleaned successfully\033[0m {}".format(debug_info[0]))
  66             passed += 1
  67     if not failed:
  68         print ("\n\033[42mAll tests cleaned successfully\033[0m")
  69     else :
  70         print ("\n\033[42mTests failed to clean successfully\033[0m")
  71     print("ran {}, passed {}, failed {}".format(len(cmds), passed, failed))
  72
  73 def run_tests(makefile_paths):
  74     cmds = [("make -j -C {} test".format(path), path) for path in makefile_paths]
  75
  76     failed = 0
  77     passed = 0
  78
  79     print("running tests")
  80     for cmd, is_successful, cmd_text, debug_info in run_cmds(cmds):
  81         if not is_successful:
  82             print("\033[41mtest failed\033[0m {}".format(debug_info[0]))
  83             failed += 1
  84         else :
  85             print("\033[42mtest passed\033[0m {}".format(debug_info[0]))
  86             passed += 1
  87     if not failed:
  88         print ("\n\033[42mAll tests passed\033[0m")
  89     else :
  90         print ("\n\033[41mTests failed\033[0m")
  91     print("ran {}, passed {}, failed {}".format(len(cmds), passed, failed))
  92
  93 def run_cmds(cmds):
  94     output = []
  95     for cmd, *debug_info in cmds:
  96         try:
  97             subprocess.check_output(cmd, shell=True)
  98             output.append((cmd, True, "todo add getting text for non failing test", debug_info))
  99         except subprocess.CalledProcessError as e:
 100             output.append((cmd, False, e.output.decode('UTF-8'), debug_info))
 101     return output
 102 if __name__ == "__main__":
 103         if len(sys.argv) == 2 and sys.argv[1] == "clean":
 104             clean_tests(test_list)
 105         elif len(sys.argv) == 1:
 106             #run the test runner.
 107             failed_tests = run_tests(test_list)
 108         else:
 109             print("run with command ./test_runner.py [clean]")