Implement a basic KVM selftest runner in Python to run selftests. Add command line options to select individual testcase file or a directory containing multiple testcase files. After selecting the tests to run, start their execution and print their final execution status (passed, failed, skipped, no run), stdout and stderr on terminal. Print execution status in colors on the terminals where it is supported to easily distinguish different statuses of the tests execution. If a test fails or times out, then return with a non-zero exit code after all of the tests execution have completed. If none of the tests fails or times out then exit with status 0 Provide some sample test configuration files to demonstrate the execution of the runner. Runner can be started from tools/testing/selftests/kvm directory as: python3 runner --dirs tests OR python3 runner --testcases \ tests/dirty_log_perf_test/no_dirty_log_protect.test This is a very basic implementation of the runner. Next patches will enhance the runner by adding more features like parallelization, dumping output to file system, time limit, out-of-tree builds run, etc. Signed-off-by: Vipin Sharma --- tools/testing/selftests/kvm/.gitignore | 4 +- .../testing/selftests/kvm/runner/__main__.py | 94 +++++++++++++++++++ .../testing/selftests/kvm/runner/selftest.py | 64 +++++++++++++ .../selftests/kvm/runner/test_runner.py | 37 ++++++++ .../2slot_5vcpu_10iter.test | 1 + .../no_dirty_log_protect.test | 1 + 6 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/kvm/runner/__main__.py create mode 100644 tools/testing/selftests/kvm/runner/selftest.py create mode 100644 tools/testing/selftests/kvm/runner/test_runner.py create mode 100644 tools/testing/selftests/kvm/tests/dirty_log_perf_test/2slot_5vcpu_10iter.test create mode 100644 tools/testing/selftests/kvm/tests/dirty_log_perf_test/no_dirty_log_protect.test diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 1d41a046a7bf..95af97b1ff9e 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -3,10 +3,12 @@ !/**/ !*.c !*.h +!*.py !*.S !*.sh +!*.test !.gitignore !config !settings !Makefile -!Makefile.kvm \ No newline at end of file +!Makefile.kvm diff --git a/tools/testing/selftests/kvm/runner/__main__.py b/tools/testing/selftests/kvm/runner/__main__.py new file mode 100644 index 000000000000..8d1a78450e41 --- /dev/null +++ b/tools/testing/selftests/kvm/runner/__main__.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright 2025 Google LLC +# +# Author: vipinsh@google.com (Vipin Sharma) + +import argparse +import logging +import os +import sys + +from test_runner import TestRunner +from selftest import SelftestStatus + + +def cli(): + parser = argparse.ArgumentParser( + prog="KVM Selftests Runner", + formatter_class=argparse.RawTextHelpFormatter, + allow_abbrev=False + ) + + parser.add_argument("-t", + "--testcases", + nargs="*", + default=[], + help="Testcases to run. Provide the space separated testcases paths") + + parser.add_argument("-d", + "--dirs", + nargs="*", + default=[], + help="Run the testcases present in the given directory and all of its sub directories. Provide the space separated paths to add multiple directories.") + + return parser.parse_args() + + +def setup_logging(): + class TerminalColorFormatter(logging.Formatter): + reset = "\033[0m" + red_bold = "\033[31;1m" + green = "\033[32m" + yellow = "\033[33m" + blue = "\033[34m" + + COLORS = { + SelftestStatus.PASSED: green, + SelftestStatus.NO_RUN: blue, + SelftestStatus.SKIPPED: yellow, + SelftestStatus.FAILED: red_bold + } + + def __init__(self, fmt=None, datefmt=None): + super().__init__(fmt, datefmt) + + def format(self, record): + return (self.COLORS.get(record.levelno, "") + + super().format(record) + self.reset) + + logger = logging.getLogger("runner") + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch_formatter = TerminalColorFormatter(fmt="%(asctime)s | %(message)s", + datefmt="%H:%M:%S") + ch.setFormatter(ch_formatter) + logger.addHandler(ch) + + +def fetch_testcases_in_dirs(dirs): + testcases = [] + for dir in dirs: + for root, child_dirs, files in os.walk(dir): + for file in files: + testcases.append(os.path.join(root, file)) + return testcases + + +def fetch_testcases(args): + testcases = args.testcases + testcases.extend(fetch_testcases_in_dirs(args.dirs)) + # Remove duplicates + testcases = list(dict.fromkeys(testcases)) + return testcases + + +def main(): + args = cli() + setup_logging() + testcases = fetch_testcases(args) + return TestRunner(testcases).start() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/testing/selftests/kvm/runner/selftest.py b/tools/testing/selftests/kvm/runner/selftest.py new file mode 100644 index 000000000000..34005f83f0c3 --- /dev/null +++ b/tools/testing/selftests/kvm/runner/selftest.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright 2025 Google LLC +# +# Author: vipinsh@google.com (Vipin Sharma) + +import pathlib +import enum +import os +import subprocess + +class SelftestStatus(enum.IntEnum): + """ + Selftest Status. Integer values are just +1 to the logging.INFO level. + """ + + PASSED = 21 + NO_RUN = 22 + SKIPPED = 23 + FAILED = 24 + + def __str__(self): + return str.__str__(self.name) + +class Selftest: + """ + Represents a single selftest. + + Extract the test execution command from test file and executes it. + """ + + def __init__(self, test_path): + test_command = pathlib.Path(test_path).read_text().strip() + if not test_command: + raise ValueError("Empty test command in " + test_path) + + test_command = os.path.join(".", test_command) + self.exists = os.path.isfile(test_command.split(maxsplit=1)[0]) + self.test_path = test_path + self.command = test_command + self.status = SelftestStatus.NO_RUN + self.stdout = "" + self.stderr = "" + + def run(self): + if not self.exists: + self.stderr = "File doesn't exists." + return + + run_args = { + "universal_newlines": True, + "shell": True, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE + } + proc = subprocess.run(self.command, **run_args) + self.stdout = proc.stdout + self.stderr = proc.stderr + + if proc.returncode == 0: + self.status = SelftestStatus.PASSED + elif proc.returncode == 4: + self.status = SelftestStatus.SKIPPED + else: + self.status = SelftestStatus.FAILED diff --git a/tools/testing/selftests/kvm/runner/test_runner.py b/tools/testing/selftests/kvm/runner/test_runner.py new file mode 100644 index 000000000000..4418777d75e3 --- /dev/null +++ b/tools/testing/selftests/kvm/runner/test_runner.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright 2025 Google LLC +# +# Author: vipinsh@google.com (Vipin Sharma) + +import logging +from selftest import Selftest +from selftest import SelftestStatus + +logger = logging.getLogger("runner") + + +class TestRunner: + def __init__(self, testcases): + self.tests = [] + + for testcase in testcases: + self.tests.append(Selftest(testcase)) + + def _log_result(self, test_result): + logger.info("*** stdout ***\n" + test_result.stdout) + logger.info("*** stderr ***\n" + test_result.stderr) + logger.log(test_result.status, + f"[{test_result.status.name}] {test_result.test_path}") + + def start(self): + ret = 0 + + for test in self.tests: + test.run() + self._log_result(test) + + if (test.status not in [SelftestStatus.PASSED, + SelftestStatus.NO_RUN, + SelftestStatus.SKIPPED]): + ret = 1 + return ret diff --git a/tools/testing/selftests/kvm/tests/dirty_log_perf_test/2slot_5vcpu_10iter.test b/tools/testing/selftests/kvm/tests/dirty_log_perf_test/2slot_5vcpu_10iter.test new file mode 100644 index 000000000000..5b8d56b44a75 --- /dev/null +++ b/tools/testing/selftests/kvm/tests/dirty_log_perf_test/2slot_5vcpu_10iter.test @@ -0,0 +1 @@ +dirty_log_perf_test -x 2 -v 5 -i 10 diff --git a/tools/testing/selftests/kvm/tests/dirty_log_perf_test/no_dirty_log_protect.test b/tools/testing/selftests/kvm/tests/dirty_log_perf_test/no_dirty_log_protect.test new file mode 100644 index 000000000000..ed3490b1d1a1 --- /dev/null +++ b/tools/testing/selftests/kvm/tests/dirty_log_perf_test/no_dirty_log_protect.test @@ -0,0 +1 @@ +dirty_log_perf_test -g -- 2.51.0.618.g983fd99d29-goog