diff options
| -rw-r--r-- | args.py | 24 | ||||
| -rw-r--r-- | config.py | 6 | ||||
| -rwxr-xr-x | main.py | 93 | ||||
| -rw-r--r-- | suite.py | 57 | ||||
| -rw-r--r-- | suites.py | 25 | ||||
| -rw-r--r-- | test.py | 105 | ||||
| -rw-r--r-- | utils.py | 170 |
7 files changed, 335 insertions, 145 deletions
@@ -0,0 +1,24 @@ +def parse_args(): + parser = argparse.ArgumentParser(description="Minishell test", epilog="Make sure read README.md") + parser.add_argument("-v", "--verbose", action="store_true", help="print test result to stdout") + parser.add_argument("-g", "--generate", type=int, help="number of new random test to generate") + parser.add_argument("-l", "--list", action="store_true", help="print available test suites") + parser.add_argument("suites", nargs='*', metavar="suite", + help="test suites to run (available suites: {})".format(available_suites_str)) + return parser.parse_args() + +def handle_args(): + # utils.verbose = args.verbose + + # check if selected suite is valid + for s in args.suites: + if s not in utils.available_suites: + print("{}: error: the `{}` suite doesn't exist, try {} --list" + .format(sys.argv[0], s, sys.argv[0])) + sys.exit(1) + + # update ignored runned_suites according to the selected ones (if no suite is selected, all are run) + if len(args.suites) != 0: + for available in State.available_suites: + if available not in args.suites: + utils.ignored_suites.append(available) @@ -9,7 +9,7 @@ MINISHELL_EXEC = "minishell" # path to reference shell (shell which will be compared minishell) # has to support the -c option (sh, bash and zsh support it) -REFERENCE_SHELL_PATH = "/bin/bash" +REFERENCE_PATH = "/bin/bash" # log file path LOG_PATH = "result.log" @@ -43,9 +43,11 @@ Perspiciatis ut maxime et libero quo voluptas consequatur illum. Pariatur porro """ LOREM = ' '.join(LOREM.split('\n')) - # do not edit MINISHELL_PATH = os.path.abspath( os.path.join(MINISHELL_DIR, MINISHELL_EXEC) ) + +# 0, 1, 2 +VERBOSE_LEVEL = 0 @@ -5,10 +5,12 @@ import sys import argparse import shutil -import utils +# import utils import config +from suite import Suite import suites + def main(): if not os.path.exists(config.EXECUTABLES_PATH): os.mkdir(config.EXECUTABLES_PATH) @@ -17,57 +19,60 @@ def main(): os.path.join(config.EXECUTABLES_PATH, cmd)) try: - suites.suite_quote() - suites.suite_echo() - suites.suite_redirection() - suites.suite_edgecases() - suites.suite_cmd_error() - suites.suite_interpolation() - suites.suite_glob() - suites.suite_escape() - suites.suite_preprocess() - suites.suite_encoding() + Suite.setup([]) + Suite.run_all() + # suites.suite_quote() + # suites.suite_echo() + # suites.suite_redirection() + # suites.suite_edgecases() + # suites.suite_cmd_error() + # suites.suite_interpolation() + # suites.suite_glob() + # suites.suite_escape() + # suites.suite_preprocess() + # suites.suite_encoding() except KeyboardInterrupt: shutil.rmtree(config.SANDBOX_PATH) if __name__ == "__main__": - available_suites_str = ", ".join(utils.available_suites) - - parser = argparse.ArgumentParser(description="Minishell test", epilog="Make sure read README.md") - parser.add_argument("-v", "--verbose", action="store_true", - help="print test result to stdout") - parser.add_argument("suites", nargs='*', metavar="suite", - help="test suites to run (available suites: {})".format(available_suites_str)) - args = parser.parse_args() - utils.verbose = args.verbose + # available_suites_str = ", ".join(utils.available_suites) + # + # parser = argparse.ArgumentParser(description="Minishell test", epilog="Make sure read README.md") + # parser.add_argument("-v", "--verbose", action="store_true", + # help="print test result to stdout") + # parser.add_argument("suites", nargs='*', metavar="suite", + # help="test suites to run (available suites: {})".format(available_suites_str)) + # args = parser.parse_args() + # utils.verbose = args.verbose # check if selected suite is valid - for s in args.suites: - if s not in utils.available_suites: - print("{}: error: `{}` isn't a valid suite, the available runned_suites are {}" - .format(sys.argv[0], s, available_suites_str)) - sys.exit(1) + # for s in args.suites: + # if s not in utils.available_suites: + # print("{}: error: `{}` isn't a valid suite, the available runned_suites are {}" + # .format(sys.argv[0], s, available_suites_str)) + # sys.exit(1) # update ignored runned_suites according to the selected ones (if no suite is selected, all are run) - if len(args.suites) != 0: - for available in utils.available_suites: - if available not in args.suites: - utils.ignored_suites.append(available) + # if len(args.suites) != 0: + # for available in utils.available_suites: + # if available not in args.suites: + # utils.ignored_suites.append(available) main() - log_file = open(config.LOG_PATH, "w") - print("Summary:") - for suite_name, results in utils.runned_suites.items(): - print("{:15} ".format(suite_name), end="") - pass_total = 0 - for (cmd, expected, actual, files, expected_files, actual_files) in results: - if utils.check(expected, actual, expected_files, actual_files): - pass_total += 1 - else: - log_file.write(utils.diff(cmd, expected, actual, files, expected_files, actual_files)) - log_file.write("=" * 80 + "\n\n") - print(utils.green("{:2} [PASS]".format(pass_total)), end=" ") - print(utils.red("{:2} [FAIL]".format(len(results) - pass_total))) - print("See", config.LOG_PATH, "for more information") - sys.exit(utils.status) + + # log_file = open(config.LOG_PATH, "w") + # print("Summary:") + # for suite_name, results in utils.runned_suites.items(): + # print("{:15} ".format(suite_name), end="") + # pass_total = 0 + # for (cmd, expected, actual, files, expected_files, actual_files) in results: + # if utils.check(expected, actual, expected_files, actual_files): + # pass_total += 1 + # else: + # log_file.write(utils.diff(cmd, expected, actual, files, expected_files, actual_files)) + # log_file.write("=" * 80 + "\n\n") + # print(utils.green("{:2} [PASS]".format(pass_total)), end=" ") + # print(utils.red("{:2} [FAIL]".format(len(results) - pass_total))) + # print("See", config.LOG_PATH, "for more information") + # sys.exit(utils.status) diff --git a/suite.py b/suite.py new file mode 100644 index 0000000..3da5f67 --- /dev/null +++ b/suite.py @@ -0,0 +1,57 @@ +import config +from test import Test + +class Suite: + available = [] + + @classmethod + def run_all(cls): + for s in cls.available: + s.run() + + @classmethod + def setup(cls, asked_names: [str]): + if len(asked_names) == 0: + asked_names = [s.name for s in cls.available] + for s in cls.available: + if s.name in asked_names: + s.generate() + cls.available = [s for s in cls.available if s.name in asked_names] + + def __init__(self, name: str): + self.name = name + self.generator_func = None + self.tests = [] + + def add(self, test): + self.tests.append(test) + + def add_generator(self, generator): + self.generator_func = generator + + def run(self): + if config.VERBOSE_LEVEL == 0: + print(self.name + ": ", end="") + else: + print("{} {:#<41}".format("#" * 39, self.name + " ")) + for t in self.tests: + t.run() + if config.VERBOSE_LEVEL == 0: + print() + + def generate(self): + self.generator_func() + + +def suite(origin): + """ decorator for a suite function (fmt: suite_[name]) """ + + name = origin.__name__[len("suite_"):] + s = Suite(name) + def test_generator(): + def test(cmd: str, setup: str = "", files: [str] = [], exports: {str, str} = {}): + s.add(Test(cmd, setup, files, exports)) + origin(test) + s.add_generator(test_generator) + Suite.available.append(s) + return test_generator @@ -1,8 +1,8 @@ import config -from utils import suite, test +from suite import suite @suite -def suite_quote(): +def suite_quote(test): test("'echo' 'bonjour'") test("'echo' 'je' 'suis' 'charles'") @@ -18,7 +18,7 @@ def suite_quote(): test('echo "\\\\"') @suite -def suite_echo(): +def suite_echo(test): test("echo bonjour") test("echo lalalala lalalalal alalalalal alalalala") test("echo lalalala lalalalal alalalalal alalalala") @@ -30,7 +30,7 @@ def suite_echo(): test("echo -n " + config.LOREM) @suite -def suite_redirection(): +def suite_redirection(test): test("echo bonjour > test", setup="", files=["test"]) test("echo > test bonjour", setup="", files=["test"]) test("> test echo bonjour", setup="", files=["test"]) @@ -85,11 +85,12 @@ def suite_redirection(): files=["abcdefghijklmnopqrstuvzxyz"]) @suite -def suite_edgecases(): +def suite_edgecases(test): test('echo "\\"" >>a"b""c" ', files=["abc"]) + test("echo " + ''.join([chr(i) for i in range(1, 127) if chr(i) not in '\n`"\'()|&><'])) @suite -def suite_cmd_error(): +def suite_cmd_error(test): test(">") test(">>") test("<") @@ -110,7 +111,7 @@ def suite_cmd_error(): test("cat <<<<< bar", setup="echo bonjour > bar") @suite -def suite_interpolation(): +def suite_interpolation(test): test("echo $TEST", exports={"TEST": "bonjour"}) test("echo $TES", exports={"TEST": "bonjour"}) test("echo $TEST_", exports={"TEST": "bonjour"}) @@ -144,7 +145,7 @@ def suite_interpolation(): test("echo $") @suite -def suite_glob(): +def suite_glob(test): test("echo *") test("echo *", setup="touch a b c") test("echo *.c", setup="touch a b c foo.c bar.c") @@ -205,7 +206,7 @@ def suite_glob(): test("echo d/*", setup="mkdir d; touch d/a d/b d/c") @suite -def suite_escape(): +def suite_escape(test): test(r"echo \a") test(r"\e\c\h\o bonjour") test(r"echo charles\ ") @@ -217,7 +218,7 @@ def suite_escape(): test(r"echo\ bonjour") @suite -def suite_preprocess(): +def suite_preprocess(test): test(r"echo \*", setup="touch a b c") test(r"echo \*\*", setup="touch a b c") test(r"echo \ *", setup="touch a b c") @@ -232,7 +233,3 @@ def suite_preprocess(): setup="mkdir src; touch src/a src/b src/c src/foo.c src/bar.c;\ mkdir inc; touch inc/a inc/b inc/c inc/foo.c inc/bar.c", exports={"A": "*/.", "B": "*.c"}) - -@suite -def suite_encoding(): - test("echo " + ''.join([chr(i) for i in range(1, 127) if chr(i) not in '\n`"\'()|&><'])) @@ -0,0 +1,105 @@ +# ############################################################################ # +# # +# ::: :::::::: # +# test.py :+: :+: :+: # +# +:+ +:+ +:+ # +# By: charles <charles.cabergs@gmail.com> +#+ +:+ +#+ # +# +#+#+#+#+#+ +#+ # +# Created: 2020/06/16 21:48:50 by charles #+# #+# # +# Updated: 2020/06/17 08:52:48 by charles ### ########.fr # +# # +# ############################################################################ # + +import os +import sys +import subprocess +import shutil +import config + +import utils + +class Result: + def __init__(self, output: str, files_content: [str]): + self.output = output + self.files_content = files_content + + def __eq__(self, other: 'Result') -> bool: + return (self.output == other.output and + all([x == y for x, y in zip(self.files_content, other.files_content)])) + + +class Test: + def __init__(self, cmd: str, setup: str = "", files: [str] = [], exports: {str: str} = {}): + self.cmd = cmd + self.setup = setup + self.files = files + self.exports = exports + + def run(self): + self.expected = self._run_sandboxed(config.REFERENCE_PATH) + self.actual = self._run_sandboxed(config.MINISHELL_PATH) + + self._put_result() + + # if not verbose: + # put_result(passed, cmd) + # if verbose: + # if not passed: + # print(diff(cmd, expected, actual, files, expected_files, actual_files, color=True)) + # else: + # self._put_line_result(passed) + + def _run_sandboxed(self, shell_path: str) -> Result: + """ run the command in a sandbox environment, return the output (stdout and stderr) of it """ + + try: + os.mkdir(config.SANDBOX_PATH) + except OSError: + pass + if self.setup != "": + try: + setup_status = subprocess.run(self.setup, + shell=True, + cwd=config.SANDBOX_PATH, + check=True) + except subprocess.CalledProcessError as e: + print("Error: `{}` setup command failed for `{}`\n\twith '{}'" + .format(setup, cmd, e.stderr.decode().strip())) + sys.exit(1) + + # TODO: add timeout + # https://docs.python.org/3/library/subprocess.html#using-the-subprocess-module + process_status = subprocess.run([shell_path, "-c", self.cmd], + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + cwd=config.SANDBOX_PATH, + env={'PATH': config.PATH_VARIABLE, **self.exports}) + output = process_status.stdout.decode() + + files_content = [] + for file_name in self.files: + try: + with open(os.path.join(config.SANDBOX_PATH, file_name), "rb") as f: + files_content.append(f.read().decode()) + except FileNotFoundError as e: + files_content.append(None) + + shutil.rmtree(config.SANDBOX_PATH) + return Result(output, files_content) + + def _put_result(self): + passed = self.actual == self.expected + if config.VERBOSE_LEVEL == 0: + sys.stdout.write(utils.green('.') if passed else utils.red('!')) + sys.stdout.flush() + elif config.VERBOSE_LEVEL == 1: + printed = self.cmd + if len(printed) > 70: + printed = printed[:67] + "..." + fmt = utils.green("{:74} [PASS]") if passed else utils.red("{:74} [FAIL]") + print(fmt.format(printed)) + elif config.VERBOSE_LEVEL == 2: + pass + # print(diff(cmd, expected, actual, files, expected_files, actual_files, color=True)) + else: + raise RuntimeError @@ -88,88 +88,88 @@ def put_result(passed: bool, cmd: str): print(red("{:74} [FAIL]".format(cmd))) -def run_sandboxed(program: str, cmd: str, setup: str = None, files: [str] = [], exports: {str, str} = {}) -> str: - """ run the command in a sandbox environment, return the output (stdout and stderr) of it """ - - try: - os.mkdir(config.SANDBOX_PATH) - except OSError: - pass - if setup is not None: - try: - setup_status = subprocess.run(setup, shell=True, cwd=config.SANDBOX_PATH, check=True) - except subprocess.CalledProcessError as e: - print("Error: `{}` setup command failed for `{}`\n\twith '{}'" - .format(setup, cmd, e.stderr.decode().strip())) - sys.exit(1) - - # TODO: add timeout - # https://docs.python.org/3/library/subprocess.html#using-the-subprocess-module - process_status = subprocess.run([program, "-c", cmd], - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - cwd=config.SANDBOX_PATH, - env={'PATH': config.PATH_VARIABLE, **exports}) - output = process_status.stdout.decode() - - output_files = [] - for file_name in files: - try: - with open(os.path.join(config.SANDBOX_PATH, file_name), "rb") as f: - output_files.append(f.read().decode()) - except FileNotFoundError as e: - output_files.append(None) - - shutil.rmtree(config.SANDBOX_PATH) - return (output, output_files) - -status = 0 -ignored_suites = [] -runned_suites = {} -current_suite = "default" -verbose = False - -def check(expected: str, actual: str, expected_files: [str], actual_files: [str]) -> bool: - return actual == expected and all([a == e for a, e in zip(actual_files, expected_files)]) - -def test(cmd: str, setup: str = None, files: [str] = [], exports: {str, str} = {}): - """ get expected and actual strings, compare them and push them to the suites result """ - - (expected, expected_files) = run_sandboxed(config.REFERENCE_SHELL_PATH, cmd, setup, files, exports) - (actual, actual_files) = run_sandboxed(config.MINISHELL_PATH, cmd, setup, files, exports) - - passed = check(expected, actual, expected_files, actual_files) - global status - if passed: - status = 1 - - if not verbose: - put_result(passed, cmd) - if verbose: - if not passed: - print(diff(cmd, expected, actual, files, expected_files, actual_files, color=True)) - else: - put_result(passed, cmd) - - if runned_suites.get(current_suite) is None: - runned_suites[current_suite] = [] - runned_suites[current_suite].append((cmd, expected, actual, files, expected_files, actual_files)) - -available_suites = [] - -def suite(origin): - """ decorator for a suite function (fmt: suite_[name]) - update the current_suite global and print it before the suite execution - """ - - name = origin.__name__[len("suite_"):] - available_suites.append(name) - def f(): - if name in ignored_suites: - return - global current_suite - current_suite = name.upper() - print("{} {:#<41}".format("#" * 39, current_suite + " ")) - origin() - print() - return f +# def run_sandboxed(program: str, cmd: str, setup: str = None, files: [str] = [], exports: {str, str} = {}) -> str: +# """ run the command in a sandbox environment, return the output (stdout and stderr) of it """ +# +# try: +# os.mkdir(config.SANDBOX_PATH) +# except OSError: +# pass +# if setup is not None: +# try: +# setup_status = subprocess.run(setup, shell=True, cwd=config.SANDBOX_PATH, check=True) +# except subprocess.CalledProcessError as e: +# print("Error: `{}` setup command failed for `{}`\n\twith '{}'" +# .format(setup, cmd, e.stderr.decode().strip())) +# sys.exit(1) +# +# # TODO: add timeout +# # https://docs.python.org/3/library/subprocess.html#using-the-subprocess-module +# process_status = subprocess.run([program, "-c", cmd], +# stderr=subprocess.STDOUT, +# stdout=subprocess.PIPE, +# cwd=config.SANDBOX_PATH, +# env={'PATH': config.PATH_VARIABLE, **exports}) +# output = process_status.stdout.decode() +# +# output_files = [] +# for file_name in files: +# try: +# with open(os.path.join(config.SANDBOX_PATH, file_name), "rb") as f: +# output_files.append(f.read().decode()) +# except FileNotFoundError as e: +# output_files.append(None) +# +# shutil.rmtree(config.SANDBOX_PATH) +# return (output, output_files) +# +# status = 0 +# ignored_suites = [] +# runned_suites = {} +# current_suite = "default" +# verbose = False +# +# def check(expected: str, actual: str, expected_files: [str], actual_files: [str]) -> bool: +# return actual == expected and all([a == e for a, e in zip(actual_files, expected_files)]) +# +# def test(cmd: str, setup: str = None, files: [str] = [], exports: {str, str} = {}): +# """ get expected and actual strings, compare them and push them to the suites result """ +# +# (expected, expected_files) = run_sandboxed(config.REFERENCE_SHELL_PATH, cmd, setup, files, exports) +# (actual, actual_files) = run_sandboxed(config.MINISHELL_PATH, cmd, setup, files, exports) +# +# passed = check(expected, actual, expected_files, actual_files) +# global status +# if passed: +# status = 1 +# +# if not verbose: +# put_result(passed, cmd) +# if verbose: +# if not passed: +# print(diff(cmd, expected, actual, files, expected_files, actual_files, color=True)) +# else: +# put_result(passed, cmd) +# +# if runned_suites.get(current_suite) is None: +# runned_suites[current_suite] = [] +# runned_suites[current_suite].append((cmd, expected, actual, files, expected_files, actual_files)) +# +# available_suites = [] +# +# def suite(origin): +# """ decorator for a suite function (fmt: suite_[name]) +# update the current_suite global and print it before the suite execution +# """ +# +# name = origin.__name__[len("suite_"):] +# available_suites.append(name) +# def f(): +# if name in ignored_suites: +# return +# global current_suite +# current_suite = name.upper() +# print("{} {:#<41}".format("#" * 39, current_suite + " ")) +# origin() +# print() +# return f |