From d0a80859f630866461e8a888b3f8fe008c8158ba Mon Sep 17 00:00:00 2001 From: Charles Cabergs Date: Fri, 11 Sep 2020 14:27:26 +0200 Subject: Added suite group and suite bonus, Added signal (not tested) --- src/args.py | 16 ++++++++++---- src/config.py | 6 +++--- src/main.py | 21 +++++++++--------- src/sandbox.py | 32 ++++++++++++++++++++++++++++ src/suite/decorator.py | 31 ++++++++++++++++----------- src/suite/suite.py | 14 ++++++++---- src/suites/__init__.py | 4 +--- src/suites/builtin.py | 14 ++++++------ src/suites/cmd.py | 20 +++++++----------- src/suites/operation.py | 54 +++++++++++++++++++++++------------------------ src/suites/parenthesis.py | 4 ++-- src/suites/path.py | 2 +- src/suites/preprocess.py | 40 +++++++++++++++++------------------ src/suites/status.py | 2 +- src/test/test.py | 33 ++++++++++++++--------------- 15 files changed, 169 insertions(+), 124 deletions(-) create mode 100644 src/sandbox.py (limited to 'src') diff --git a/src/args.py b/src/args.py index 2d0d57a..c9e637b 100644 --- a/src/args.py +++ b/src/args.py @@ -6,7 +6,7 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 18:24:32 by charles #+# #+# # -# Updated: 2020/09/10 13:52:37 by charles ### ########.fr # +# Updated: 2020/09/11 14:22:47 by charles ### ########.fr # # # # ############################################################################ # @@ -23,16 +23,24 @@ def parse_args(): help="increase verbosity level (e.g -vv == 2)" ) parser.add_argument( - "-b", "--build", action="store_true", - help="build minishell and exit" + "-b", "--bonus", action="store_true", + help="enable bonus tests" + ) + parser.add_argument( + "-n", "--no-bonus", action="store_true", + help="disable bonus tests" ) parser.add_argument( "-l", "--list", action="store_true", help="print available test suites" ) + parser.add_argument( + "-m", "--make", action="store_true", + help="make minishell and exit" + ) parser.add_argument( "suites", nargs='*', metavar="suite", - help="test suites to run (-h for more information)" + help="test suites/group to run" ) tmp = parser.parse_args() if tmp.verbose is None: diff --git a/src/config.py b/src/config.py index ca697f5..59c7dad 100644 --- a/src/config.py +++ b/src/config.py @@ -6,7 +6,7 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 18:24:19 by charles #+# #+# # -# Updated: 2020/09/11 12:21:14 by charles ### ########.fr # +# Updated: 2020/09/11 14:21:14 by charles ### ########.fr # # # # ############################################################################ # @@ -26,8 +26,8 @@ MINISHELL_DIR = ".." # minishell executable MINISHELL_EXEC = "minishell" -# build minishell before executing the test if set to True -MINISHELL_BUILD = True +# make minishell before executing the test if set to True +MINISHELL_MAKE = True # path to reference shell (shell which will be compared minishell) # has to support the -c option (sh, bash and zsh support it) diff --git a/src/main.py b/src/main.py index a71a485..3ad1ae4 100755 --- a/src/main.py +++ b/src/main.py @@ -19,29 +19,26 @@ import distutils.spawn import subprocess import config +import sandbox from args import parse_args from suite import Suite -import suites.builtin -import suites.cmd -import suites.preprocess -import suites.operation -import suites.parenthesis -import suites.status -import suites.path +from suites import * def main(): args = parse_args() if args.list: print("The available suites are:") print('\n'.join([" - " + s.name for s in Suite.available])) + print("Groups:") + print('\n'.join([" - " + ', '.join(s.groups) for s in Suite.available])) sys.exit(0) - if config.MINISHELL_BUILD or args.build: + if config.MINISHELL_MAKE or args.make: try: subprocess.run(["make", "-C", config.MINISHELL_DIR], check=True) except subprocess.CalledProcessError: sys.exit(1) - if args.build: + if args.make: sys.exit(0) if os.path.exists(config.EXECUTABLES_PATH): shutil.rmtree(config.EXECUTABLES_PATH) @@ -52,11 +49,15 @@ def main(): config.VERBOSE_LEVEL = args.verbose + if args.bonus: + config.BONUS = True + if args.no_bonus: + config.BONUS = False Suite.setup(args.suites) try: Suite.run_all() except KeyboardInterrupt: - shutil.rmtree(config.SANDBOX_PATH) + sandbox.remove() Suite.summarize() Suite.save_log() diff --git a/src/sandbox.py b/src/sandbox.py new file mode 100644 index 0000000..e900076 --- /dev/null +++ b/src/sandbox.py @@ -0,0 +1,32 @@ +# ############################################################################ # +# # +# ::: :::::::: # +# sandbox.py :+: :+: :+: # +# +:+ +:+ +:+ # +# By: charles +#+ +:+ +#+ # +# +#+#+#+#+#+ +#+ # +# Created: 2020/09/11 13:48:07 by charles #+# #+# # +# Updated: 2020/09/11 13:50:08 by charles ### ########.fr # +# # +# ############################################################################ # + +import os +import glob +import shutil +import subprocess + +import config + +def create(): + try: + os.mkdir(config.SANDBOX_PATH) + except OSError: + pass + + +def remove(): + try: + shutil.rmtree(config.SANDBOX_PATH) + except: + subprocess.run(["chmod", "777", *glob.glob(config.SANDBOX_PATH + "/*")], check=True) + subprocess.run(["rm", "-rf", config.SANDBOX_PATH], check=True) diff --git a/src/suite/decorator.py b/src/suite/decorator.py index 55c9de6..4f1aaa9 100644 --- a/src/suite/decorator.py +++ b/src/suite/decorator.py @@ -6,22 +6,29 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/09/11 12:28:00 by charles #+# #+# # -# Updated: 2020/09/11 12:28:14 by charles ### ########.fr # +# Updated: 2020/09/11 14:13:34 by charles ### ########.fr # # # # ############################################################################ # from suite import Suite from test import Test +import inspect -def suite(origin): - """ decorator for a suite function (fmt: suite_[name]) """ +def suite(groups: [str] = [], bonus: bool = False): + def suite_wrapper(origin): + """ decorator for a suite function (fmt: suite_[name]) """ - name = origin.__name__[len("suite_"):] - s = Suite(name) - def test_generator(): - def test(*args, **kwargs): - s.add(Test(*args, **kwargs)) - origin(test) - s.add_generator(test_generator) - Suite.available.append(s) - return test_generator + mod_name = inspect.getmodule(origin).__name__[len("suites."):] + # print(mod_name) + + name = origin.__name__[len("suite_"):] + s = Suite(name, groups + [mod_name], bonus) + def test_generator(): + def test(*args, **kwargs): + s.add(Test(*args, **kwargs)) + origin(test) + s.add_generator(test_generator) + Suite.available.append(s) + return test_generator + + return suite_wrapper diff --git a/src/suite/suite.py b/src/suite/suite.py index fee4aa9..d796cf7 100644 --- a/src/suite/suite.py +++ b/src/suite/suite.py @@ -6,12 +6,11 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 18:24:29 by charles #+# #+# # -# Updated: 2020/09/11 12:27:47 by charles ### ########.fr # +# Updated: 2020/09/11 14:18:01 by charles ### ########.fr # # # # ############################################################################ # import config -from test import Test class Suite: @@ -26,7 +25,12 @@ class Suite: def setup(cls, asked_names: [str]): if len(asked_names) == 0: asked_names = [s.name for s in cls.available] - cls.available = [s for s in cls.available if s.name in asked_names] + if not config.BONUS: + cls.available = [s for s in cls.available if not s.bonus] + cls.available = list(set( + [s for s in cls.available if s.name in asked_names] + + [s for s in cls.available if any([g for g in s.groups if g in asked_names])] + )) for s in cls.available: s.generate() @@ -34,8 +38,10 @@ class Suite: def available_names(cls) -> [str]: return [s.name for s in cls.available] - def __init__(self, name: str): + def __init__(self, name: str, groups: [str], bonus: bool = False): self.name = name + self.groups = groups + self.bonus = bonus self.generator_func = None self.tests = [] diff --git a/src/suites/__init__.py b/src/suites/__init__.py index 68bad1f..b6b3b68 100644 --- a/src/suites/__init__.py +++ b/src/suites/__init__.py @@ -6,7 +6,7 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 18:24:48 by charles #+# #+# # -# Updated: 2020/09/09 15:20:22 by charles ### ########.fr # +# Updated: 2020/09/11 13:25:26 by charles ### ########.fr # # # # ############################################################################ # @@ -15,5 +15,3 @@ import glob modules = glob.glob(os.path.join(os.path.dirname(__file__), "*.py")) __all__ = [os.path.basename(f)[:-3] for f in modules if os.path.isfile(f) and not f.endswith("__init__.py")] - -# print(__all__) diff --git a/src/suites/builtin.py b/src/suites/builtin.py index 30297e0..da146e0 100644 --- a/src/suites/builtin.py +++ b/src/suites/builtin.py @@ -15,7 +15,7 @@ import os import config from suite import suite -@suite +@suite() def suite_echo(test): test("echo") test("echo bonjour") @@ -40,7 +40,7 @@ def suite_echo(test): test('echo -n a "" b "" c "" d') test("echo '' '' ''") -@suite +@suite() def suite_export(test): test("export") # test("export A=; env | grep A=; echo $A") @@ -90,7 +90,7 @@ def suite_export(test): test("export $TEST", exports={"TEST": "A=a"}) -@suite +@suite() def suite_cd(test): test("cd .; pwd; echo $PWD"); test("cd ..; pwd; echo $PWD"); @@ -182,7 +182,7 @@ def suite_cd(test): test("cd d", setup="mkdir -m 7777 d") test("cd d", setup="mkdir -m 0000 d") -@suite +@suite() def suite_unset(test): test("unset") test("unset A; echo $A", setup="export A=a") @@ -207,7 +207,7 @@ def suite_unset(test): setup="export A=a B=b C=c") test("unset A", setup="export A=a B=b C=c") -@suite +@suite() def suite_pwd(test): test("pwd") test("pwd", setup="cd ..") @@ -218,14 +218,14 @@ def suite_pwd(test): test("pwd | cat -e") test("cd lnk; rmdir ../d; pwd", setup="mkdir d; ln -s d lnk") -@suite +@suite() def suite_env(test): test("env") # TODO ordering doesn't mater flag test("env", setup="export A=a") test("env", setup="export A=a B=b C=c") test("env | cat -e", setup="export A=a B=b C=c") -@suite +@suite() def suite_exit(test): test("exit") test("exit 1") diff --git a/src/suites/cmd.py b/src/suites/cmd.py index 1302ae3..4fc5f55 100644 --- a/src/suites/cmd.py +++ b/src/suites/cmd.py @@ -6,7 +6,7 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 15:11:46 by charles #+# #+# # -# Updated: 2020/09/10 14:25:40 by charles ### ########.fr # +# Updated: 2020/09/11 14:25:04 by charles ### ########.fr # # # # ############################################################################ # @@ -16,7 +16,7 @@ import config from suite import suite -@suite +@suite() def suite_redirection(test): test("echo bonjour > test", setup="", files=["test"]) test("echo > test bonjour", setup="", files=["test"]) @@ -96,9 +96,10 @@ def suite_redirection(test): test("cat < doesnotexist") - -@suite -def suite_edgecases(test): +@suite() +def suite_cmd(test): + test("notfound") + test("notfound a b c") test('echo "\\"" >>a"b""c" ', files=["abc"]) test("echo " + ''.join([chr(i) for i in range(1, 127) if chr(i) not in '\n`"\'()|&><'])) test("echo foo>bar", files=["bar"]) @@ -106,12 +107,7 @@ def suite_edgecases(test): test("echo foo> bar", files=["bar"]) test("echo foo > bar", files=["bar"]) -@suite -def suite_cmd(test): - test("notfound") - test("notfound a b c") - -@suite +@suite(bonus=True) def suite_cmd_variable(test): test("A=a sh -c 'echo $A'") test("A=a B=b sh -c 'echo $A$B'") @@ -202,7 +198,7 @@ def suite_cmd_variable(test): test("'BONJOURJESUIS''=''a' sh -c 'echo $BONJOURJESUIS'") test('"BONJOURJESUIS""=""a" sh -c "echo $BONJOURJESUIS"') -@suite +@suite() def suite_cmd_path(test): ls_path = distutils.spawn.find_executable("ls") cat_path = distutils.spawn.find_executable("cat") diff --git a/src/suites/operation.py b/src/suites/operation.py index 3c89589..d58953b 100644 --- a/src/suites/operation.py +++ b/src/suites/operation.py @@ -6,13 +6,13 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 18:24:52 by charles #+# #+# # -# Updated: 2020/07/19 10:23:22 by charles ### ########.fr # +# Updated: 2020/09/11 14:19:14 by charles ### ########.fr # # # # ############################################################################ # from suite import suite -@suite +@suite() def suite_end(test): test("echo bonjour; echo je") test("echo bonjour ;echo je") @@ -35,7 +35,30 @@ def suite_end(test): test("ls doesnotexists; echo bonjour") test("echo bonjour; ls doesnotexists") -@suite +@suite() +def suite_pipe(test): + test("echo bonjour | cat") + test("echo bonjour | cat -e") + test("ls | cat -e", setup="touch a b c d; mkdir m1 m2 m3") + test("ls -l | cat -e", setup="touch a b c d; mkdir m1 m2 m3") + test("ls -l | cat -e | cat | cat | cat", setup="touch a b c d; mkdir m1 m2 m3") + test("ls -l | cat -e | cat -e | cat -e | cat -e", setup="touch a b c d; mkdir m1 m2 m3") + test("ls -l | cat -e < a", setup="touch a b c d; mkdir m1 m2 m3; echo bonjour > a") + + # TODO special test for potential segfault + # test("echo|") + # test("echo |") + # test("echo | ") + test("|cat") + test("| cat") + test(" | cat") + + test("echo a | export A=a; echo $A") + test("export A=a | cat; echo $A") + test("echo a | A=a; echo $A") + test("A=a | cat; echo $A") + +@suite(bonus=True) def suite_and(test): test("echo bonjour&& echo je") test("echo bonjour &&echo je") @@ -58,7 +81,7 @@ def suite_and(test): test("ls doesnotexists&& echo bonjour") test("echo bonjour&& ls doesnotexists") -@suite +@suite(bonus=True) def suite_or(test): test("echo bonjour|| echo je") test("echo bonjour ||echo je") @@ -80,26 +103,3 @@ def suite_or(test): test("ls doesnotexists || echo bonjour") test("ls doesnotexists|| echo bonjour") test("echo bonjour|| ls doesnotexists") - -@suite -def suite_pipe(test): - test("echo bonjour | cat") - test("echo bonjour | cat -e") - test("ls | cat -e", setup="touch a b c d; mkdir m1 m2 m3") - test("ls -l | cat -e", setup="touch a b c d; mkdir m1 m2 m3") - test("ls -l | cat -e | cat | cat | cat", setup="touch a b c d; mkdir m1 m2 m3") - test("ls -l | cat -e | cat -e | cat -e | cat -e", setup="touch a b c d; mkdir m1 m2 m3") - test("ls -l | cat -e < a", setup="touch a b c d; mkdir m1 m2 m3; echo bonjour > a") - - # TODO special test for potential segfault - # test("echo|") - # test("echo |") - # test("echo | ") - test("|cat") - test("| cat") - test(" | cat") - - test("echo a | export A=a; echo $A") - test("export A=a | cat; echo $A") - test("echo a | A=a; echo $A") - test("A=a | cat; echo $A") diff --git a/src/suites/parenthesis.py b/src/suites/parenthesis.py index a06fdda..98320e8 100644 --- a/src/suites/parenthesis.py +++ b/src/suites/parenthesis.py @@ -6,13 +6,13 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 18:24:57 by charles #+# #+# # -# Updated: 2020/07/15 18:24:57 by charles ### ########.fr # +# Updated: 2020/09/11 14:23:35 by charles ### ########.fr # # # # ############################################################################ # from suite import suite -@suite +@suite(bonus=True) def suite_parenthesis(test): test("(echo bonjour)") test("(echo bonjour )") diff --git a/src/suites/path.py b/src/suites/path.py index b30215f..e4a650b 100644 --- a/src/suites/path.py +++ b/src/suites/path.py @@ -13,7 +13,7 @@ import config from suite import suite -@suite +@suite() def suite_path(test): test("a", setup="mkdir path && cp /bin/ls ./path/a && chmod 000 ./path/a", exports={"PATH": "path"}) test("a", setup="mkdir path && cp /bin/ls ./path/a && chmod 001 ./path/a", exports={"PATH": "path"}) diff --git a/src/suites/preprocess.py b/src/suites/preprocess.py index a34e18d..2f36e64 100644 --- a/src/suites/preprocess.py +++ b/src/suites/preprocess.py @@ -6,14 +6,14 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 18:25:00 by charles #+# #+# # -# Updated: 2020/09/09 12:50:34 by charles ### ########.fr # +# Updated: 2020/09/11 14:24:47 by charles ### ########.fr # # # # ############################################################################ # import config from suite import suite -@suite +@suite() def suite_quote(test): test("'echo' 'bonjour'") test("'echo' 'je' 'suis' 'charles'") @@ -56,7 +56,7 @@ def suite_quote(test): test("'''''''e''''''''''c''''''''''''h''''''''o''''''''''''''''''''' bonjour") test('"""""""e""""""""""c""""""""""""h""""""""o""""""""""""""""""""" bonjour') -@suite +@suite() def suite_interpolation(test): test("echo $TEST", exports={"TEST": "bonjour"}) test("echo $TES", exports={"TEST": "bonjour"}) @@ -171,7 +171,7 @@ def suite_interpolation(test): test('echo $A$B$C', exports={"A": "", "B": "", "C": ""}) -@suite +@suite() def suite_escape(test): test(r"echo \a") test(r"\e\c\h\o bonjour") @@ -193,24 +193,8 @@ def suite_escape(test): test(r" \ echo bonjour") test(r" \ echo bonjour") -# @suite -# def suite_preprocess(test): -# test(r"echo \*", setup="touch a b c") -# test(r"echo \*\*", setup="touch a b c") -# test(r"echo \ *", setup="touch a b c") -# test(r"echo *\.c", setup="touch a.c b.c c.c") -# test(r"echo *.\c", setup="touch a.c b.c c.c") -# test(r"echo *.c\ ", setup="touch a.c b.c c.c") -# test("echo $A$B", -# setup="mkdir src; touch src/a src/b src/c src/foo.c src/bar.c;\ -# mkdir inc; touch inc/a inc/b inc/c inc/foo.c inc/bar.c", -# exports={"A": "*", "B": "/*.c"}) -# test("echo $A$B", -# setup="mkdir src; touch src/a src/b src/c src/foo.c src/bar.c;\ -# mkdir inc; touch inc/a inc/b inc/c inc/foo.c inc/bar.c", -# exports={"A": "*/.", "B": "*.c"}) -# @suite +# @suite(bonus=True) # def suite_glob(test): # test("echo *") # test("echo *", setup="touch a b c") @@ -404,3 +388,17 @@ def suite_escape(test): # test("echo *", setup="touch a; ln -s a b; ln -s b c; ln -s c d") # test("echo d/*", setup="mkdir d; touch a b c d/d d/e d/f") # test("echo d/*", setup="mkdir d; touch a b c d/d d/e d/f; chmod 000 d") +# test(r"echo \*", setup="touch a b c") +# test(r"echo \*\*", setup="touch a b c") +# test(r"echo \ *", setup="touch a b c") +# test(r"echo *\.c", setup="touch a.c b.c c.c") +# test(r"echo *.\c", setup="touch a.c b.c c.c") +# test(r"echo *.c\ ", setup="touch a.c b.c c.c") +# test("echo $A$B", +# setup="mkdir src; touch src/a src/b src/c src/foo.c src/bar.c;\ +# mkdir inc; touch inc/a inc/b inc/c inc/foo.c inc/bar.c", +# exports={"A": "*", "B": "/*.c"}) +# test("echo $A$B", +# setup="mkdir src; touch src/a src/b src/c src/foo.c src/bar.c;\ +# mkdir inc; touch inc/a inc/b inc/c inc/foo.c inc/bar.c", +# exports={"A": "*/.", "B": "*.c"}) diff --git a/src/suites/status.py b/src/suites/status.py index 62c076e..416b69a 100644 --- a/src/suites/status.py +++ b/src/suites/status.py @@ -12,7 +12,7 @@ from suite import suite -@suite +@suite() def suite_status(test): test("echo $?") test("echo; echo $?") diff --git a/src/test/test.py b/src/test/test.py index 9674240..f75600a 100644 --- a/src/test/test.py +++ b/src/test/test.py @@ -6,19 +6,19 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/06/16 21:48:50 by charles #+# #+# # -# Updated: 2020/09/11 12:24:33 by charles ### ########.fr # +# Updated: 2020/09/11 13:50:16 by charles ### ########.fr # # # # ############################################################################ # import os import sys import subprocess -import shutil -import glob +import time import config from test.captured import Captured from test.result import Result +import sandbox class Test: @@ -27,13 +27,15 @@ class Test: setup: str = "", files: [str] = [], exports: {str: str} = {}, - timeout: float = config.TIMEOUT): + timeout: float = config.TIMEOUT, + signal = None): self.cmd = cmd self.setup = setup self.files = files self.exports = exports self.result = None self.timeout = timeout + self.signal = signal def run(self): expected = self._run_sandboxed(config.REFERENCE_PATH, "-c") @@ -54,10 +56,7 @@ class Test: capture the content of the watched files after the command is run """ - try: - os.mkdir(config.SANDBOX_PATH) - except OSError: - pass + sandbox.create() if self.setup != "": try: setup_status = subprocess.run( @@ -86,10 +85,14 @@ class Test: **self.exports, }, ) - try: - process.wait(timeout=self.timeout) - except subprocess.TimeoutExpired: - return Captured.timeout() + if self.signal is not None: + time.sleep(0.2) + process.send_signal(self.signal) + else: + try: + process.wait(timeout=self.timeout) + except subprocess.TimeoutExpired: + return Captured.timeout() try: stdout, _ = process.communicate() @@ -105,9 +108,5 @@ class Test: files_content.append(f.read().decode()) except FileNotFoundError as e: files_content.append(None) - try: - shutil.rmtree(config.SANDBOX_PATH) - except: - subprocess.run(["chmod", "777", *glob.glob(config.SANDBOX_PATH + "/*")], check=True) - subprocess.run(["rm", "-rf", config.SANDBOX_PATH], check=True) + sandbox.remove() return Captured(output, process.returncode, files_content) -- cgit