Putting everything in Test and Suite class

8 files changed, 218 insertions, 298 deletions

diff --git a/README.md b/README.md
index 65eb18e..5c07be3 100644
--- a/README.md
+++ b/README.md
@@ -61,7 +61,7 @@ A test suite is a group of related tests.
 
 ```
 @suite
-def suite_yoursuitename():
+def suite_yoursuitename(test):
     test(...)
     test(...)
     test(...)
diff --git a/args.py b/args.py
index 3fa4b7d..2d7455b 100644
--- a/args.py
+++ b/args.py
@@ -1,24 +1,25 @@
-def parse_args():
-    parser = argparse.ArgumentParser(description="Minishell test", epilog="Make sure read README.md")
-    parser.add_argument("-v", "--verbose", action="store_true", help="print test result to stdout")
-    parser.add_argument("-g", "--generate", type=int, help="number of new random test to generate")
-    parser.add_argument("-l", "--list", action="store_true", help="print available test suites")
-    parser.add_argument("suites", nargs='*', metavar="suite",
-                        help="test suites to run (available suites: {})".format(available_suites_str))
-    return parser.parse_args()
-
-def handle_args():
-    # utils.verbose = args.verbose
+import argparse
 
-    # check if selected suite is valid
-    for s in args.suites:
-        if s not in utils.available_suites:
-            print("{}: error: the `{}` suite doesn't exist, try {} --list"
-                    .format(sys.argv[0], s, sys.argv[0]))
-            sys.exit(1)
 
-    # update ignored runned_suites according to the selected ones (if no suite is selected, all are run)
-    if len(args.suites) != 0:
-        for available in State.available_suites:
-            if available not in args.suites:
-                utils.ignored_suites.append(available)
+def parse_args():
+    parser = argparse.ArgumentParser(description="Minishell test", epilog="Make sure read README.md")
+    parser.add_argument(
+        "-v", "--verbose", action="count",
+        help="increase verbosity level (e.g -vv == 2)"
+    )
+    parser.add_argument(
+        "-g", "--generate", metavar="NUMBER", type=int,
+        help="number of new random test to generate"
+    )
+    parser.add_argument(
+        "-l", "--list", action="store_true",
+        help="print available test suites"
+    )
+    parser.add_argument(
+        "suites", nargs='*', metavar="suite",
+        help="test suites to run (-h for more information)"
+    )
+    tmp = parser.parse_args()
+    if tmp.verbose is None:
+        tmp.verbose = 0
+    return tmp
diff --git a/config.py b/config.py
index 0e0d858..7a0f083 100644
--- a/config.py
+++ b/config.py
@@ -50,4 +50,4 @@ MINISHELL_PATH = os.path.abspath(
 )
 
 # 0, 1, 2
-VERBOSE_LEVEL = 0
+VERBOSE_LEVEL = 1
diff --git a/main.py b/main.py
index b311440..8242072 100755
--- a/main.py
+++ b/main.py
@@ -2,15 +2,13 @@
 
 import os
 import sys
-import argparse
 import shutil
 
-# import utils
 import config
+from args import parse_args
 from suite import Suite
 import suites
 
-
 def main():
     if not os.path.exists(config.EXECUTABLES_PATH):
         os.mkdir(config.EXECUTABLES_PATH)
@@ -18,61 +16,23 @@ def main():
         shutil.copy(os.path.join("/usr/bin", cmd),  # search whole PATH
                     os.path.join(config.EXECUTABLES_PATH, cmd))
 
+    args = parse_args()
+    if args.list:
+        print("The available suites are:")
+        print('\n'.join([" - " + s.name for s in Suite.available]))
+        sys.exit(0)
+
+    config.VERBOSE_LEVEL = args.verbose
+    Suite.setup(args.suites)
     try:
-        Suite.setup([])
         Suite.run_all()
-        # suites.suite_quote()
-        # suites.suite_echo()
-        # suites.suite_redirection()
-        # suites.suite_edgecases()
-        # suites.suite_cmd_error()
-        # suites.suite_interpolation()
-        # suites.suite_glob()
-        # suites.suite_escape()
-        # suites.suite_preprocess()
-        # suites.suite_encoding()
     except KeyboardInterrupt:
         shutil.rmtree(config.SANDBOX_PATH)
 
+    Suite.summarize()
+    Suite.save_log()
+    print("See", config.LOG_PATH, "for more information")
 
-if __name__ == "__main__":
-    # available_suites_str = ", ".join(utils.available_suites)
-    #
-    # parser = argparse.ArgumentParser(description="Minishell test", epilog="Make sure read README.md")
-    # parser.add_argument("-v", "--verbose", action="store_true",
-    #                     help="print test result to stdout")
-    # parser.add_argument("suites", nargs='*', metavar="suite",
-    #                     help="test suites to run (available suites: {})".format(available_suites_str))
-    # args = parser.parse_args()
-    # utils.verbose = args.verbose
-
-    # check if selected suite is valid
-    # for s in args.suites:
-    #     if s not in utils.available_suites:
-    #         print("{}: error: `{}` isn't a valid suite, the available runned_suites are {}"
-    #                 .format(sys.argv[0], s, available_suites_str))
-    #         sys.exit(1)
-
-    # update ignored runned_suites according to the selected ones (if no suite is selected, all are run)
-    # if len(args.suites) != 0:
-    #     for available in utils.available_suites:
-    #         if available not in args.suites:
-    #             utils.ignored_suites.append(available)
 
+if __name__ == "__main__":
     main()
-
-    # log_file = open(config.LOG_PATH, "w")
-    # print("Summary:")
-    # for suite_name, results in utils.runned_suites.items():
-    #     print("{:15} ".format(suite_name), end="")
-    #     pass_total = 0
-    #     for (cmd, expected, actual, files, expected_files, actual_files) in results:
-    #         if utils.check(expected, actual, expected_files, actual_files):
-    #             pass_total += 1
-    #         else:
-    #             log_file.write(utils.diff(cmd, expected, actual, files, expected_files, actual_files))
-    #             log_file.write("=" * 80 + "\n\n")
-    #     print(utils.green("{:2} [PASS]".format(pass_total)), end="    ")
-    #     print(utils.red("{:2} [FAIL]".format(len(results) - pass_total)))
-    # print("See", config.LOG_PATH, "for more information")
-    # sys.exit(utils.status)
diff --git a/suite.py b/suite.py
index 3da5f67..7d14230 100644
--- a/suite.py
+++ b/suite.py
@@ -13,10 +13,13 @@ class Suite:
     def setup(cls, asked_names: [str]):
         if len(asked_names) == 0:
             asked_names = [s.name for s in cls.available]
-        for s in cls.available:
-            if s.name in asked_names:
-                s.generate()
         cls.available = [s for s in cls.available if s.name in asked_names]
+        for s in cls.available:
+            s.generate()
+
+    @classmethod
+    def available_names(cls) -> [str]:
+        return [s.name for s in cls.available]
 
     def __init__(self, name: str):
         self.name = name
@@ -42,6 +45,35 @@ class Suite:
     def generate(self):
         self.generator_func()
 
+    def total(self) -> (int, int):
+        passed_total = 0
+        for t in self.tests:
+            if t.result is None:
+                return (-1, -1)
+            if t.result.passed:
+                passed_total += 1
+        return (passed_total, len(self.tests) - passed_total)
+
+    @classmethod
+    def summarize(cls):
+        print("\nSummary:")
+        for s in cls.available:
+            (pass_total, fail_total) = s.total()
+            if pass_total == -1:
+                continue
+            print("{:<15} \033[32m{:2} [PASS]\033[0m  \033[31m{:2} [FAIL]\033[0m"
+                    .format(s.name, pass_total, fail_total))
+
+    @classmethod
+    def save_log(cls):
+        with open(config.LOG_PATH, "w") as log_file:
+            for s in cls.available:
+                for t in s.tests:
+                    if t.result is not None and t.result.failed:
+                        t.result.colored = False
+                        t.result.set_colors()
+                        log_file.write(t.result.full_diff() + '\n')
+
 
 def suite(origin):
     """ decorator for a suite function (fmt: suite_[name]) """
diff --git a/suites.py b/suites.py
index 4c49ba0..b49dd28 100644
--- a/suites.py
+++ b/suites.py
@@ -78,11 +78,11 @@ def suite_redirection(test):
     test("cat<test<je", setup="echo bonjour > test; echo salut > je")
 
     test("echo bonjour > a'b'c'd'e'f'g'h'i'j'k'l'm'n'o'p'q'r's't'u'v'w'x'y'z'",
-            files=["abcdefghijklmnopqrstuvzxyz"])
+            files=["abcdefghijklmnopqrstuvwxyz"])
     test('echo bonjour > a"b"c"d"e"f"g"h"i"j"k"l"m"n"o"p"q"r"s"t"u"v"w"x"y"z"',
-            files=["abcdefghijklmnopqrstuvzxyz"])
+            files=["abcdefghijklmnopqrstuvwxyz"])
     test('echo bonjour > a\'b\'c"d"e\'f\'g"h"i\'j\'k"l"m\'n\'o"p\'q\'r"s\'t\'u"v"w"x"y\'z\'',
-            files=["abcdefghijklmnopqrstuvzxyz"])
+            files=["abcdefghijklmnopqrstuvwxyz"])
 
 @suite
 def suite_edgecases(test):
diff --git a/test.py b/test.py
index 329a3ad..cb502d5 100644
--- a/test.py
+++ b/test.py
@@ -6,7 +6,7 @@
 #    By: charles <charles.cabergs@gmail.com>        +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/06/16 21:48:50 by charles           #+#    #+#              #
-#    Updated: 2020/06/17 08:52:48 by charles          ###   ########.fr        #
+#    Updated: 2020/06/17 11:22:22 by charles          ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
@@ -16,9 +16,7 @@ import subprocess
 import shutil
 import config
 
-import utils
-
-class Result:
+class Captured:
     def __init__(self, output: str, files_content: [str]):
         self.output = output
         self.files_content = files_content
@@ -27,6 +25,133 @@ class Result:
         return (self.output == other.output and
                 all([x == y for x, y in zip(self.files_content, other.files_content)]))
 
+class Result:
+    RED_CHARS   = "\033[31m"
+    GREEN_CHARS = "\033[32m"
+    BLUE_CHARS  = "\033[34m"
+    BOLD_CHARS  = "\033[1m"
+    CLOSE_CHARS = "\033[0m"
+
+    def __init__(self, cmd: str, file_names: [str], expected: Captured, actual: Captured):
+        self.cmd = cmd
+        self.file_names = file_names
+        self.expected = expected
+        self.actual = actual
+        self.colored = True
+        self.set_colors()
+
+    def toggle_colors(self):
+        self.colored = not self.colored
+
+    def set_colors(self):
+        if self.colored:
+            self.color_red   = self.RED_CHARS
+            self.color_green = self.GREEN_CHARS
+            self.color_blue  = self.BLUE_CHARS
+            self.color_bold  = self.BOLD_CHARS
+            self.color_close = self.CLOSE_CHARS
+        else:
+            self.color_red   = ""
+            self.color_green = ""
+            self.color_blue  = ""
+            self.color_bold  = ""
+            self.color_close = ""
+
+    def green(self, s):
+        return self.color_green + s + self.color_close
+
+    def red(self, s):
+        return self.color_red + s + self.color_close
+
+    def blue(self, s):
+        return self.color_blue + s + self.color_close
+
+    def bold(self, s):
+        return self.color_bold + s + self.color_close
+
+    @property
+    def passed(self):
+        return self.actual == self.expected
+
+    @property
+    def failed(self):
+        return not self.passed
+
+    def __repr__(self):
+        if config.VERBOSE_LEVEL == 0:
+            return self.green('.') if self.passed else self.red('!')
+        elif config.VERBOSE_LEVEL == 1:
+            printed = self.cmd[:]
+            if len(printed) > 70:
+                printed = printed[:67] + "..."
+            fmt = self.green("{:74} [PASS]") if self.passed else self.red("{:74} [FAIL]")
+            return fmt.format(printed)
+        elif config.VERBOSE_LEVEL == 2:
+            return self.full_diff()
+        else:
+            raise RuntimeError
+
+    def put(self):
+        if config.VERBOSE_LEVEL == 2 and self.passed:
+            return
+        print(self, end="")
+        if config.VERBOSE_LEVEL == 0:
+            sys.stdout.flush()
+        else:
+            print()
+
+    def header(self, title: str) -> str:
+        return self.bold("|---------------------------------------{:-<40}".format(title))
+
+    @property
+    def expected_header(self) -> str:
+        return self.green(self.header("EXPECTED"))
+
+    @property
+    def actual_header(self) -> str:
+        return self.red(self.header("ACTUAL"))
+
+    def indicator(self, title: str, prefix: str) -> str:
+        return self.bold(self.blue(prefix + " " + title))
+
+    def file_diff(self, file_name: str, expected: str, actual: str) -> str:
+        return (
+            self.indicator("FILE {}".format(file_name), "|#") + '\n'
+            + self.expected_header + '\n'
+            + ("FROM TEST: File not created\n" if expected is None else self.cat_e(expected))
+            + self.actual_header + '\n'
+            + ("FROM TEST: File not created\n" if actual is None else self.cat_e(actual))
+        )
+
+    def files_diff(self):
+        return '\n'.join([self.file_diff(n, e, a) for n, e, a in
+                          zip(self.file_names,
+                              self.expected.files_content,
+                              self.actual.files_content)])
+
+    def output_diff(self) -> str:
+        return (
+            self.indicator("STATUS: TODO", "| ") + '\n'
+            + self.expected_header + '\n'
+            + self.cat_e(self.expected.output)
+            + self.actual_header + '\n'
+            + self.cat_e(self.actual.output)
+        )
+
+    def full_diff(self) -> str:
+        return (self.indicator("WITH {}".format(self.cmd), "|>") + '\n'
+                + self.output_diff()
+                + self.files_diff()
+                + "=" * 80 + '\n')
+
+    def cat_e(self, s: str) -> str:
+        ret = "$\n".join(s.split('\n'))
+        if len(ret) < 2:
+            return ret
+        if ret[-1] != '\n':
+            ret += '\n'
+        return ret
+
 
 class Test:
     def __init__(self, cmd: str, setup: str = "", files: [str] = [], exports: {str: str} = {}):
@@ -34,23 +159,20 @@ class Test:
         self.setup = setup
         self.files = files
         self.exports = exports
+        self.result = None
 
     def run(self):
-        self.expected = self._run_sandboxed(config.REFERENCE_PATH)
-        self.actual   = self._run_sandboxed(config.MINISHELL_PATH)
+        expected = self._run_sandboxed(config.REFERENCE_PATH)
+        actual   = self._run_sandboxed(config.MINISHELL_PATH)
+        self.result = Result(self.cmd, self.files, expected, actual)
+        self.result.put()
 
-        self._put_result()
+    def _run_sandboxed(self, shell_path: str) -> Captured:
+        """ run the command in a sandbox environment
 
-        # if not verbose:
-        #     put_result(passed, cmd)
-        # if verbose:
-        #     if not passed:
-        #         print(diff(cmd, expected, actual, files, expected_files, actual_files, color=True))
-        #     else:
-        # self._put_line_result(passed)
-
-    def _run_sandboxed(self, shell_path: str) -> Result:
-        """ run the command in a sandbox environment, return the output (stdout and stderr) of it """
+            capture the output (stdout and stderr)
+            capture the content of the watched files after the command is run
+        """
 
         try:
             os.mkdir(config.SANDBOX_PATH)
@@ -58,10 +180,7 @@ class Test:
             pass
         if self.setup != "":
             try:
-                setup_status = subprocess.run(self.setup,
-                                              shell=True,
-                                              cwd=config.SANDBOX_PATH,
-                                              check=True)
+                setup_status = subprocess.run(self.setup, shell=True, cwd=config.SANDBOX_PATH, check=True)
             except subprocess.CalledProcessError as e:
                 print("Error: `{}` setup command failed for `{}`\n\twith '{}'"
                       .format(setup, cmd, e.stderr.decode().strip()))
@@ -76,6 +195,7 @@ class Test:
                                         env={'PATH': config.PATH_VARIABLE, **self.exports})
         output = process_status.stdout.decode()
 
+        # capture watched files content
         files_content = []
         for file_name in self.files:
             try:
@@ -83,23 +203,5 @@ class Test:
                     files_content.append(f.read().decode())
             except FileNotFoundError as e:
                 files_content.append(None)
-
         shutil.rmtree(config.SANDBOX_PATH)
-        return Result(output, files_content)
-
-    def _put_result(self):
-        passed = self.actual == self.expected
-        if config.VERBOSE_LEVEL == 0:
-            sys.stdout.write(utils.green('.') if passed else utils.red('!'))
-            sys.stdout.flush()
-        elif config.VERBOSE_LEVEL == 1:
-            printed = self.cmd
-            if len(printed) > 70:
-                printed = printed[:67] + "..."
-            fmt = utils.green("{:74} [PASS]") if passed else utils.red("{:74} [FAIL]")
-            print(fmt.format(printed))
-        elif config.VERBOSE_LEVEL == 2:
-            pass
-            # print(diff(cmd, expected, actual, files, expected_files, actual_files, color=True))
-        else:
-            raise RuntimeError
+        return Captured(output, files_content)
diff --git a/utils.py b/utils.py
deleted file mode 100644
index 1a6cde9..0000000
--- a/utils.py
+++ /dev/null
@@ -1,175 +0,0 @@
-import os
-import sys
-import subprocess
-import shutil
-
-import config
-
-COLOR_RED = "\033[32m"
-COLOR_GREEN = "\033[31m"
-COLOR_BLUE = "\033[34m"
-COLOR_CLOSE = "\033[0m"
-
-BOLD = "\033[1m"
-
-def green(s: str) -> str:
-    return COLOR_RED + s + COLOR_CLOSE
-
-def red(s: str) -> str:
-    return COLOR_GREEN + s + COLOR_CLOSE
-
-def expected_line(color: bool) -> str:
-    s = "|---------------------------------------EXPECTED--------------------------------"
-    return BOLD + COLOR_GREEN + s + COLOR_CLOSE if color else s
-
-def actual_line(color: bool) -> str:
-    s = "|---------------------------------------ACTUAL----------------------------------"
-    return BOLD + COLOR_RED + s + COLOR_CLOSE if color else s
-
-def file_line(file_name, color: bool) -> str:
-    s = "|# FILE " + file_name
-    return BOLD + COLOR_BLUE + s + COLOR_CLOSE if color else s
-
-def status_line(status, color: bool) -> str:
-    s = "|> STATUS: " + status
-    return BOLD + COLOR_BLUE + s + COLOR_CLOSE if color else s
-
-
-
-def diff_file(file_name: str, expected: str, actual: str, color: bool = False) -> str:
-    return """\
-{}
-{}
-{}\
-{}
-{}\
-""".format(file_line(file_name, color), expected_line(color), expected, actual_line(color),
-           "FROM TEST: File not created\n" if actual is None else actual)
-
-def diff_output(expected: str, actual: str, color: bool = False) -> str:
-    return """\
-{}
-{}
-{}\
-{}
-{}\
-""".format(status_line("TODO", color), expected_line(color), expected, actual_line(color), actual)
-
-def diff(cmd: str, expected: str, actual: str,
-        files: [str], expected_files: [str], actual_files: [str],
-        color: bool = False) -> str:
-    s = ""
-    if color:
-        s = BOLD + COLOR_BLUE + "|> WITH " + cmd + COLOR_CLOSE + "\n"
-    else:
-        s = "|> WITH " + cmd + "\n"
-    if expected != actual:
-        s += diff_output(expected, actual, color)
-
-    strs = []
-    for file_name, e, a in zip(files, expected_files, actual_files):
-        if a != e:
-            tmp = ""
-            if expected != actual:
-                tmp += "-" * 80 + "\n"
-            tmp += diff_file(file_name, e, a, color)
-            strs.append(tmp)
-    s += ("-" * 80 + "\n").join(strs)
-    return s
-
-
-def put_result(passed: bool, cmd: str):
-    if len(cmd) > 70:
-        cmd = cmd[:67] + "..."
-
-    if passed:
-        print(green("{:74} [PASS]".format(cmd)))
-    else:
-        print(red("{:74} [FAIL]".format(cmd)))
-
-
-# def run_sandboxed(program: str, cmd: str, setup: str = None, files: [str] = [], exports: {str, str} = {}) -> str:
-#     """ run the command in a sandbox environment, return the output (stdout and stderr) of it """
-#
-#     try:
-#         os.mkdir(config.SANDBOX_PATH)
-#     except OSError:
-#         pass
-#     if setup is not None:
-#         try:
-#             setup_status = subprocess.run(setup, shell=True, cwd=config.SANDBOX_PATH, check=True)
-#         except subprocess.CalledProcessError as e:
-#             print("Error: `{}` setup command failed for `{}`\n\twith '{}'"
-#                   .format(setup, cmd, e.stderr.decode().strip()))
-#             sys.exit(1)
-#
-#     # TODO: add timeout
-#     # https://docs.python.org/3/library/subprocess.html#using-the-subprocess-module
-#     process_status = subprocess.run([program, "-c", cmd],
-#                                     stderr=subprocess.STDOUT,
-#                                     stdout=subprocess.PIPE,
-#                                     cwd=config.SANDBOX_PATH,
-#                                     env={'PATH': config.PATH_VARIABLE, **exports})
-#     output = process_status.stdout.decode()
-#
-#     output_files = []
-#     for file_name in files:
-#         try:
-#             with open(os.path.join(config.SANDBOX_PATH, file_name), "rb") as f:
-#                 output_files.append(f.read().decode())
-#         except FileNotFoundError as e:
-#             output_files.append(None)
-#
-#     shutil.rmtree(config.SANDBOX_PATH)
-#     return (output, output_files)
-#
-# status = 0
-# ignored_suites = []
-# runned_suites = {}
-# current_suite = "default"
-# verbose = False
-#
-# def check(expected: str, actual: str, expected_files: [str], actual_files: [str]) -> bool:
-#     return actual == expected and all([a == e for a, e in zip(actual_files, expected_files)])
-#
-# def test(cmd: str, setup: str = None, files: [str] = [], exports: {str, str} = {}):
-#     """ get expected and actual strings, compare them and push them to the suites result """
-#
-#     (expected, expected_files) = run_sandboxed(config.REFERENCE_SHELL_PATH, cmd, setup, files, exports)
-#     (actual, actual_files) = run_sandboxed(config.MINISHELL_PATH, cmd, setup, files, exports)
-#
-#     passed = check(expected, actual, expected_files, actual_files)
-#     global status
-#     if passed:
-#         status = 1
-#
-#     if not verbose:
-#         put_result(passed, cmd)
-#     if verbose:
-#         if not passed:
-#             print(diff(cmd, expected, actual, files, expected_files, actual_files, color=True))
-#         else:
-#             put_result(passed, cmd)
-#
-#     if runned_suites.get(current_suite) is None:
-#         runned_suites[current_suite] = []
-#     runned_suites[current_suite].append((cmd, expected, actual, files, expected_files, actual_files))
-#
-# available_suites = []
-#
-# def suite(origin):
-#     """ decorator for a suite function (fmt: suite_[name])
-#         update the current_suite global and print it before the suite execution
-#     """
-#
-#     name = origin.__name__[len("suite_"):]
-#     available_suites.append(name)
-#     def f():
-#         if name in ignored_suites:
-#             return
-#         global current_suite
-#         current_suite = name.upper()
-#         print("{} {:#<41}".format("#" * 39, current_suite + " "))
-#         origin()
-#         print()
-#     return f