Added memory leak checking

10 files changed, 176 insertions, 84 deletions

diff --git a/README.md b/README.md
index 5c173ae..27b3644 100644
--- a/README.md
+++ b/README.md
@@ -11,24 +11,28 @@ The default path to your project is `..` but you can change it the the [configur
 ```sh
 $ ./run  # run all tests
 
-$ ./run --help
-usage: run [-h] [-v] [-b] [-n] [-l] [-m] [-p] [suite [suite ...]]
+$❯ ./run -h
+usage: run [-h] [-k] [-x] [-v] [-b] [-n] [-l] [-m] [-p] [suite [suite ...]]
 
 Minishell test
 
 positional arguments:
-  suite           Test suites/group to run.
-                  It tries to be smart and autocomplete the suite name
-                  (e.g ./run int -> ./run preprocess/interpolation)
+  suite              Test suites/group to run. It tries to be smart and
+                     autocomplete the suite name (e.g ./run int -> ./run
+                     preprocess/interpolation)
 
 optional arguments:
-  -h, --help      show this help message and exit
-  -v, --verbose   Increase verbosity level (e.g -vv == 2)
-  -b, --bonus     Enable bonus tests
-  -n, --no-bonus  Disable bonus tests
-  -l, --list      Print available test suites
-  -m, --make      Make minishell and exit
-  -p, --pager     After running the test, display the result in a pager of your choice
+  -h, --help         show this help message and exit
+  -k, --check-leaks  Run valgrind on tests (disable usual comparison with
+                     bash)
+  -x, --exit-first   Exit on first fail
+  -v, --verbose      Increase verbosity level (e.g -vv == 2)
+  -b, --bonus        Enable bonus tests
+  -n, --no-bonus     Disable bonus tests
+  -l, --list         Print available test suites
+  -m, --make         Make minishell and exit
+  -p, --pager        After running the test, display the result in a pager of
+                     your choice ./run --help
 ```
 
 ## Test compatibility
@@ -60,6 +64,10 @@ Their is 3 different method to enable the bonus tests:
 * Set the environment variable `MINISHELL_TEST_BONUS` to `yes`  
   (e.g `echo 'export MINISHELL_TEST_BONUS=yes' >> ~/.zshrc`)
 
+## Memory leaks
+
+`./run -kx`
+
 ## Linux
 
 The tester will try to convert to output/status code of bash on Linux to the one on Mac.
diff --git a/src/args.py b/src/args.py
index 9110073..b606285 100644
--- a/src/args.py
+++ b/src/args.py
@@ -6,7 +6,7 @@
 #    By: charles <charles.cabergs@gmail.com>        +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/07/15 18:24:32 by charles           #+#    #+#              #
-#    Updated: 2020/09/27 11:08:49 by charles          ###   ########.fr        #
+#    Updated: 2020/10/07 18:21:33 by cacharle         ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
@@ -18,6 +18,14 @@ def parse_args():
 
     parser = argparse.ArgumentParser(description="Minishell test")
     parser.add_argument(
+        "-k", "--check-leaks", action="store_true",
+        help="Run valgrind on tests (disable usual comparison with bash)"
+    )
+    parser.add_argument(
+        "-x", "--exit-first", action="store_true",
+        help="Exit on first fail"
+    )
+    parser.add_argument(
         "-v", "--verbose", action="count",
         help="Increase verbosity level (e.g -vv == 2)"
     )
diff --git a/src/config.py b/src/config.py
index e77a94d..566ffe8 100644
--- a/src/config.py
+++ b/src/config.py
@@ -6,7 +6,7 @@
 #    By: charles <charles.cabergs@gmail.com>        +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/07/15 18:24:19 by charles           #+#    #+#              #
-#    Updated: 2020/10/07 08:07:25 by charles          ###   ########.fr        #
+#    Updated: 2020/10/07 18:21:46 by cacharle         ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
@@ -16,6 +16,7 @@
 
 import os
 import shutil
+import distutils.spawn
 
 # run the bonus tests
 # can be changed with `export MINISHELL_TEST_BONUS=yes` in your shell rc file.
@@ -59,6 +60,7 @@ PATH_VARIABLE = os.path.abspath(EXECUTABLES_PATH)
 # default test timeout
 TIMEOUT = 0.5
 
+
 LOREM = """
 Mollitia asperiores assumenda excepturi et ipsa. Nihil corporis facere aut a rem consequatur.
 Quas molestiae corporis et quibusdam maiores. Molestiae sed unde aut at sed.
@@ -77,13 +79,23 @@ Perspiciatis ut maxime et libero quo voluptas consequatur illum. Pariatur porro
 LOREM = ' '.join(LOREM.split('\n'))
 
 ###############################################################################
-# do not edit
+# You probably shouldn't edit after                                           #
 ###############################################################################
 
 MINISHELL_PATH = os.path.abspath(
     os.path.join(MINISHELL_DIR, MINISHELL_EXEC)
 )
 
+VALGRIND_CMD = [
+    distutils.spawn.find_executable("valgrind"),
+    # "valgrind",
+    "--trace-children=no",
+    "--leak-check=yes",
+    "--child-silent-after-fork=yes",
+    "--show-leak-kinds=definite",
+    MINISHELL_PATH,
+]
+
 # 0, 1, 2
 VERBOSE_LEVEL = 1
 
@@ -95,3 +107,7 @@ if TERM_COLS < 40:
     raise RuntimeError("You're terminal isn't wide enough")
 
 PLATFORM = os.uname().sysname
+
+EXIT_FIRST = False
+
+CHECK_LEAKS = False
diff --git a/src/hooks.py b/src/hooks.py
index 8439c0a..2d55bbb 100644
--- a/src/hooks.py
+++ b/src/hooks.py
@@ -6,12 +6,11 @@
 #    By: charles <me@cacharle.xyz>                  +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/09/11 16:10:20 by charles           #+#    #+#              #
-#    Updated: 2020/10/07 08:27:49 by charles          ###   ########.fr        #
+#    Updated: 2020/10/07 15:18:13 by cacharle         ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
 import re
-import os
 import sys
 
 import config
diff --git a/src/main.py b/src/main.py
index e9d83f4..204abee 100755
--- a/src/main.py
+++ b/src/main.py
@@ -65,6 +65,9 @@ def main():
         config.BONUS = True
     if args.no_bonus:
         config.BONUS = False
+    config.EXIT_FIRST = args.exit_first
+    config.CHECK_LEAKS = args.check_leaks
+
     Suite.setup(args.suites)
     try:
         Suite.run_all()
diff --git a/src/suite/suite.py b/src/suite/suite.py
index 61f1cbc..dc5611a 100644
--- a/src/suite/suite.py
+++ b/src/suite/suite.py
@@ -6,15 +6,29 @@
 #    By: charles <charles.cabergs@gmail.com>        +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/07/15 18:24:29 by charles           #+#    #+#              #
-#    Updated: 2020/10/06 17:10:43 by cacharle         ###   ########.fr        #
+#    Updated: 2020/10/07 18:24:20 by cacharle         ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
 import sys
+import tty
+import termios
 
 import config
 
 
+# # from: https://stackoverflow.com/questions/510357
+# def getchar():
+#     fd = sys.stdin.fileno()
+#     old_settings = termios.tcgetattr(fd)
+#     try:
+#         tty.setraw(sys.stdin.fileno())
+#         char = sys.stdin.read(1)
+#     finally:
+#         termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+#     return char
+
+
 class Suite:
     available = []
 
@@ -22,7 +36,8 @@ class Suite:
     def run_all(cls):
         """Run all available suites"""
         for s in cls.available:
-            s.run()
+            if not s.run() and config.EXIT_FIRST:
+                break
 
     @classmethod
     def setup(cls, asked_names: [str]):
@@ -39,28 +54,28 @@ class Suite:
                 names.append(name)
                 continue
             matches = [n for n in suite_names
-                       if n.find("/") != -1
-                       and n[n.find("/") + 1:].startswith(name)
-                       or n.startswith(name)]
+                    if n.find("/") != -1
+                    and n[n.find("/") + 1:].startswith(name)
+                    or n.startswith(name)]
             if len(matches) == 1:
                 names.append(matches[0])
             elif len(matches) != 0 and all([n.startswith(name) for n in matches]):
                 names.extend(matches)
             elif len(matches) > 2:
                 print(("Ambiguous name `{}` match the following suites\n\t{}\n"
-                      "Try to run with -l to see the available suites")
-                      .format(name, ', '.join(matches)))
+                    "Try to run with -l to see the available suites")
+                    .format(name, ', '.join(matches)))
                 sys.exit(1)
             elif len(matches) == 0:
                 print(("Name `{}` doesn't match any suite/group name\n\t"
-                      "Try to run with -l to see the available suites")
-                      .format(name))
+                    "Try to run with -l to see the available suites")
+                    .format(name))
                 sys.exit(1)
 
         cls.available = list(set(
             [s for s in cls.available if s.name in names]
             + [s for s in cls.available if any([g for g in s.groups if g in names])]
-        ))
+            ))
         cls.available.sort(key=lambda s: s.name)
         for s in cls.available:
             s.generator_func()
@@ -89,7 +104,7 @@ class Suite:
     BLUE_CHARS  = "\033[34m"
     CLOSE_CHARS = "\033[0m"
 
-    def run(self):
+    def run(self) -> bool:
         """Run all test in the suite"""
         if config.VERBOSE_LEVEL == 0:
             print(self.name + ": ", end="")
@@ -99,11 +114,14 @@ class Suite:
                 " " + self.name + " ",
                 self.CLOSE_CHARS,
                 width=config.TERM_COLS
-            ))
-        for t in self.tests:
-            t.run()
+                ))
+            for t in self.tests:
+                t.run()
+                if config.EXIT_FIRST and t.result.failed:
+                    return False
         if config.VERBOSE_LEVEL == 0:
             print()
+        return True
 
     def total(self) -> (int, int):
         """Returns the total of passed and failed tests"""
@@ -128,9 +146,9 @@ class Suite:
             pass_sum += pass_total
             fail_sum += fail_total
             print("{:.<{width}} \033[32m{:3} [PASS]\033[0m \033[31m{:3} [FAIL]\033[0m"
-                  .format(s.name + " ", pass_total, fail_total, width=config.TERM_COLS - 22))
+                    .format(s.name + " ", pass_total, fail_total, width=config.TERM_COLS - 22))
         print("{:.<{width}} \033[32m{:3} [PASS]\033[0m \033[31m{:3} [FAIL]\033[0m"
-              .format("TOTAL ", pass_sum, fail_sum, width=config.TERM_COLS - 22))
+                .format("TOTAL ", pass_sum, fail_sum, width=config.TERM_COLS - 22))
 
     @classmethod
     def save_log(cls):
diff --git a/src/suites/path.py b/src/suites/path.py
index 3bf38c6..d5dabb6 100644
--- a/src/suites/path.py
+++ b/src/suites/path.py
@@ -6,7 +6,7 @@
 #    By: charles <me@cacharle.xyz>                  +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/09/09 15:12:58 by charles           #+#    #+#              #
-#    Updated: 2020/10/07 11:12:56 by cacharle         ###   ########.fr        #
+#    Updated: 2020/10/07 15:16:49 by cacharle         ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
@@ -65,7 +65,8 @@ def suite_path(test):
     test("a", setup=mode_fmt.format("6777"), exports={"PATH": "path"})
     test("a", setup=mode_fmt.format("7777"), exports={"PATH": "path"})
     test("a", setup=mode_fmt.format("0000"), exports={"PATH": "path"})
-    test("b", setup="mkdir path && cp " + whoami_path + " ./path/a && ln -s ./path/a ./path/b", exports={"PATH": "path"})
+    test("b", setup="mkdir path && cp " + whoami_path + " ./path/a && ln -s ./path/a ./path/b",
+         exports={"PATH": "path"})
     test("b", setup="mkdir path && ln -s " + whoami_path + " ./path/b", exports={"PATH": "path"})
     test("a", setup="mkdir path && mkfifo path/a")
     test("a", setup="mkdir path && mkfifo path/a && chmod 777 path/a")
@@ -102,7 +103,7 @@ def suite_path_variable(test):
     test("whoami", exports={"PATH": "/usr/bin:/usr/bin:/usr/bin:/usr/bin"})
     test("whoami", exports={"PATH": "     /sbin      "})
     test("whoami", exports={"PATH": "/sbin:/sbin:/sbin:/sbin"})
-    test("whoami", exports={"PATH": ""})
+    test("whoami", exports={"PATH": ""})  # error message explicit enough
     test("whoami", exports={"PATH": ":"})
     test("whoami", exports={"PATH": ":::::::::::::::::::"})
     test("whoami", exports={"PATH": "/asdfasdf"})
diff --git a/src/test/captured.py b/src/test/captured.py
index f855212..4a9966d 100644
--- a/src/test/captured.py
+++ b/src/test/captured.py
@@ -6,7 +6,7 @@
 #    By: charles <me@cacharle.xyz>                  +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/09/11 12:16:25 by charles           #+#    #+#              #
-#    Updated: 2020/09/11 20:42:05 by charles          ###   ########.fr        #
+#    Updated: 2020/10/07 18:25:05 by cacharle         ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
diff --git a/src/test/result.py b/src/test/result.py
index 5e7c2e9..30ce31e 100644
--- a/src/test/result.py
+++ b/src/test/result.py
@@ -6,11 +6,12 @@
 #    By: charles <me@cacharle.xyz>                  +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/09/11 12:17:34 by charles           #+#    #+#              #
-#    Updated: 2020/10/06 16:56:30 by cacharle         ###   ########.fr        #
+#    Updated: 2020/10/07 18:53:27 by cacharle         ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
 import sys
+import re
 
 import config
 from test.captured import Captured
@@ -23,50 +24,51 @@ class Result:
     BOLD_CHARS  = "\033[1m"
     CLOSE_CHARS = "\033[0m"
 
-    def __init__(self, cmd: str, file_names: [str], expected: Captured, actual: Captured):
+    def __init__(
+        self,
+        cmd: str,
+        file_names: [str],
+        expected: Captured,
+        actual: Captured,
+        leak_output: str = None
+    ):
         """Result class
-           cmd:        runned command
-           file_names: names of watched files
-           expected:   expected capture
-           actual:     actual capture
+           cmd:         runned command
+           file_names:  names of watched files
+           expected:    expected capture
+           actual:      actual capture
+           leak_output: output of valgrind in check leak mode
         """
         self.cmd = cmd
         self.file_names = file_names
         self.expected = expected
         self.actual = actual
         self.colored = True
+        self.leak_output = leak_output
         self.set_colors()
 
-    def set_colors(self):
-        """Set colors strings on or off based on self.colored"""
-        if self.colored:
-            self.color_red   = self.RED_CHARS
-            self.color_green = self.GREEN_CHARS
-            self.color_blue  = self.BLUE_CHARS
-            self.color_bold  = self.BOLD_CHARS
-            self.color_close = self.CLOSE_CHARS
-        else:
-            self.color_red   = ""
-            self.color_green = ""
-            self.color_blue  = ""
-            self.color_bold  = ""
-            self.color_close = ""
+    @staticmethod
+    def leak(cmd: str, leak_output: str = None):
+        return Result(cmd, None, None, None, leak_output)
 
-    def green(self, s):
-        return self.color_green + s + self.color_close
-
-    def red(self, s):
-        return self.color_red + s + self.color_close
-
-    def blue(self, s):
-        return self.color_blue + s + self.color_close
-
-    def bold(self, s):
-        return self.color_bold + s + self.color_close
+    @property
+    def lost_bytes(self):
+        m = re.search(
+            r"definitely lost: (?P<bytes>[0-9,]+) bytes in [0-9,]+ blocks",
+            self.leak_output
+        )
+        if m is None:
+            raise RuntimeError(
+                "valgrind output parsing failed for `{}`:\n{}"
+                .format(self.cmd, self.leak_output)
+            )
+        return int(m.group("bytes"))
 
     @property
     def passed(self):
         """Check if the result passed"""
+        if self.leak_output is not None:
+            return self.lost_bytes == 0
         return self.actual == self.expected
 
     @property
@@ -153,10 +155,12 @@ class Result:
 
     def full_diff(self) -> str:
         """Concat all difference reports"""
-        return (self.indicator("WITH {}".format(self.escaped_cmd), "|>") + '\n'
-                + self.output_diff()
-                + self.files_diff()
-                + "=" * 80 + '\n')
+        rest = ""
+        if self.leak_output is not None:
+            rest = self.leak_output
+        else:
+            rest = (self.output_diff() + self.files_diff() + "=" * 80 + '\n')
+        return (self.indicator("WITH {}".format(self.escaped_cmd), "|>") + '\n' + rest)
 
     def cat_e(self, s: str) -> str:
         """Pass a string through a cat -e like output"""
@@ -176,3 +180,30 @@ class Result:
                 .replace("\v", "\\v")
                 .replace("\r", "\\r")
                 .replace("\f", "\\f"))
+
+    def set_colors(self):
+        """Set colors strings on or off based on self.colored"""
+        if self.colored:
+            self.color_red   = self.RED_CHARS
+            self.color_green = self.GREEN_CHARS
+            self.color_blue  = self.BLUE_CHARS
+            self.color_bold  = self.BOLD_CHARS
+            self.color_close = self.CLOSE_CHARS
+        else:
+            self.color_red   = ""
+            self.color_green = ""
+            self.color_blue  = ""
+            self.color_bold  = ""
+            self.color_close = ""
+
+    def green(self, s):
+        return self.color_green + s + self.color_close
+
+    def red(self, s):
+        return self.color_red + s + self.color_close
+
+    def blue(self, s):
+        return self.color_blue + s + self.color_close
+
+    def bold(self, s):
+        return self.color_bold + s + self.color_close
diff --git a/src/test/test.py b/src/test/test.py
index c4e183c..52b6db3 100644
--- a/src/test/test.py
+++ b/src/test/test.py
@@ -6,14 +6,14 @@
 #    By: charles <charles.cabergs@gmail.com>        +#+  +:+       +#+         #
 #                                                 +#+#+#+#+#+   +#+            #
 #    Created: 2020/06/16 21:48:50 by charles           #+#    #+#              #
-#    Updated: 2020/10/07 08:12:00 by charles          ###   ########.fr        #
+#    Updated: 2020/10/07 18:54:13 by cacharle         ###   ########.fr        #
 #                                                                              #
 # ############################################################################ #
 
 import os
 import sys
 import subprocess
-# import time
+#import time
 
 import config
 from test.captured import Captured
@@ -57,8 +57,17 @@ class Test:
 
     def run(self):
         """ Run the test for minishell and the reference shell and print the result out """
-        expected = self._run_sandboxed(config.REFERENCE_PATH, config.REFERENCE_ARGS + ["-c"])
-        actual   = self._run_sandboxed(config.MINISHELL_PATH, ["-c"])
+
+        if config.CHECK_LEAKS:
+            self.hook = []
+            self.hook_status = []
+            captured = self._run_sandboxed([*config.VALGRIND_CMD, "-c"])
+            self.result = Result.leak(self.cmd, captured.output)
+            self.result.put()
+            return
+
+        expected = self._run_sandboxed([config.REFERENCE_PATH, *config.REFERENCE_ARGS, "-c"])
+        actual   = self._run_sandboxed([config.MINISHELL_PATH, "-c"])
         s = self.cmd
         if self.setup != "":
             s = "[SETUP {}] {}".format(self.setup, s)
@@ -68,7 +77,7 @@ class Test:
         self.result = Result(s, self.files, expected, actual)
         self.result.put()
 
-    def _run_sandboxed(self, shell_path: str, shell_options: str) -> Captured:
+    def _run_sandboxed(self, shell_cmd: [str]) -> Captured:
         """ Run the command in a sandbox environment """
         with sandbox.context():
             if self.setup != "":
@@ -88,15 +97,15 @@ class Test:
                                   "no stderr" if e.stdout is None
                                   else e.stdout.decode().strip()))
                     sys.exit(1)
-            return self._run_capture(shell_path, shell_options)
+            return self._run_capture(shell_cmd)
 
-    def _run_capture(self, shell_path: str, shell_options: str) -> Captured:
+    def _run_capture(self, shell_cmd: [str]) -> Captured:
         """ Capture the output (stdout and stderr)
             Capture the content of the watched files after the command is run
         """
         # run the command in the sandbox
         process = subprocess.Popen(
-            [shell_path, *shell_options, self.cmd],
+            [*shell_cmd, self.cmd],
             stderr=subprocess.STDOUT,
             stdout=subprocess.PIPE,
             cwd=config.SANDBOX_PATH,
@@ -107,13 +116,12 @@ class Test:
             },
         )
         # if self.signal is not None:
-        #     time.sleep(0.2)
+        #     time.sleep(0.1)
         #     process.send_signal(self.signal)
-        # else:
 
         # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate
         try:
-            stdout, _ = process.communicate(timeout=self.timeout)
+            stdout, _ = process.communicate(timeout=(self.timeout if not config.CHECK_LEAKS else 10))
         except subprocess.TimeoutExpired:
             process.kill()
             # _, _ = process.communicate(timeout=2)
@@ -132,7 +140,7 @@ class Test:
             except FileNotFoundError:
                 files_content.append(None)
 
-        # sandbox.remove()
+        # apply output/status hooks
         for h in self.hook:
             output = h(output)
         for h in self.hook_status: