From 7cbaf473ca385cd64978a2d6f25f2df6af76bdb9 Mon Sep 17 00:00:00 2001 From: Charles Cabergs Date: Tue, 2 Mar 2021 14:33:51 +0100 Subject: Refactoring test.result.Result --- docs/conf.py | 3 + docs/developers.rst | 12 ++ minishell_test/colors.py | 39 ++++++ minishell_test/config.py | 2 +- minishell_test/suite/suite.py | 14 +- minishell_test/test/captured.py | 48 ++++--- minishell_test/test/result.py | 301 ++++++++++++++++++++-------------------- minishell_test/test/test.py | 32 ++--- tests/test/test_captured.py | 15 +- tests/test/test_result.py | 82 +++++++++++ 10 files changed, 339 insertions(+), 209 deletions(-) create mode 100644 minishell_test/colors.py create mode 100644 tests/test/test_result.py diff --git a/docs/conf.py b/docs/conf.py index d60ba56..41202d8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,6 +34,7 @@ release = '1.0.1' # ones. extensions = [ "sphinx.ext.extlinks", + "sphinx.ext.autodoc", "sphinxcontrib.programoutput", ] @@ -65,6 +66,8 @@ html_context = { "github_version": "master", } +autodoc_typehints = "description" + # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". diff --git a/docs/developers.rst b/docs/developers.rst index 024f100..1ce1150 100644 --- a/docs/developers.rst +++ b/docs/developers.rst @@ -3,6 +3,18 @@ Developers ========== +.. autoclass:: minishell_test.test.captured.CapturedCommand + :members: + :special-members: __init__, __eq__ + +.. autoclass:: minishell_test.test.captured.CapturedTimeout + :members: + :special-members: __eq__ + +.. autoclass:: minishell_test.test.result.BaseResult + :members: + :special-members: __init__ + Install requirements -------------------- diff --git a/minishell_test/colors.py b/minishell_test/colors.py new file mode 100644 index 0000000..7b6cf45 --- /dev/null +++ b/minishell_test/colors.py @@ -0,0 +1,39 @@ +_DEFAULTS = { + "red": "\033[31m", + "green": "\033[32m", + "blue": "\033[34m", + "bold": "\033[1m", + "close": "\033[0m", +} + +_COLORS = { + "red": _DEFAULTS["red"], + "green": _DEFAULTS["green"], + "blue": _DEFAULTS["blue"], + "bold": _DEFAULTS["bold"], + "close": _DEFAULTS["close"], +} + + +def disable() -> None: + _COLORS["red"] = "" + _COLORS["green"] = "" + _COLORS["blue"] = "" + _COLORS["bold"] = "" + _COLORS["close"] = "" + + +def green(s: str) -> str: + return _COLORS["green"] + s + _COLORS["close"] + + +def red(s: str) -> str: + return _COLORS["red"] + s + _COLORS["close"] + + +def blue(s: str) -> str: + return _COLORS["blue"] + s + _COLORS["close"] + + +def bold(s: str) -> str: + return _COLORS["bold"] + s + _COLORS["close"] diff --git a/minishell_test/config.py b/minishell_test/config.py index 00d147c..2af1348 100644 --- a/minishell_test/config.py +++ b/minishell_test/config.py @@ -6,7 +6,7 @@ # By: cacharle +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2021/02/26 09:40:36 by cacharle #+# #+# # -# Updated: 2021/02/28 11:19:13 by cacharle ### ########.fr # +# Updated: 2021/03/01 19:30:33 by cacharle ### ########.fr # # # # ############################################################################ # diff --git a/minishell_test/suite/suite.py b/minishell_test/suite/suite.py index b1aba4f..b058569 100644 --- a/minishell_test/suite/suite.py +++ b/minishell_test/suite/suite.py @@ -6,7 +6,7 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/07/15 18:24:29 by charles #+# #+# # -# Updated: 2021/02/27 12:07:59 by cacharle ### ########.fr # +# Updated: 2021/03/02 11:12:35 by cacharle ### ########.fr # # # # ############################################################################ # @@ -145,12 +145,12 @@ class Suite: self.CLOSE_CHARS, width=Config.term_cols )) - for i, t in enumerate(self.tests): - if Config.range is not None: - if not (Config.range[0] <= i <= Config.range[1]): - continue - t.run(i) - if Config.exit_first and t.result is not None and t.result.failed: + if Config.range is not None: + self.tests = self.test[Config.range[0] : Config.range[1] + 1] + for i, test in enumerate(self.tests): + result = test.run() + print(result.to_string(i)) + if Config.exit_first and result is not None and result.failed: return False return True diff --git a/minishell_test/test/captured.py b/minishell_test/test/captured.py index 6481dcf..bb09579 100644 --- a/minishell_test/test/captured.py +++ b/minishell_test/test/captured.py @@ -6,43 +6,49 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/09/11 12:16:25 by charles #+# #+# # -# Updated: 2021/03/01 16:15:31 by cacharle ### ########.fr # +# Updated: 2021/03/02 10:32:19 by cacharle ### ########.fr # # # # ############################################################################ # -from typing import List, Optional +from typing import List, Optional, Union -class Captured: +class CapturedCommand: def __init__( self, output: str, status: int, files_content: List[Optional[str]], - is_timeout: bool = False ): - """Captured class - output: captured content - status: command status - files_content: content of the files altered by the command - is_timeout: the command has timed out + """Captured command + + :param output: + Command output + :param status: + Command return status code + :param files_content: + Content of the files altered by the command """ self.output = output self.status = status self.files_content = files_content - self.is_timeout = is_timeout def __eq__(self, other: object) -> bool: - if not isinstance(other, Captured): + if not isinstance(other, CapturedCommand): return False - if self.is_timeout: - return self.is_timeout == other.is_timeout - return (self.output == other.output and - self.status == other.status and - all(x == y for x, y in zip(self.files_content, other.files_content))) - - @staticmethod - def timeout(): - """Create a new captured timeout""" - return Captured("", 0, [], is_timeout=True) + return ( + self.output == other.output and + self.status == other.status and + all(x == y for x, y in zip(self.files_content, other.files_content)) + ) + + +class CapturedTimeout(): + """Captured timeout""" + + def __eq__(self, other: object) -> bool: + return isinstance(other, CapturedTimeout) + + +CapturedType = Union[CapturedCommand, CapturedTimeout] diff --git a/minishell_test/test/result.py b/minishell_test/test/result.py index 93c576a..566520a 100644 --- a/minishell_test/test/result.py +++ b/minishell_test/test/result.py @@ -6,64 +6,64 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/09/11 12:17:34 by charles #+# #+# # -# Updated: 2021/02/27 12:28:20 by cacharle ### ########.fr # +# Updated: 2021/03/02 14:17:01 by cacharle ### ########.fr # # # # ############################################################################ # import re -from typing import Match, List, Optional +from typing import Match, List, Optional, Union from minishell_test.config import Config -from minishell_test.test.captured import Captured +from minishell_test.colors import green, red, blue, bold +from minishell_test.test.captured import CapturedCommand, CapturedTimeout, CapturedType class BaseResult: - RED_CHARS = "\033[31m" - GREEN_CHARS = "\033[32m" - BLUE_CHARS = "\033[34m" - BOLD_CHARS = "\033[1m" - CLOSE_CHARS = "\033[0m" - def __init__(self, cmd: str): - self.cmd = cmd - self.colored = True - self.set_colors() + """ + :param cmd: + The command executed to get to the result + """ + self._raw_cmd = cmd @property - def passed(self): + def passed(self) -> bool: """Check if the result passed""" raise NotImplementedError @property - def failed(self): + def failed(self) -> bool: """Check if the result failed""" return not self.passed - def __repr__(self): - """Returns a representation of the result based on the verbosity""" - printed = self._escaped_cmd[:] - if Config.show_range: - printed = "{:2}: ".format(self.index) + printed - if len(printed) > Config.term_cols - 7: - printed = printed[:Config.term_cols - 10] + "..." - fmt = self.green("{:{width}} [PASS]") if self.passed else self.red("{:{width}} [FAIL]") - return fmt.format(printed, width=Config.term_cols - 7) - - def put(self, index: int) -> None: - """Print a summary of the result""" - self.index = index - print(self) + @property + def __repr__(self) -> str: + raise NotImplementedError - def indicator(self, title: str, prefix: str) -> str: - return self.bold(self.blue(prefix + " " + title)) + def summarize(self, index: int) -> str: + """Summary of the result - def full_diff(self) -> str: - raise NotImplementedError + :param index: + The test index to print when :option:`--show-range` is enabled + :returns: + A summary of the result on one line, + it's length is the width of the terminal. + """ + printed = self._cmd[:] + if Config.show_range: + printed = f"{index:2}: {printed}" + width = Config.term_cols - len(" [PASS]") + if len(printed) > width: + printed = printed[:width - 3] + "..." + if self.passed: + return green(f"{printed:{width}} [PASS]") + else: + return red(f"{printed:{width}} [FAIL]") @property - def _escaped_cmd(self): - """Escape common control characters""" - c = self.cmd + def _cmd(self) -> str: + """The result command with the common control characters escaped""" + c = self._raw_cmd c = c.replace("\t", "\\t") c = c.replace("\n", "\\n") c = c.replace("\v", "\\v") @@ -72,75 +72,109 @@ class BaseResult: return c @property - def _header_with(self): - return self.indicator("WITH {}".format(self._escaped_cmd), "|>") + '\n' - - def set_colors(self): - """Set colors strings on or off based on self.colored""" - if self.colored: - self.color_red = self.RED_CHARS - self.color_green = self.GREEN_CHARS - self.color_blue = self.BLUE_CHARS - self.color_bold = self.BOLD_CHARS - self.color_close = self.CLOSE_CHARS - else: - self.color_red = "" - self.color_green = "" - self.color_blue = "" - self.color_bold = "" - self.color_close = "" - - def green(self, s): - return self.color_green + s + self.color_close - - def red(self, s): - return self.color_red + s + self.color_close + def _cmd_header(self): + return f"|> WITH {self._cmd}\n" - def blue(self, s): - return self.color_blue + s + self.color_close - - def bold(self, s): - return self.color_bold + s + self.color_close class Result(BaseResult): def __init__( self, - cmd: str, + cmd: str, file_names: List[str], - expected: Captured, - actual: Captured, + expected: CapturedType, + actual: CapturedType, ): """Result class - cmd: runned command - file_names: names of watched files - expected: expected capture - actual: actual capture + + :param cmd: + runned command + :param file_names: + names of watched files + :param expected: + expected capture + :param actual: + actual capture """ - self.file_names = file_names - self.expected = expected - self.actual = actual super().__init__(cmd) + self.file_names = file_names + self.expected = expected + self.actual = actual @property def passed(self): return self.actual == self.expected - def header(self, title: str) -> str: - return self.bold("|---------------------------------------{:-<40}".format(title)) + def __repr__(self) -> str: + """Concat all difference reports""" + return ( + self._cmd_header + + self._cmd_diff() + + self._files_diff() + + "=" * 80 + '\n' + ) + + def _cmd_diff(self) -> str: + """Difference in command output""" + if isinstance(self.actual, CapturedTimeout): + return "TIMEOUT\n" + out = "" + if self.expected.status != self.actual.status: + out = f"| STATUS: expected {self.expected.status} actual {self.actual.status}\n" + if self.expected.output != self.actual.output: + out += self._content_diff(self.expected.output, self.actual.output) + return out + + _FILE_NOT_CREATED_MESSAGE = "FROM TEST: File not created\n" + + def _files_diff(self): + """Difference between watched files""" + + if isinstance(self.actual, CapturedTimeout): + return "" + + def diff(name, expected, actual): + expected = expected or self._FILE_NOT_CREATED_MESSAGE + actual = actual or self._FILE_NOT_CREATED_MESSAGE + return f"|# FILE {file_name}\n" + self._content_diff(expected, actual) + + return '\n'.join([ + diff(name, expected, actual) + for name, expected, actual in + zip( + self.file_names, + self.expected.files_content, + self.actual.files_content + ) + if expected != actual + ]) + + def _content_diff(self, expected: str, actual: str) -> str: + return ( + self._expected_header + + self._show_newlines(expected) + + self._actual_header + + self._show_newlines(actual) + ) + + def _header(self, title: str) -> str: + """Create a 80 characters wide header in the format ``-- title --``""" + return f"|{'-' * 40}{title:-<40}\n" @property - def expected_header(self) -> str: - return self.green(self.header("EXPECTED")) + '\n' + def _expected_header(self) -> str: + return self._header("EXPECTED") @property - def actual_header(self) -> str: - return self.red(self.header("ACTUAL")) + '\n' + def _actual_header(self) -> str: + return self._header("ACTUAL") - def cat_e(self, s: Optional[str]) -> str: - """Pass a string through a cat -e like output""" - if s is None: - return "FROM TEST: File not created\n" + def _show_newlines(self, s: str) -> str: + """Add a ``$`` at the end of each newline + + If the string doesn't end with a newline add one but doesn't add a + ``$`` to represent it. + """ s = s.replace("\n", "$\n") if len(s) < 2: return s @@ -148,85 +182,48 @@ class Result(BaseResult): s += '\n' return s - def file_diff(self, file_name: str, expected: Optional[str], actual: Optional[str]) -> str: - """Difference between 2 files""" - if expected == actual: - return "" - file_header = self.indicator("FILE {}".format(file_name), "|#") + '\n' - return ( - file_header + - self.expected_header + - self.cat_e(expected) + - self.actual_header + - self.cat_e(actual) - ) - def files_diff(self): - """Difference between watched files""" - return '\n'.join([self.file_diff(n, e, a) for n, e, a in - zip(self.file_names, - self.expected.files_content, - self.actual.files_content) - if e != a]) +class LeakResultException(Exception): + def __init__(self, result: 'LeakResult'): + self._result = result - def output_diff(self) -> str: - """Difference in command output""" - out = "" - if self.actual.is_timeout: - return "TIMEOUT\n" - if self.expected.status != self.actual.status: - out += self.indicator( - "STATUS: expected {} actual {}" - .format(self.expected.status, self.actual.status), "| " - ) + '\n' - if self.expected.output != self.actual.output: - out += (self.expected_header + - self.cat_e(self.expected.output) + - self.actual_header + - self.cat_e(self.actual.output)) - return out - - def full_diff(self) -> str: - """Concat all difference reports""" - return self._header_with + self.output_diff() + self.files_diff() + "=" * 80 + '\n' + def __str__(self) -> str: + return f"valgrind output parsing failed for `{self._result._cmd}`:\n{self._result._captured.output}" class LeakResult(BaseResult): - def __init__(self, cmd: str, captured: Captured): - self.captured = captured + def __init__(self, cmd: str, captured: CapturedType): + self._captured = captured super().__init__(cmd) + def __repr__(self) -> str: + return self._cmd_header + self.captured.output + + @property + def passed(self) -> str: + if isinstance(self._captured, CapturedTimeout): + return False + return self._lost_bytes == 0 + + _VALGRIND_OK_MESSAGE = "All heap blocks were freed -- no leaks are possible" + + @property + def _lost_bytes(self): + # Some versions of valgrind don't output `definitely` and `indirectly` + # when no leaks are found. + if self._captured.output.find(self._VALGRIND_OK_MESSAGE) != -1: + return 0 + definite_match = self._search_leak_kind("definitely") + indirect_match = self._search_leak_kind("indirectly") + definite_bytes = int(definite_match.group("bytes").replace(",", "")) + indirect_bytes = int(indirect_match.group("bytes").replace(",", "")) + return definite_bytes + indirect_bytes + def _search_leak_kind(self, kind: str) -> Match: match = re.search( r"==\d+==\s+" + kind + r" lost: (?P[0-9,]+) bytes in [0-9,]+ blocks", - self.captured.output + self._captured.output ) if match is None: - raise RuntimeError( - "valgrind output parsing failed for `{}`:\n{}" - .format(self.cmd, self.captured.output) - ) + raise LeakResultException(self) return match - - @property - def _lost_bytes(self): - if self.captured.output.find("All heap blocks were freed -- no leaks are possible") != -1: - definite_bytes = 0 - indirect_bytes = 0 - else: - definite_match = self._search_leak_kind("definitely") - indirect_match = self._search_leak_kind("indirectly") - definite_bytes = int(definite_match.group("bytes").replace(",", "")) - indirect_bytes = int(indirect_match.group("bytes").replace(",", "")) - return definite_bytes + indirect_bytes - - @property - def passed(self): - """Check if the result passed""" - if self.captured.is_timeout: - return False - return self._lost_bytes == 0 - - def full_diff(self) -> str: - """Concat all difference reports""" - return self._header_with + self.captured.output diff --git a/minishell_test/test/test.py b/minishell_test/test/test.py index f45b8b4..372f9f2 100644 --- a/minishell_test/test/test.py +++ b/minishell_test/test/test.py @@ -6,7 +6,7 @@ # By: charles +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2020/06/16 21:48:50 by charles #+# #+# # -# Updated: 2021/03/01 16:02:35 by cacharle ### ########.fr # +# Updated: 2021/03/02 11:10:28 by cacharle ### ########.fr # # # # ############################################################################ # @@ -18,7 +18,7 @@ from pathlib import Path from typing import Optional, List, Dict, Union, Callable from minishell_test.config import Config -from minishell_test.test.captured import Captured +from minishell_test.test.captured import CapturedCommand, CapturedTimeout, CapturedType from minishell_test.test.result import Result, LeakResult from minishell_test import sandbox @@ -50,7 +50,6 @@ class Test: self.setup = setup self.files = files self.exports = exports - self.result: Optional[Union[Result, LeakResult]] = None self.timeout = timeout if timeout > 0 else Config.timeout_test if not isinstance(hook, list): hook = [hook] @@ -59,23 +58,20 @@ class Test: self.hook = hook self.hook_status = hook_status - def run(self, index: int) -> None: + def run(self) -> Union[Result, LeakResult]: """ Run the test for minishell and the reference shell and print the result out """ if Config.check_leaks: self.hook = [] self.hook_status = [] captured = self._run_sandboxed([*Config.valgrind_cmd, "-c"]) - self.result = LeakResult(self.full_cmd, captured) - self.result.put(index) - return + return LeakResult(self.full_cmd, captured) expected = self._run_sandboxed([Config.shell_reference_path, *Config.shell_reference_args, "-c"]) actual = self._run_sandboxed([Config.minishell_exec_path, "-c"]) - self.result = Result(self.full_cmd, self.files, expected, actual) - self.result.put(index) + return Result(self.full_cmd, self.files, expected, actual) - def _run_sandboxed(self, shell_cmd: List[Union[str, Path]]) -> Captured: + def _run_sandboxed(self, shell_cmd: List[Union[str, Path]]) -> CapturedType: """ Run the command in a sandbox environment """ with sandbox.context(): if self.setup != "": @@ -97,7 +93,7 @@ class Test: sys.exit(1) return self._run_capture(shell_cmd) - def _run_capture(self, shell_cmd: List[Union[str, Path]]) -> Captured: + def _run_capture(self, shell_cmd: List[Union[str, Path]]) -> CapturedType: """ Capture the output (stdout and stderr) Capture the content of the watched files after the command is run """ @@ -121,7 +117,7 @@ class Test: except subprocess.TimeoutExpired: process.kill() # _, _ = process.communicate(timeout=2) - return Captured.timeout() + return CapturedTimeout() try: output = stdout.decode() except UnicodeDecodeError: @@ -150,7 +146,7 @@ class Test: lines[i] = Config.minishell_prefix + line[len(Config.shell_reference_prefix):] output = '\n'.join(lines) - return Captured(output, process.returncode, files_content) + return CapturedCommand(output, process.returncode, files_content) @property def full_cmd(self) -> str: @@ -166,10 +162,10 @@ class Test: @classmethod def try_run(cls, cmd: str) -> str: test = Test(cmd) - test.run(0) - if isinstance(test.result, LeakResult): - return test.result.captured.output - elif isinstance(test.result, Result): - return test.result.actual.output + result = test.run(0) + if isinstance(result, LeakResult): + return result.captured.output + elif isinstance(result, Result): + return result.actual.output else: return "No output" diff --git a/tests/test/test_captured.py b/tests/test/test_captured.py index 2c085c6..3d4beca 100644 --- a/tests/test/test_captured.py +++ b/tests/test/test_captured.py @@ -6,26 +6,25 @@ # By: cacharle +#+ +:+ +#+ # # +#+#+#+#+#+ +#+ # # Created: 2021/03/01 15:55:11 by cacharle #+# #+# # -# Updated: 2021/03/01 16:19:23 by cacharle ### ########.fr # +# Updated: 2021/03/02 10:14:23 by cacharle ### ########.fr # # # # ############################################################################ # import pytest import copy -from minishell_test.test.captured import Captured +from minishell_test.test.captured import CapturedCommand, CapturedTimeout @pytest.fixture def captured(): - return Captured("foo", 0, ["file1", "file2"], False) + return CapturedCommand("foo", 0, ["file1", "file2"]) def test_init(captured): assert "foo" == captured.output assert 0 == captured.status assert ["file1", "file2"] == captured.files_content - assert not captured.is_timeout def test_eq(captured): @@ -42,9 +41,5 @@ def test_eq(captured): c2 = copy.deepcopy(captured) c2.files_content = ["asdfasdf"] assert captured != c2 - assert captured != Captured.timeout() - assert Captured.timeout() == Captured.timeout() - - -def test_timeout(): - assert Captured.timeout().is_timeout + assert captured != CapturedTimeout() + assert CapturedTimeout() == CapturedTimeout() diff --git a/tests/test/test_result.py b/tests/test/test_result.py new file mode 100644 index 0000000..35c6213 --- /dev/null +++ b/tests/test/test_result.py @@ -0,0 +1,82 @@ +# ############################################################################ # +# # +# ::: :::::::: # +# test_result.py :+: :+: :+: # +# +:+ +:+ +:+ # +# By: cacharle +#+ +:+ +#+ # +# +#+#+#+#+#+ +#+ # +# Created: 2021/03/01 16:26:34 by cacharle #+# #+# # +# Updated: 2021/03/02 14:21:14 by cacharle ### ########.fr # +# # +# ############################################################################ # + +import pytest + +from minishell_test.config import Config +from minishell_test import colors + +colors.disable() +Config.init([]) + +from minishell_test.test.result import BaseResult, Result, LeakResult +from minishell_test.test.captured import CapturedCommand + + +class TestBaseResult: + @pytest.fixture + def base_result(self): + return BaseResult("echo bonjour") + + def test_passed(self, base_result): + with pytest.raises(NotImplementedError): + base_result.passed + + def test_failed(self, base_result): + with pytest.raises(NotImplementedError): + base_result.failed + + def test_repr(self, base_result): + with pytest.raises(NotImplementedError): + base_result.__repr__() + + def test_cmd(self, base_result): + assert "echo bonjour" == base_result._cmd + assert "foo\\nbar" == BaseResult("foo\nbar")._cmd + assert "foo\\tbar" == BaseResult("foo\tbar")._cmd + assert "foo\\vbar" == BaseResult("foo\vbar")._cmd + assert "foo\\rbar" == BaseResult("foo\rbar")._cmd + assert "foo\\fbar" == BaseResult("foo\fbar")._cmd + + def test_summarize(self, base_result): + pass + + + + +class TestResult: + @pytest.fixture + def result_pass(self): + return Result( + "echo bonjour", + [], + CapturedCommand("bonjour", 0, []), + CapturedCommand("bonjour", 0, []), + ) + + @pytest.fixture + def result_fail(self): + return Result( + "echo bonjour", + [], + CapturedCommand("bonjour", 0, []), + CapturedCommand("aurevoir", 0, []), + ) + + def test_passed(self, result_pass, result_fail): + assert result_pass.passed + assert not result_fail.passed + + def test_failed(self, result_pass, result_fail): + assert not result_pass.failed + assert result_fail.failed + -- cgit