diff --git a/test/ets-stdlib-excluded.txt b/test/ets-stdlib-excluded.txt new file mode 100644 index 0000000000000000000000000000000000000000..477b2617224b6755c1d21cccba9f320653e7c062 --- /dev/null +++ b/test/ets-stdlib-excluded.txt @@ -0,0 +1,3 @@ +tests/stdlib/std/math/sqrt-negative-01.ets +tests/stdlib/std/math/sqrt-positive-03.ets +tests/stdlib/std/math/sqrt-positive-04.ets \ No newline at end of file diff --git a/test/runner/params.py b/test/runner/params.py index df994ed45dfbbb1e809c7fc9ac9d0a1981636953..5e5a16748e26bb79214ae3d3a4e0813c1bd01eb7 100644 --- a/test/runner/params.py +++ b/test/runner/params.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, List from configuration_kind import ConfigurationKind @@ -8,22 +8,21 @@ from fail_kind import FailKind @dataclass class TestEnv: args: Any - - conf_kind: ConfigurationKind - - cmd_prefix: List[str] cmd_env: Any + conf_kind: ConfigurationKind + cmd_prefix: List[str] = field(default_factory=list) + + es2panda: str = "" + es2panda_args : List[str] = field(default_factory=list) - es2panda: str - - runtime: str - runtime_args: List[str] + runtime: str = "" + runtime_args: List[str] = field(default_factory=list) - arkaout: str - aot_args: List[str] + arkaout: str = "" + aot_args: List[str] = field(default_factory=list) - ark_quick: str - quick_args: List[str] + ark_quick: str = "" + quick_args: List[str] = field(default_factory=list) util: Any = None diff --git a/test/runner/runner.py b/test/runner/runner.py index 607e8cb259d7b94e607f7d7a262b8843ab18255c..995ced502111b5e486cd01a10c4f02917fd9f228 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -4,6 +4,7 @@ from dotenv import load_dotenv from runner_js_hermes import RunnerJSHermes from runner_js_parser import RunnerJSParser from runner_js_test262 import RunnerJSTest262 +from runner_ets_stdlib import RunnerETS_STDLIB from starter import get_args @@ -27,6 +28,9 @@ def main(): if args.hermes: runners.append(RunnerJSHermes(args)) + if args.ets_stdlib: + runners.append(RunnerETS_STDLIB(args)) + failed_tests = 0 for runner in runners: diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 506e7c46ac7eaf72b9c4a4f6622aa69b5a542034..0aadc492aef39a203740a07a0a82b88e0cb03088 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -180,6 +180,7 @@ class Runner: ) ))) + def create_test(self, test_file, flags, is_ignored): pass diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py new file mode 100644 index 0000000000000000000000000000000000000000..0934ae127abbc3c99e8a19cd91fb3668a840e99c --- /dev/null +++ b/test/runner/runner_ets.py @@ -0,0 +1,31 @@ +from os import path +from pathlib import Path + +from runner_file_based import RunnerFileBased + + +class RunnerETS(RunnerFileBased): + def __init__(self, args, name: str): + RunnerFileBased.__init__(self, args, name) + self.stdlib_path = path.join(self.test_root, "stdlib") + self.stdlib_name = "etsstdlib.abc" + self.stdlib_path = path.join(self.build_dir, "etsstdlib.abc") + self.stdlib_src_path = path.join(self.test_root, "stdlib") + self._check_binary_artefacts() + + self.test_env.es2panda_args = [f"--stdlib={self.stdlib_src_path}", + "--gen-stdlib=false", + "--extension=ets", + "--opt-level=0"] + self.test_env.runtime_args = [f"--boot-panda-files={self.stdlib_path}", + "--load-runtimes=ets"] + + def _check_binary_artefacts(self): + stdlib_path_obj = Path(self.stdlib_path) + stdlib_src_path_obj = Path(self.stdlib_src_path) + + if not stdlib_path_obj.is_file(): + raise FileNotFoundError("Hmm ... standard library abc file not found") + + if not stdlib_src_path_obj.is_dir(): + raise FileNotFoundError("Hmm ... Source code of standard library was not found") diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py new file mode 100644 index 0000000000000000000000000000000000000000..07cd0111925eebffc9d385255aad825c029493c1 --- /dev/null +++ b/test/runner/runner_ets_stdlib.py @@ -0,0 +1,21 @@ +import os + +from runner_base import get_test_id +from runner_ets import RunnerETS +from test_ets import TestETS + + +class RunnerETS_STDLIB(RunnerETS): + def __init__(self, args): + RunnerETS.__init__(self, args, "ets-stdlib") + self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") + + self.collect_excluded_test_lists() + self.collect_ignored_test_lists() + + self.add_directory(self.stdlib_test_path, "ets", []) + + def create_test(self, test_file, flags, is_ignored): + test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) + test.ignored = is_ignored + return test diff --git a/test/runner/runner_js.py b/test/runner/runner_file_based.py similarity index 97% rename from test/runner/runner_js.py rename to test/runner/runner_file_based.py index 6d18d102fcff18ec9e284f03c32b5759447c1da4..d2c61ffcef4ce6331c681087ae2e85ef2d557060 100644 --- a/test/runner/runner_js.py +++ b/test/runner/runner_file_based.py @@ -24,7 +24,7 @@ INDEX_TEST_NAME = "${TestName}" INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" -class RunnerJS(Runner): +class RunnerFileBased(Runner): def __init__(self, args, name): Runner.__init__(self, args, name) self.cmd_env = environ.copy() @@ -50,7 +50,11 @@ class RunnerJS(Runner): # we don't want to interpret asan failures as SyntaxErrors self.cmd_env[san] = ":exitcode=255" - self.es2panda = path.join(self.build_dir, "bin", "es2panda") + if args.custom_es2panda_path: + self.es2panda = args.custom_es2panda_path + else: + self.es2panda = path.join(self.build_dir, "bin", "es2panda") + if not path.isfile(self.es2panda): raise Exception(f"Cannot find es2panda binary: {self.es2panda}") @@ -113,6 +117,7 @@ class RunnerJS(Runner): cmd_prefix=self.cmd_prefix, cmd_env=self.cmd_env, es2panda=self.es2panda, + es2panda_args=[], runtime=self.runtime, runtime_args=self.runtime_args, arkaout=self.arkaot, diff --git a/test/runner/runner_js_hermes.py b/test/runner/runner_js_hermes.py index 79cf36021cc9df70ccbad6336c7d216c8e78be7d..9c20648b9dd99e78745f0cb7bde8833b80858d20 100644 --- a/test/runner/runner_js_hermes.py +++ b/test/runner/runner_js_hermes.py @@ -1,12 +1,12 @@ from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_hermes import TestJSHermes from util_hermes import UtilHermes -class RunnerJSHermes(RunnerJS): +class RunnerJSHermes(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "hermes") + RunnerFileBased.__init__(self, args, "hermes") self.collect_excluded_test_lists() self.collect_ignored_test_lists() diff --git a/test/runner/runner_js_parser.py b/test/runner/runner_js_parser.py index 2a134867c091804abb6c883bdff5ce08782d7f54..d10fe8befaf6e869635fd20d66a4c2bbfdc88dcf 100644 --- a/test/runner/runner_js_parser.py +++ b/test/runner/runner_js_parser.py @@ -1,11 +1,11 @@ from os import path from runner_base import get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_parser import TestJSParser -class RunnerJSParser(RunnerJS): +class RunnerJSParser(RunnerFileBased): def __init__(self, args): super(RunnerJSParser, self).__init__(args, "parser-js") diff --git a/test/runner/runner_js_test262.py b/test/runner/runner_js_test262.py index 21a8807233c9f6e6b2b5c99fce3e99375d3d406e..d16af1f0741a2e3944db271a7121e46b7f33dd7d 100644 --- a/test/runner/runner_js_test262.py +++ b/test/runner/runner_js_test262.py @@ -1,14 +1,14 @@ from os import path from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_test262 import TestJSTest262 from util_test262 import UtilTest262 -class RunnerJSTest262(RunnerJS): +class RunnerJSTest262(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "test262-ark") + RunnerFileBased.__init__(self, args, "test262-ark") self.ignored_name_prefix = "test262" self.collect_excluded_test_lists(test_name=self.ignored_name_prefix) diff --git a/test/runner/starter.py b/test/runner/starter.py index a1f1792f1637e4c3a3060aea093b0b10793104f2..736db9aaaec6a89df7b1a120994b8db45f51a905 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -21,7 +21,8 @@ def is_file(parser, arg): def check_timeout(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError(f"{value} is an invalid timeout value") + raise argparse.ArgumentTypeError( + f"{value} is an invalid timeout value") return ivalue @@ -43,7 +44,13 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - + parser.add_argument( + '--ets-stdlib', action='store_true', dest='ets_stdlib', + default=False, help='run test against ETS STDLIB') + parser.add_argument( + '--custom-es2panda', action='store', dest='custom_es2panda_path', + type=lambda arg: is_file(parser, arg), + default=False, help='custom path to es2panda binary file') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), help='directory with test file. If not set the module directory is used') diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py new file mode 100644 index 0000000000000000000000000000000000000000..474c8fef704e01095fc0e78d99cd10e0981733f8 --- /dev/null +++ b/test/runner/test_ets.py @@ -0,0 +1,77 @@ +from os import path, makedirs +from typing import Tuple + +from fail_kind import FailKind +from params import Params, TestReport +from configuration_kind import ConfigurationKind +from test_js import TestJS + + +class TestETS(TestJS): + def __init__(self, test_env, test_path, flags, test_id, update_expected=False): + TestJS.__init__(self, test_env, test_path, flags, test_id, update_expected) + # If test fails it contains reason (of FailKind enum) of first failed step + # It's supposed if the first step is failed then no step is executed further + self.fail_kind = None + self.main_entry_point = "ETSGLOBAL::main" + self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") + makedirs(self.bytecode_path, exist_ok=True) + test_basename = path.basename(self.path) + self.is_negative_test = test_basename.startswith("n.") + self.test_an = path.join(self.bytecode_path, f"{self.test_id}.an") + + def do_run(self): + self.passed, self.report, compiled_file, self.fail_kind = self._run_compiler() + if not self.passed: + return self + + # Run aot if required + if self.test_env.conf_kind in [ConfigurationKind.AOT, ConfigurationKind.AOT_FULL]: + self.passed, self.report, self.fail_kind = self.run_aot( + self.test_an, + compiled_file, + lambda o, e, rc: rc == 0 + ) + + if not self.passed: + return self + + self.passed, self.report, self.fail_kind = self.run_runtime(self.test_an, compiled_file, lambda _, _2, rc : self._runtime_result_validator(rc) ) + return self + + def _runtime_result_validator(self, return_code: int) -> bool: + """ + :return: True if test is successful, False if failed + """ + if self.is_negative_test: + return return_code != 0 + else: + return return_code == 0 + + def _run_compiler(self) -> Tuple[bool, TestReport, str, FailKind]: + test_basename = path.basename(self.path) + test_abc = f"{test_basename}.abc" + output_path = path.join(self.bytecode_path, test_abc) + + es2panda_flags = [] + es2panda_flags.extend(self.test_env.es2panda_args) + es2panda_flags.append(f"--output={output_path}") + es2panda_flags.append(self.path) + + params = Params( + executor=self.test_env.es2panda, + flags=es2panda_flags, + env=self.test_env.cmd_env, + timeout=self.test_env.args.es2panda_timeout, + fail_kind_fail=FailKind.ES2PANDA_FAIL, + fail_kind_timeout=FailKind.ES2PANDA_TIMEOUT, + fail_kind_other=FailKind.ES2PANDA_OTHER, + ) + + passed, report, fail_kind = self.run_one_step( + name="es2panda", + params=params, + result_validator=lambda _, _2, rc: rc == 0 + ) + + return (passed, report, output_path, fail_kind) diff --git a/test/runner/test_js.py b/test/runner/test_js.py index 589ec13ca5108a9b454f785428b5b791515763ed..fb05a128b99d9ab56ae138abb2e0c13f96715a32 100644 --- a/test/runner/test_js.py +++ b/test/runner/test_js.py @@ -15,6 +15,7 @@ class TestJS(Test): # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None + self.main_entry_point = "_GLOBAL::func_main_0" def run_one_step(self, name, params: Params, result_validator): cmd = self.test_env.cmd_prefix + [params.executor] @@ -91,7 +92,7 @@ class TestJS(Test): if self.test_env.conf_kind == ConfigurationKind.IRTOC: ark_flags.extend(['--interpreter-type=irtoc']) - ark_flags.extend([test_abc, "_GLOBAL::func_main_0"]) + ark_flags.extend([test_abc, self.main_entry_point]) params = Params( timeout=self.test_env.args.timeout,