From dc5a704c49fa9c854b61037d3a5f4c9296d93427 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 8 Apr 2019 06:04:35 +0200 Subject: [PATCH 01/19] fix: nonzero return values should produce a fail --- vunit/test_suites.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 295e7dcde..994960412 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -172,9 +172,12 @@ def run(self, output_path, read_output): results = self._read_test_results(file_name=get_result_file_name(output_path)) # Do not run post check unless all passed - for status in results.values(): - if status != PASSED: - return results + for name in results: + if not sim_ok: + results[name] = FAILED + if results[name] == PASSED: + continue + return results if not self._config.call_post_check(output_path, read_output): for name in self._test_cases: From 496eb19726b2ca03289a8d4bf2fed36b2d69158a Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 8 Apr 2019 22:09:28 +0200 Subject: [PATCH 02/19] apply fix only with VHDL 2008 --- vunit/test_suites.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 994960412..8c5fde59b 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -173,7 +173,7 @@ def run(self, output_path, read_output): # Do not run post check unless all passed for name in results: - if not sim_ok: + if self._simulator_if._vhdl_standard == "2008" and not sim_ok: results[name] = FAILED if results[name] == PASSED: continue From bed5a28ebb270d8c4db9b9cfe694735785fac2e2 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 8 Apr 2019 22:55:54 +0200 Subject: [PATCH 03/19] fix lint --- vunit/ghdl_interface.py | 6 ++++++ vunit/test_suites.py | 15 +++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/vunit/ghdl_interface.py b/vunit/ghdl_interface.py index 18e8bae5f..39942559c 100644 --- a/vunit/ghdl_interface.py +++ b/vunit/ghdl_interface.py @@ -93,6 +93,12 @@ def __init__(self, # pylint: disable=too-many-arguments self._backend = backend self._vhdl_standard = None + def get_vhdl_standard(self): + """ + Return VHDL standard version + """ + return self._vhdl_standard + @classmethod def determine_backend(cls, prefix): """ diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 8c5fde59b..999b1e2ac 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -171,13 +171,16 @@ def run(self, output_path, read_output): results = self._read_test_results(file_name=get_result_file_name(output_path)) + try: + if self._simulator_if.get_vhdl_standard() == "2008" and not sim_ok: + return dict(map((lambda name: (name, FAILED)), results)) + except: + pass + # Do not run post check unless all passed - for name in results: - if self._simulator_if._vhdl_standard == "2008" and not sim_ok: - results[name] = FAILED - if results[name] == PASSED: - continue - return results + for status in results.values(): + if status != PASSED: + return results if not self._config.call_post_check(output_path, read_output): for name in self._test_cases: From 839d5608f2b36bb364f1b588c91ed5ccf0cb5d95 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 8 Apr 2019 23:57:59 +0200 Subject: [PATCH 04/19] fix lint (do not use try/except) --- vunit/test_suites.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 999b1e2ac..5d8402ce2 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -171,11 +171,10 @@ def run(self, output_path, read_output): results = self._read_test_results(file_name=get_result_file_name(output_path)) - try: - if self._simulator_if.get_vhdl_standard() == "2008" and not sim_ok: - return dict(map((lambda name: (name, FAILED)), results)) - except: - pass + if hasattr(self._simulator_if, 'get_vhdl_standard') and \ + self._simulator_if.get_vhdl_standard() == "2008" and \ + not sim_ok: + return dict(map((lambda name: (name, FAILED)), results)) # Do not run post check unless all passed for status in results.values(): From 05423531654b520e015ee6512c839b107497d5ad Mon Sep 17 00:00:00 2001 From: umarcor Date: Tue, 9 Apr 2019 00:04:35 +0200 Subject: [PATCH 05/19] fix lint (replace map on lamba) --- vunit/test_suites.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 5d8402ce2..173bd91f9 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -174,7 +174,7 @@ def run(self, output_path, read_output): if hasattr(self._simulator_if, 'get_vhdl_standard') and \ self._simulator_if.get_vhdl_standard() == "2008" and \ not sim_ok: - return dict(map((lambda name: (name, FAILED)), results)) + return dict((name, FAILED) for name in results) # Do not run post check unless all passed for status in results.values(): From ecbcc636f2d5bc789487288d292efa8a444a7d95 Mon Sep 17 00:00:00 2001 From: umarcor Date: Tue, 9 Apr 2019 02:37:22 +0200 Subject: [PATCH 06/19] fix acceptance tests: must preserve 'skipped' results --- vunit/test_suites.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 173bd91f9..9b203f9f0 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -174,7 +174,7 @@ def run(self, output_path, read_output): if hasattr(self._simulator_if, 'get_vhdl_standard') and \ self._simulator_if.get_vhdl_standard() == "2008" and \ not sim_ok: - return dict((name, FAILED) for name in results) + return dict((name, FAILED) if name is PASSED else (name, results[name]) for name in results) # Do not run post check unless all passed for status in results.values(): From 93ac20663fbe484b56c9162e9c7b24fa8fa85150 Mon Sep 17 00:00:00 2001 From: umarcor Date: Wed, 10 Apr 2019 04:19:39 +0200 Subject: [PATCH 07/19] add has_valid_exit_code method to class simulator_if --- vunit/ghdl_interface.py | 6 +++--- vunit/simulator_interface.py | 6 ++++++ vunit/test_suites.py | 6 ++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/vunit/ghdl_interface.py b/vunit/ghdl_interface.py index 39942559c..dd31d6831 100644 --- a/vunit/ghdl_interface.py +++ b/vunit/ghdl_interface.py @@ -93,11 +93,11 @@ def __init__(self, # pylint: disable=too-many-arguments self._backend = backend self._vhdl_standard = None - def get_vhdl_standard(self): + def has_valid_exit_code(self): """ - Return VHDL standard version + Return if the simulation should fail with nonzero exit codes """ - return self._vhdl_standard + return self._vhdl_standard == "2008" @classmethod def determine_backend(cls, prefix): diff --git a/vunit/simulator_interface.py b/vunit/simulator_interface.py index 96fe99452..96a7f02ff 100644 --- a/vunit/simulator_interface.py +++ b/vunit/simulator_interface.py @@ -138,6 +138,12 @@ def supports_vhdl_package_generics(cls): """ return False + def has_valid_exit_code(self): + """ + Return if the simulation should fail with nonzero exit codes + """ + return False + def merge_coverage(self, file_name, args): # pylint: disable=unused-argument, no-self-use """ Hook for simulator interface to creating coverage reports diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 9b203f9f0..c521159ed 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -171,10 +171,8 @@ def run(self, output_path, read_output): results = self._read_test_results(file_name=get_result_file_name(output_path)) - if hasattr(self._simulator_if, 'get_vhdl_standard') and \ - self._simulator_if.get_vhdl_standard() == "2008" and \ - not sim_ok: - return dict((name, FAILED) if name is PASSED else (name, results[name]) for name in results) + if self._simulator_if.has_valid_exit_code() and not sim_ok: + return dict((name, FAILED) if results[name] is PASSED else (name, results[name]) for name in results) # Do not run post check unless all passed for status in results.values(): From 8744341b9841e62372acdbcd2c3d5db642768b0a Mon Sep 17 00:00:00 2001 From: umarcor Date: Wed, 10 Apr 2019 04:52:51 +0200 Subject: [PATCH 08/19] use function instead of method --- vunit/simulator_interface.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vunit/simulator_interface.py b/vunit/simulator_interface.py index 96a7f02ff..0ca303955 100644 --- a/vunit/simulator_interface.py +++ b/vunit/simulator_interface.py @@ -138,7 +138,8 @@ def supports_vhdl_package_generics(cls): """ return False - def has_valid_exit_code(self): + @staticmethod + def has_valid_exit_code(): """ Return if the simulation should fail with nonzero exit codes """ From 470d2da3903dcc5c0856f33fc347a4e621a21473 Mon Sep 17 00:00:00 2001 From: umarcor Date: Wed, 10 Apr 2019 16:58:26 +0200 Subject: [PATCH 09/19] force fail only if all tests pass in the presence of non-zero exit code --- vunit/test_suites.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/vunit/test_suites.py b/vunit/test_suites.py index c521159ed..01675752c 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -171,14 +171,15 @@ def run(self, output_path, read_output): results = self._read_test_results(file_name=get_result_file_name(output_path)) - if self._simulator_if.has_valid_exit_code() and not sim_ok: - return dict((name, FAILED) if results[name] is PASSED else (name, results[name]) for name in results) - # Do not run post check unless all passed for status in results.values(): if status != PASSED: return results + # Force fail if all tests pass in the presence of non-zero exit code + if self._simulator_if.has_valid_exit_code() and not sim_ok: + return dict((name, FAILED) if results[name] is PASSED else (name, results[name]) for name in results) + if not self._config.call_post_check(output_path, read_output): for name in self._test_cases: results[name] = FAILED From 07ea721ce009c75ca83084837f64fec0f9fcc27f Mon Sep 17 00:00:00 2001 From: umarcor Date: Wed, 10 Apr 2019 20:48:23 +0200 Subject: [PATCH 10/19] fix skipped --- vunit/test_suites.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 01675752c..1bc7d8ffb 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -171,15 +171,20 @@ def run(self, output_path, read_output): results = self._read_test_results(file_name=get_result_file_name(output_path)) - # Do not run post check unless all passed + # If any test failed, return results for status in results.values(): - if status != PASSED: + if status == FAILED: return results # Force fail if all tests pass in the presence of non-zero exit code if self._simulator_if.has_valid_exit_code() and not sim_ok: return dict((name, FAILED) if results[name] is PASSED else (name, results[name]) for name in results) + # Do not run post check unless all passed + for status in results.values(): + if status != PASSED: + return results + if not self._config.call_post_check(output_path, read_output): for name in self._test_cases: results[name] = FAILED From dcc63a03ddf88f72ddb94c4274e100e0271278ff Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 11 Apr 2019 05:54:31 +0200 Subject: [PATCH 11/19] move exit code evaluation to new func '_check_results'; add tests in test_test_suites --- vunit/test/unit/test_test_suites.py | 36 +++++++++++++++++++++++++++++ vunit/test_suites.py | 26 ++++++++++++++------- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/vunit/test/unit/test_test_suites.py b/vunit/test/unit/test_test_suites.py index 592fa34ba..59773c16c 100644 --- a/vunit/test/unit/test_test_suites.py +++ b/vunit/test/unit/test_test_suites.py @@ -13,6 +13,7 @@ from vunit.test_suites import (TestRun) from vunit.test_report import (PASSED, SKIPPED, FAILED) from vunit.test.common import create_tempdir +from vunit.simulator_interface import SimulatorInterface class TestTestSuites(TestCase): @@ -107,3 +108,38 @@ def _read_test_results(contents, expected_test_cases): test_suite_name=None, test_cases=expected_test_cases) return run._read_test_results(file_name=file_name) # pylint: disable=protected-access + + def test_exit_code(self): + self.assertEqual(self._test_exit_code(True), False) + self.assertEqual(self._test_exit_code(False), False) + self.assertEqual(self._test_exit_code(True, True), False) + self.assertEqual(self._test_exit_code(False, True), True) + + @staticmethod + def _test_exit_code(sim_ok=True, with_has_valid_func=False): + """ + Helper method to test the check_results function + """ + with create_tempdir() as path: + file_name = join(path, "vunit_results") + with open(file_name, "w") as fptr: + fptr.write("""\ +test_start:test1 +test_suite_done +""") + sim_if = SimulatorInterface + if with_has_valid_func: + @staticmethod + def func(): + return True + sim_if.has_valid_exit_code = func + + run = TestRun(simulator_if=sim_if, + config=None, + elaborate_only=False, + test_suite_name=None, + test_cases=["test1"]) + + results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access + done, _ = run._check_results(results, sim_ok) # pylint: disable=protected-access + return done diff --git a/vunit/test_suites.py b/vunit/test_suites.py index 1bc7d8ffb..afee7b92a 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -171,25 +171,35 @@ def run(self, output_path, read_output): results = self._read_test_results(file_name=get_result_file_name(output_path)) + done, results = self._check_results(results, sim_ok) + if done: + return results + + if not self._config.call_post_check(output_path, read_output): + for name in self._test_cases: + results[name] = FAILED + + return results + + def _check_results(self, results, sim_ok): + """ + Test the results and the exit code; return True the status of any test is not PASSED + """ # If any test failed, return results for status in results.values(): if status == FAILED: - return results + return True, results # Force fail if all tests pass in the presence of non-zero exit code if self._simulator_if.has_valid_exit_code() and not sim_ok: - return dict((name, FAILED) if results[name] is PASSED else (name, results[name]) for name in results) + return True, dict((name, FAILED) if results[name] is PASSED else (name, results[name]) for name in results) # Do not run post check unless all passed for status in results.values(): if status != PASSED: - return results + return True, results - if not self._config.call_post_check(output_path, read_output): - for name in self._test_cases: - results[name] = FAILED - - return results + return False, results def _simulate(self, output_path): """ From 1f1cff9462a5c849d20311790e6b540c119977ed Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 11 Apr 2019 04:12:56 +0200 Subject: [PATCH 12/19] add tb_same_sim_all_pass_nonzero --- .../vhdl/tb_same_sim_all_pass_nonzero.vhd | 40 +++++++++++++++++++ vunit/test/acceptance/test_artificial.py | 11 +++++ 2 files changed, 51 insertions(+) create mode 100644 vunit/test/acceptance/artificial/vhdl/tb_same_sim_all_pass_nonzero.vhd diff --git a/vunit/test/acceptance/artificial/vhdl/tb_same_sim_all_pass_nonzero.vhd b/vunit/test/acceptance/artificial/vhdl/tb_same_sim_all_pass_nonzero.vhd new file mode 100644 index 000000000..5e951dbc6 --- /dev/null +++ b/vunit/test/acceptance/artificial/vhdl/tb_same_sim_all_pass_nonzero.vhd @@ -0,0 +1,40 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this file, +-- You can obtain one at http://mozilla.org/MPL/2.0/. +-- +-- Copyright (c) 2014-2019, Lars Asplund lars.anders.asplund@gmail.com + +library vunit_lib; +context vunit_lib.vunit_context; + +entity tb_same_sim_all_pass_nonzero is + generic ( + output_path : string; + runner_cfg : string); +end entity; + +architecture vunit_test_bench of tb_same_sim_all_pass_nonzero is +begin + test_runner : process + variable counter : integer := 1; + begin + test_runner_setup(runner, runner_cfg); + while test_suite loop + if run("Test 1") then + wait for 10 ns; + report "Test 1"; + assert counter = 1; + counter := counter + 1; + elsif run("Test 2") then + wait for 10 ns; + report "Test 2"; + assert counter = 2; + counter := counter + 1; + end if; + end loop; + vunit_lib.runner_pkg.p_disable_simulation_exit(runner_state); + test_runner_cleanup(runner); + assert false severity error; + wait; + end process; +end architecture; diff --git a/vunit/test/acceptance/test_artificial.py b/vunit/test/acceptance/test_artificial.py index 4719f0792..d1944b2c3 100644 --- a/vunit/test/acceptance/test_artificial.py +++ b/vunit/test/acceptance/test_artificial.py @@ -117,6 +117,14 @@ def _test_run_selected_tests_in_same_sim_test_bench(self, run_file): ("failed", "lib.tb_same_sim_some_fail.Test 2"), ("skipped", "lib.tb_same_sim_some_fail.Test 3")]) + self.check(run_file, + exit_code=1, + clean=False, + args=["*same_sim_all_pass_nonzero*Test 1*", "*same_sim_all_pass_nonzero*Test 2*"]) + check_report(self.report_file, [ + ("failed", "lib.tb_same_sim_all_pass_nonzero.Test 1"), + ("failed", "lib.tb_same_sim_all_pass_nonzero.Test 2")]) + @unittest.skipUnless(simulator_is("modelsim"), "Only modelsim supports verilog") def test_artificial_verilog(self): self.check(self.artificial_run_verilog, @@ -201,6 +209,9 @@ def test_exit_0_flag(self): ("failed", "lib.tb_same_sim_some_fail.Test 2"), ("skipped", "lib.tb_same_sim_some_fail.Test 3"), + ("failed", "lib.tb_same_sim_all_pass_nonzero.Test 1"), + ("failed", "lib.tb_same_sim_all_pass_nonzero.Test 2"), + ("passed", "lib.tb_with_checks.Test passing check"), ("failed", "lib.tb_with_checks.Test failing check"), ("failed", "lib.tb_with_checks.Test non-stopping failing check"), From b01285df37220c652d3ecaf54b005cc0a0f098ee Mon Sep 17 00:00:00 2001 From: umarcor Date: Sat, 13 Apr 2019 22:00:53 +0200 Subject: [PATCH 13/19] Revert "add tb_same_sim_all_pass_nonzero" This reverts commit 1e5e630ac9c0767e35e7aa25e7b9fdcee88643ed. --- .../vhdl/tb_same_sim_all_pass_nonzero.vhd | 40 ------------------- vunit/test/acceptance/test_artificial.py | 11 ----- 2 files changed, 51 deletions(-) delete mode 100644 vunit/test/acceptance/artificial/vhdl/tb_same_sim_all_pass_nonzero.vhd diff --git a/vunit/test/acceptance/artificial/vhdl/tb_same_sim_all_pass_nonzero.vhd b/vunit/test/acceptance/artificial/vhdl/tb_same_sim_all_pass_nonzero.vhd deleted file mode 100644 index 5e951dbc6..000000000 --- a/vunit/test/acceptance/artificial/vhdl/tb_same_sim_all_pass_nonzero.vhd +++ /dev/null @@ -1,40 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this file, --- You can obtain one at http://mozilla.org/MPL/2.0/. --- --- Copyright (c) 2014-2019, Lars Asplund lars.anders.asplund@gmail.com - -library vunit_lib; -context vunit_lib.vunit_context; - -entity tb_same_sim_all_pass_nonzero is - generic ( - output_path : string; - runner_cfg : string); -end entity; - -architecture vunit_test_bench of tb_same_sim_all_pass_nonzero is -begin - test_runner : process - variable counter : integer := 1; - begin - test_runner_setup(runner, runner_cfg); - while test_suite loop - if run("Test 1") then - wait for 10 ns; - report "Test 1"; - assert counter = 1; - counter := counter + 1; - elsif run("Test 2") then - wait for 10 ns; - report "Test 2"; - assert counter = 2; - counter := counter + 1; - end if; - end loop; - vunit_lib.runner_pkg.p_disable_simulation_exit(runner_state); - test_runner_cleanup(runner); - assert false severity error; - wait; - end process; -end architecture; diff --git a/vunit/test/acceptance/test_artificial.py b/vunit/test/acceptance/test_artificial.py index d1944b2c3..4719f0792 100644 --- a/vunit/test/acceptance/test_artificial.py +++ b/vunit/test/acceptance/test_artificial.py @@ -117,14 +117,6 @@ def _test_run_selected_tests_in_same_sim_test_bench(self, run_file): ("failed", "lib.tb_same_sim_some_fail.Test 2"), ("skipped", "lib.tb_same_sim_some_fail.Test 3")]) - self.check(run_file, - exit_code=1, - clean=False, - args=["*same_sim_all_pass_nonzero*Test 1*", "*same_sim_all_pass_nonzero*Test 2*"]) - check_report(self.report_file, [ - ("failed", "lib.tb_same_sim_all_pass_nonzero.Test 1"), - ("failed", "lib.tb_same_sim_all_pass_nonzero.Test 2")]) - @unittest.skipUnless(simulator_is("modelsim"), "Only modelsim supports verilog") def test_artificial_verilog(self): self.check(self.artificial_run_verilog, @@ -209,9 +201,6 @@ def test_exit_0_flag(self): ("failed", "lib.tb_same_sim_some_fail.Test 2"), ("skipped", "lib.tb_same_sim_some_fail.Test 3"), - ("failed", "lib.tb_same_sim_all_pass_nonzero.Test 1"), - ("failed", "lib.tb_same_sim_all_pass_nonzero.Test 2"), - ("passed", "lib.tb_with_checks.Test passing check"), ("failed", "lib.tb_with_checks.Test failing check"), ("failed", "lib.tb_with_checks.Test non-stopping failing check"), From a37521db3de3577fd0fef3babe10f30d61c3b15c Mon Sep 17 00:00:00 2001 From: umarcor Date: Sat, 13 Apr 2019 22:08:48 +0200 Subject: [PATCH 14/19] add comment; replace with_has_valid_func with has_valid_exit_code --- vunit/test/unit/test_test_suites.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/vunit/test/unit/test_test_suites.py b/vunit/test/unit/test_test_suites.py index 59773c16c..ba22fc45a 100644 --- a/vunit/test/unit/test_test_suites.py +++ b/vunit/test/unit/test_test_suites.py @@ -110,13 +110,16 @@ def _read_test_results(contents, expected_test_cases): return run._read_test_results(file_name=file_name) # pylint: disable=protected-access def test_exit_code(self): + """ + Test that results are overwritten when all are PASSED but the exit code is nonzero + """ self.assertEqual(self._test_exit_code(True), False) self.assertEqual(self._test_exit_code(False), False) self.assertEqual(self._test_exit_code(True, True), False) self.assertEqual(self._test_exit_code(False, True), True) @staticmethod - def _test_exit_code(sim_ok=True, with_has_valid_func=False): + def _test_exit_code(sim_ok=True, has_valid_exit_code=False): """ Helper method to test the check_results function """ @@ -128,11 +131,10 @@ def _test_exit_code(sim_ok=True, with_has_valid_func=False): test_suite_done """) sim_if = SimulatorInterface - if with_has_valid_func: - @staticmethod - def func(): - return True - sim_if.has_valid_exit_code = func + @staticmethod + def func(): + return has_valid_exit_code + sim_if.has_valid_exit_code = func run = TestRun(simulator_if=sim_if, config=None, From b50abcbb579e06e7ef9c04d7cd759703f70c8b61 Mon Sep 17 00:00:00 2001 From: umarcor Date: Sat, 13 Apr 2019 22:10:29 +0200 Subject: [PATCH 15/19] it is ok to pass SKIPPED to post_check, although it should never happen anyway --- vunit/test_suites.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vunit/test_suites.py b/vunit/test_suites.py index afee7b92a..e544d5f23 100644 --- a/vunit/test_suites.py +++ b/vunit/test_suites.py @@ -194,11 +194,6 @@ def _check_results(self, results, sim_ok): if self._simulator_if.has_valid_exit_code() and not sim_ok: return True, dict((name, FAILED) if results[name] is PASSED else (name, results[name]) for name in results) - # Do not run post check unless all passed - for status in results.values(): - if status != PASSED: - return True, results - return False, results def _simulate(self, output_path): From 3c0c21703b692bff04647bdf1bc113959c12b983 Mon Sep 17 00:00:00 2001 From: umarcor Date: Sat, 13 Apr 2019 22:18:48 +0200 Subject: [PATCH 16/19] style --- vunit/test/unit/test_test_suites.py | 63 ++++++++++------------------- 1 file changed, 21 insertions(+), 42 deletions(-) diff --git a/vunit/test/unit/test_test_suites.py b/vunit/test/unit/test_test_suites.py index ba22fc45a..86277a98f 100644 --- a/vunit/test/unit/test_test_suites.py +++ b/vunit/test/unit/test_test_suites.py @@ -22,77 +22,53 @@ class TestTestSuites(TestCase): """ def test_missing_results_fails_all(self): - self.assertEqual( - self._read_test_results(contents=None, - expected_test_cases=["test1", "test2"]), - {"test1": FAILED, "test2": FAILED}) + self._read_test_results( {"test1": FAILED, "test2": FAILED}, None) def test_read_results_all_passed(self): - self.assertEqual( - self._read_test_results(contents="""\ + self._read_test_results( {"test1": PASSED, "test2": PASSED}, """\ test_start:test1 test_start:test2 test_suite_done -""", - expected_test_cases=["test1", "test2"]), - {"test1": PASSED, "test2": PASSED}) +""") def test_read_results_suite_not_done(self): - self.assertEqual( - self._read_test_results(contents="""\ + self._read_test_results( {"test1": PASSED, "test2": FAILED}, """\ test_start:test1 test_start:test2 -""", - expected_test_cases=["test1", "test2"]), - {"test1": PASSED, "test2": FAILED}) +""") - self.assertEqual( - self._read_test_results(contents="""\ + self._read_test_results( {"test1": FAILED, "test2": PASSED}, """\ test_start:test2 test_start:test1 -""", - expected_test_cases=["test1", "test2"]), - {"test1": FAILED, "test2": PASSED}) +""") def test_read_results_skipped_test(self): - self.assertEqual( - self._read_test_results(contents="""\ + self._read_test_results( {"test1": PASSED, "test2": SKIPPED, "test3": SKIPPED}, """\ test_start:test1 test_suite_done -""", - expected_test_cases=["test1", "test2", "test3"]), - {"test1": PASSED, "test2": SKIPPED, "test3": SKIPPED}) +""") def test_read_results_anonynmous_test_pass(self): - self.assertEqual( - self._read_test_results(contents="""\ + self._read_test_results( {None: PASSED}, """\ test_suite_done -""", - expected_test_cases=[None]), - {None: PASSED}) +""") def test_read_results_anonynmous_test_fail(self): - self.assertEqual( - self._read_test_results(contents="""\ -""", - expected_test_cases=[None]), - {None: FAILED}) + self._read_test_results( {None: FAILED}, """\ +""") def test_read_results_unknown_test(self): try: - self._read_test_results( - contents="""\ + self._read_test_results( ["test1"], """\ test_start:test1 test_start:test3 -test_suite_done""", - expected_test_cases=["test1"]) +test_suite_done""", False) except RuntimeError as exc: self.assertIn("unknown test case test3", str(exc)) else: assert False, "RuntimeError not raised" - @staticmethod - def _read_test_results(contents, expected_test_cases): + def _read_test_results(self, expected, contents, do_assert=True): """ Helper method to test the read_test_results function """ @@ -106,8 +82,11 @@ def _read_test_results(contents, expected_test_cases): config=None, elaborate_only=False, test_suite_name=None, - test_cases=expected_test_cases) - return run._read_test_results(file_name=file_name) # pylint: disable=protected-access + test_cases=expected) + results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access + if do_assert: + self.assertEqual(results, expected) + return results def test_exit_code(self): """ From de49a01ca9d21ee82e11be3f95383ad6d03c710c Mon Sep 17 00:00:00 2001 From: umarcor Date: Sat, 13 Apr 2019 22:36:41 +0200 Subject: [PATCH 17/19] add multiple test cases to test_exit_code: passed; passed skipped; failed skipped; passed failed; and passed failed skipped --- vunit/test/unit/test_test_suites.py | 50 +++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/vunit/test/unit/test_test_suites.py b/vunit/test/unit/test_test_suites.py index 86277a98f..bdae72ec9 100644 --- a/vunit/test/unit/test_test_suites.py +++ b/vunit/test/unit/test_test_suites.py @@ -90,25 +90,45 @@ def _read_test_results(self, expected, contents, do_assert=True): def test_exit_code(self): """ - Test that results are overwritten when all are PASSED but the exit code is nonzero + Test that results are overwritten when none is FAILED but the exit code is nonzero """ - self.assertEqual(self._test_exit_code(True), False) - self.assertEqual(self._test_exit_code(False), False) - self.assertEqual(self._test_exit_code(True, True), False) - self.assertEqual(self._test_exit_code(False, True), True) - @staticmethod - def _test_exit_code(sim_ok=True, has_valid_exit_code=False): + def test(contents, results, expected=None, werechecked=[True, True, True, True]): + self._test_exit_code(contents, results, True, False, werechecked[0]) + self._test_exit_code(contents, results, False, False, werechecked[1]) + self._test_exit_code(contents, results, True, True, werechecked[2]) + r = results + if expected is not None: + r = expected + self._test_exit_code(contents, r, False, True, werechecked[3]) + + test("""\ntest_start:test1\ntest_suite_done\n""", + {"test1": PASSED}, + {"test1": FAILED}, + [False, False, False, True] + ) + + test("""\ntest_start:test1\ntest_suite_done\n""", + {"test1": PASSED, "test2": SKIPPED}, + {"test1": FAILED, "test2": SKIPPED}, + [False, False, False, True] + ) + + test("""\ntest_start:test1\n""", {"test1": FAILED, "test2": SKIPPED} ) + contents = """\ntest_start:test1\ntest_start:test2\n""" + test(contents, {"test1": PASSED, "test2": FAILED}) + test(contents, {"test1": PASSED, "test2": FAILED, "test3": SKIPPED}) + + def _test_exit_code(self, contents, expected, sim_ok=True, has_valid_exit_code=False, waschecked=False): """ Helper method to test the check_results function """ with create_tempdir() as path: file_name = join(path, "vunit_results") - with open(file_name, "w") as fptr: - fptr.write("""\ -test_start:test1 -test_suite_done -""") + if contents is not None: + with open(file_name, "w") as fptr: + fptr.write(contents) + sim_if = SimulatorInterface @staticmethod def func(): @@ -119,8 +139,10 @@ def func(): config=None, elaborate_only=False, test_suite_name=None, - test_cases=["test1"]) + test_cases=expected) results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access - done, _ = run._check_results(results, sim_ok) # pylint: disable=protected-access + done, results = run._check_results(results, sim_ok) # pylint: disable=protected-access + self.assertEqual(results, expected) + self.assertEqual(done, waschecked) return done From cdd5663a926c575fa2d2095f7483fdeceb3f737a Mon Sep 17 00:00:00 2001 From: umarcor Date: Sun, 14 Apr 2019 00:02:08 +0200 Subject: [PATCH 18/19] fix lint --- vunit/test/unit/test_test_suites.py | 45 +++++++++++++++++------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/vunit/test/unit/test_test_suites.py b/vunit/test/unit/test_test_suites.py index bdae72ec9..0351cd26c 100644 --- a/vunit/test/unit/test_test_suites.py +++ b/vunit/test/unit/test_test_suites.py @@ -22,44 +22,44 @@ class TestTestSuites(TestCase): """ def test_missing_results_fails_all(self): - self._read_test_results( {"test1": FAILED, "test2": FAILED}, None) + self._read_test_results({"test1": FAILED, "test2": FAILED}, None) def test_read_results_all_passed(self): - self._read_test_results( {"test1": PASSED, "test2": PASSED}, """\ + self._read_test_results({"test1": PASSED, "test2": PASSED}, """\ test_start:test1 test_start:test2 test_suite_done """) def test_read_results_suite_not_done(self): - self._read_test_results( {"test1": PASSED, "test2": FAILED}, """\ + self._read_test_results({"test1": PASSED, "test2": FAILED}, """\ test_start:test1 test_start:test2 """) - self._read_test_results( {"test1": FAILED, "test2": PASSED}, """\ + self._read_test_results({"test1": FAILED, "test2": PASSED}, """\ test_start:test2 test_start:test1 """) def test_read_results_skipped_test(self): - self._read_test_results( {"test1": PASSED, "test2": SKIPPED, "test3": SKIPPED}, """\ + self._read_test_results({"test1": PASSED, "test2": SKIPPED, "test3": SKIPPED}, """\ test_start:test1 test_suite_done """) def test_read_results_anonynmous_test_pass(self): - self._read_test_results( {None: PASSED}, """\ + self._read_test_results({None: PASSED}, """\ test_suite_done """) def test_read_results_anonynmous_test_fail(self): - self._read_test_results( {None: FAILED}, """\ + self._read_test_results({None: FAILED}, """\ """) def test_read_results_unknown_test(self): try: - self._read_test_results( ["test1"], """\ + self._read_test_results(["test1"], """\ test_start:test1 test_start:test3 test_suite_done""", False) @@ -83,9 +83,9 @@ def _read_test_results(self, expected, contents, do_assert=True): elaborate_only=False, test_suite_name=None, test_cases=expected) - results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access + results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access if do_assert: - self.assertEqual(results, expected) + self.assertEqual(results, expected) return results def test_exit_code(self): @@ -93,28 +93,35 @@ def test_exit_code(self): Test that results are overwritten when none is FAILED but the exit code is nonzero """ - def test(contents, results, expected=None, werechecked=[True, True, True, True]): - self._test_exit_code(contents, results, True, False, werechecked[0]) + def test(contents, results, expected=None, werechecked=None): + """ + Test the four combinations of 'sim_ok' and 'has_valid_exit_code' + """ + if werechecked is None: + werechecked = [True, True, True, True] + self._test_exit_code(contents, results, True, False, werechecked[0]) self._test_exit_code(contents, results, False, False, werechecked[1]) - self._test_exit_code(contents, results, True, True, werechecked[2]) - r = results + self._test_exit_code(contents, results, True, True, werechecked[2]) + val = results if expected is not None: - r = expected - self._test_exit_code(contents, r, False, True, werechecked[3]) + val = expected + self._test_exit_code(contents, val, False, True, werechecked[3]) - test("""\ntest_start:test1\ntest_suite_done\n""", + test( + """\ntest_start:test1\ntest_suite_done\n""", {"test1": PASSED}, {"test1": FAILED}, [False, False, False, True] ) - test("""\ntest_start:test1\ntest_suite_done\n""", + test( + """\ntest_start:test1\ntest_suite_done\n""", {"test1": PASSED, "test2": SKIPPED}, {"test1": FAILED, "test2": SKIPPED}, [False, False, False, True] ) - test("""\ntest_start:test1\n""", {"test1": FAILED, "test2": SKIPPED} ) + test("""\ntest_start:test1\n""", {"test1": FAILED, "test2": SKIPPED}) contents = """\ntest_start:test1\ntest_start:test2\n""" test(contents, {"test1": PASSED, "test2": FAILED}) test(contents, {"test1": PASSED, "test2": FAILED, "test3": SKIPPED}) From bb8d0937d8dde10d3fdd7de8778a7278d98b4d18 Mon Sep 17 00:00:00 2001 From: umarcor Date: Sun, 14 Apr 2019 15:42:55 +0200 Subject: [PATCH 19/19] cleanup --- vunit/test/unit/test_test_suites.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/vunit/test/unit/test_test_suites.py b/vunit/test/unit/test_test_suites.py index 0351cd26c..0f34fff21 100644 --- a/vunit/test/unit/test_test_suites.py +++ b/vunit/test/unit/test_test_suites.py @@ -62,13 +62,13 @@ def test_read_results_unknown_test(self): self._read_test_results(["test1"], """\ test_start:test1 test_start:test3 -test_suite_done""", False) +test_suite_done""") except RuntimeError as exc: self.assertIn("unknown test case test3", str(exc)) else: assert False, "RuntimeError not raised" - def _read_test_results(self, expected, contents, do_assert=True): + def _read_test_results(self, expected, contents): """ Helper method to test the read_test_results function """ @@ -84,8 +84,7 @@ def _read_test_results(self, expected, contents, do_assert=True): test_suite_name=None, test_cases=expected) results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access - if do_assert: - self.assertEqual(results, expected) + self.assertEqual(results, expected) return results def test_exit_code(self): @@ -149,7 +148,7 @@ def func(): test_cases=expected) results = run._read_test_results(file_name=file_name) # pylint: disable=protected-access - done, results = run._check_results(results, sim_ok) # pylint: disable=protected-access - self.assertEqual(results, expected) - self.assertEqual(done, waschecked) - return done + self.assertEqual( + run._check_results(results, sim_ok), # pylint: disable=protected-access + (waschecked, expected) + )