From aeb9524daa222bed0a225d774c255af0c2b0e3e6 Mon Sep 17 00:00:00 2001 From: tcezard Date: Fri, 24 Jul 2026 15:32:25 +0100 Subject: [PATCH 1/8] Add error handling for missing validation outputs --- eva_sub_cli/validators/validator.py | 33 +++++++--- tests/test_validator.py | 93 ++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 9 deletions(-) diff --git a/eva_sub_cli/validators/validator.py b/eva_sub_cli/validators/validator.py index 57ee390..d3f962c 100755 --- a/eva_sub_cli/validators/validator.py +++ b/eva_sub_cli/validators/validator.py @@ -418,6 +418,11 @@ def _load_fasta_check_results(self): fasta_check = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', f'{fasta_file_name}_check.yml')) if not fasta_check: + error_txt = f'Cannot locate sequence check results for {fasta_file_name}. The process might have failed.' + self.error(error_txt) + self.results[FASTA_CHECK][fasta_file_name] = { + 'all_insdc': False, 'sequences': [], 'connection_error': error_txt + } continue with open(fasta_check) as open_yaml: self.results[FASTA_CHECK][fasta_file_name] = yaml.safe_load(open_yaml) @@ -428,8 +433,10 @@ def _load_sample_check_results(self): with open(self._sample_check_yaml) as open_yaml: self.results[SAMPLE_CHECK] = yaml.safe_load(open_yaml) self.results[SAMPLE_CHECK]['report_path'] = self._sample_check_yaml - - self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: True}) + self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: True}) + else: + self.error(f'Cannot locate sample check results. The process might have failed.') + self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: False}) def _load_evidence_check_results(self): self.results[EVIDENCE_TYPE_CHECK] = {} @@ -437,7 +444,10 @@ def _load_evidence_check_results(self): with open(self._evidence_type_check_yaml) as open_yaml: self.results[EVIDENCE_TYPE_CHECK] = yaml.safe_load(open_yaml) self.results[EVIDENCE_TYPE_CHECK]['report_path'] = self._evidence_type_check_yaml - self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: True}) + self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: True}) + else: + self.error(f'Cannot locate evidence type check results. The process might have failed.') + self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: False}) self._update_metadata_with_evidence_type() def _collect_metadata_results(self): @@ -465,7 +475,12 @@ def collect_biovalidator_validation_results(self): """ metadata_check_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'metadata_validation.txt')) - errors = parse_biovalidator_validation_results(metadata_check_file) + if metadata_check_file: + errors = parse_biovalidator_validation_results(metadata_check_file) + else: + error_txt = (f"Cannot locate metadata check file. The process might have failed.") + self.error(error_txt) + errors = [{'property': '/', 'description': error_txt}] self.results[METADATA_CHECK].update({ 'json_report_path': metadata_check_file, 'json_errors': errors @@ -554,9 +569,8 @@ def _collect_file_info_to_metadata(self): file_path_2_file_size[vcf_file] = file_size file_name_2_file_size[os.path.basename(vcf_file)] = file_size else: - error_txt = f"Cannot locate file_info.txt at {os.path.join(self.output_dir, 'other_validations', 'file_info.txt')}" + error_txt = f"Cannot locate file_info.txt. The process might have failed." self.error(error_txt) - raise FileNotFoundError(error_txt) if self.metadata_json_post_validation: metadata = EvaMetadataJson(self.metadata_json_post_validation) @@ -593,9 +607,9 @@ def _collect_file_info_to_metadata(self): errors.append({'property': '/', 'description': error_txt}) metadata.write(self.metadata_json_post_validation) else: - error_txt = f'Cannot locate the metadata in JSON format in {os.path.join(self.output_dir, "metadata.json")}' + error_txt = f'Cannot locate the metadata in JSON format. The process might have failed.' self.error(error_txt) - raise FileNotFoundError(error_txt) + errors.append({'property': '/', 'description': error_txt}) if errors: if 'json_errors' in self.results[METADATA_CHECK]: self.results[METADATA_CHECK]['json_errors'].extend(errors) @@ -640,6 +654,9 @@ def _collect_trim_down_metrics(self): vcf_name, _ = os.path.splitext(basename) trimmed_down_metrics = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', f'{vcf_name}_trim_down.yml')) + if not trimmed_down_metrics: + self.error(f'Cannot locate trim down metrics for {vcf_name}. The process might have failed.') + continue with open(trimmed_down_metrics) as open_file: metrics = yaml.safe_load(open_file) shallow_validation_required = shallow_validation_required or metrics['trim_down_required'] diff --git a/tests/test_validator.py b/tests/test_validator.py index 6e0dd37..51ba52a 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -9,7 +9,8 @@ import eva_sub_cli from eva_sub_cli.metadata import EvaMetadataJson from eva_sub_cli.validators.validator import (Validator, VALIDATION_OUTPUT_DIR, VCF_CHECK, READY_FOR_SUBMISSION_TO_EVA, - RUN_STATUS_KEY, METADATA_CHECK, PASS, TRIM_DOWN, SHALLOW_VALIDATION) + RUN_STATUS_KEY, METADATA_CHECK, PASS, TRIM_DOWN, SHALLOW_VALIDATION, + FASTA_CHECK, SAMPLE_CHECK, EVIDENCE_TYPE_CHECK) from tests.test_utils import create_mapping_file expected_validation_results = { @@ -184,6 +185,25 @@ def create_validator(self, submission_dir): ) return Validator(mapping_file, submission_dir, metadata_json=self.metadata_json_file) + def create_validator_with_copied_output(self, submission_dir, metadata_json=None, metadata_xlsx=None, + shallow_validation=False, validation_tasks=None): + """ + Copy the shared validation_output fixture tree into a fresh submission_dir so a test can delete + individual output files to simulate an incomplete Nextflow run without affecting other tests. + """ + shutil.copytree(os.path.join(self.output_dir, VALIDATION_OUTPUT_DIR), + os.path.join(submission_dir, VALIDATION_OUTPUT_DIR)) + mapping_file = os.path.join(submission_dir, 'vcf_files_mapping.csv') + create_mapping_file(mapping_file, + [os.path.join(self.vcf_files, 'input_passed.vcf')], + [os.path.join(self.fasta_files, 'input_passed.fa')], + [os.path.join(self.assembly_reports, 'input_passed.txt')]) + kwargs = {} + if validation_tasks is not None: + kwargs['validation_tasks'] = validation_tasks + return Validator(mapping_file, submission_dir, metadata_json=metadata_json, metadata_xlsx=metadata_xlsx, + shallow_validation=shallow_validation, **kwargs) + def test_clean_up_output_dir_moves_intermediate_files_and_removes_nextflow_work_dir(self): with TemporaryDirectory() as submission_dir: validator = self.create_validator(submission_dir) @@ -666,6 +686,77 @@ def test__check_consent_statement_is_needed_for_submission(self): } assert self.validator_json._check_consent_statement_is_needed_for_submission() is True + def test__collect_file_info_to_metadata_missing_file_info_txt(self): + # Test for a nextflow run that did not complete and never produced file_info.txt + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'file_info.txt')) + self.run_collect_results(validator) + assert validator.results == self.format_data_structure(expected_validation_results) + def test__collect_file_info_to_metadata_missing_metadata_json(self): + # Test for a nextflow run that did not complete and never produced metadata.json + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir) + os.remove(os.path.join(validator.output_dir, 'metadata.json')) + validator._collect_validation_workflow_results() + expected_error = { + 'property': '/', + 'description': f'Cannot locate the metadata in JSON format. The process might have failed.' + } + assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False + def test_collect_biovalidator_validation_results_missing_report(self): + # Test for a nextflow run that did not complete and never produced metadata_validation.txt + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file, + validation_tasks=[METADATA_CHECK]) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_validation.txt')) + validator._collect_validation_workflow_results() + expected_error = { + 'property': '/', + 'description': f'Cannot locate metadata check file. The process might have failed.' + } + assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False + + def test__collect_trim_down_metrics_missing_yml(self): + # Test for a nextflow run that did not complete and never produced input_passed_trim_down.yml + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file, + shallow_validation=True) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'input_passed_trim_down.yml')) + validator._collect_validation_workflow_results() + assert validator.results[SHALLOW_VALIDATION][TRIM_DOWN] is False + assert validator.vcf_files[0] not in validator.results[SHALLOW_VALIDATION]['metrics'] + + def test__load_sample_check_results_missing_yaml(self): + # Test for a nextflow run that did not complete and never produced sample_checker.yml + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'sample_checker.yml')) + validator._load_sample_check_results() + assert validator.results[SAMPLE_CHECK][RUN_STATUS_KEY] is False + + def test__load_evidence_check_results_missing_yaml(self): + # Test for a nextflow run that did not complete and never produced evidence_type_checker.yml + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'evidence_type_checker.yml')) + validator._load_evidence_check_results() + assert validator.results[EVIDENCE_TYPE_CHECK][RUN_STATUS_KEY] is False + def test__load_fasta_check_results_missing_yaml(self): + # Test for a nextflow run that did not complete and never produced input_passed.fa_check.yml + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'input_passed.fa_check.yml')) + validator._collect_validation_workflow_results() + fasta_result = validator.results[FASTA_CHECK]['input_passed.fa'] + assert fasta_result['all_insdc'] is False + assert 'connection_error' in fasta_result + validator._assess_validation_results() + assert validator.results[FASTA_CHECK][PASS] is False From 35907077f76a8ad8cf599be09b053ba0210c9bcc Mon Sep 17 00:00:00 2001 From: tcezard Date: Wed, 29 Jul 2026 17:09:50 +0100 Subject: [PATCH 2/8] Standardize `run_status` values and enhance error handling for validation tasks. --- eva_sub_cli/jinja_templates/html/report.html | 23 ++++++-- eva_sub_cli/jinja_templates/text/report.txt | 45 ++++++++++------ eva_sub_cli/validators/validator.py | 53 ++++++++++++------- .../expected_metadata_json_report.txt | 16 +++--- .../expected_metadata_xlsx_report.txt | 16 +++--- .../expected_shallow_metadata_xlsx_report.txt | 16 +++--- tests/test_report.py | 40 +++++++------- tests/test_validator.py | 44 +++++++-------- 8 files changed, 150 insertions(+), 103 deletions(-) diff --git a/eva_sub_cli/jinja_templates/html/report.html b/eva_sub_cli/jinja_templates/html/report.html index 8d71fa8..d48d2f6 100644 --- a/eva_sub_cli/jinja_templates/html/report.html +++ b/eva_sub_cli/jinja_templates/html/report.html @@ -13,6 +13,13 @@
{{ expand_icon }} {{ icon }} {{ text }}
{% endmacro %} +{% macro report_crashed_task(task_name) %} + {% set icon = "❗" %} {# exclamation point ❗#} + {% set row_class = "report-section info" %} + {% set expand_icon = "" %} +
{{ expand_icon }} {{ icon }} {{ task_name }} did not complete successfully. Check the logs for more information.
+{% endmacro %} + @@ -73,8 +80,10 @@

Metadata validation results

For requirements, please refer to the EVA website. {% set run_status = validation_results.get('metadata_check', {}).get('run_status', '') %} - {% if run_status %} + {% if run_status == 'success' %} {{ metadata_validation_report(validation_results) }} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('Metadata check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -87,13 +96,15 @@

VCF validation results

Also checks whether the variants' reference alleles match against the reference assembly. {% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %} - {% if run_status %} + {% if run_status == 'success' %} {% for file_name in vcf_files %} {% if file_name != "pass"%}

{{ file_name }}

{{ file_validation_report(validation_results, file_name) }} {% endif %} {% endfor %} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('Vcf check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -105,8 +116,10 @@

Sample name concordance check

Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. {% set run_status = validation_results.get('sample_check', {}).get('run_status', '') %} - {% if run_status %} + {% if run_status == 'success' %} {{ sample_name_check_report(validation_results)}} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('Sample name concordance') }} {% else %} {{ validation_not_run_yet_message() }} @@ -120,13 +133,15 @@

Reference genome INSDC check

Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. {% set run_status = validation_results.get('fasta_check', {}).get('run_status', '') %} - {% if run_status %} + {% if run_status == 'success' %} {% for file_name in fasta_files %} {% if file_name != "pass"%}

{{ file_name }}

{{ fasta_check_report(validation_results, file_name) }} {% endif %} {% endfor %} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('INSDC check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} diff --git a/eva_sub_cli/jinja_templates/text/report.txt b/eva_sub_cli/jinja_templates/text/report.txt index d087b12..83ee50e 100644 --- a/eva_sub_cli/jinja_templates/text/report.txt +++ b/eva_sub_cli/jinja_templates/text/report.txt @@ -10,6 +10,11 @@ {{ icon }} {{ text }} {%- endmacro %} +{% macro report_crashed_task(task_name) -%} +{% set icon = "\u2757" %} {# exclamation point ❗#} +{{ icon }} {{ task_name }} did not complete successfully. Check the logs for more information. +{%- endmacro %} + VALIDATION REPORT eva-sub-cli v{{cli_version}} @@ -29,8 +34,10 @@ Ensures that required fields are present and values are formatted correctly. For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data). {% set run_status = validation_results.get('metadata_check', {}).get('run_status', '') %} -{% if run_status %} +{% if run_status == 'success' %} {{ metadata_validation_report(validation_results) }} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('Metadata check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -41,13 +48,15 @@ Checks whether each file is compliant with the VCF specification (http://samtool Also checks whether the variants' reference alleles match against the reference assembly. {% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %} -{% if run_status %} -{% for file_name in vcf_files %} -{% if file_name != "pass"%} - {{ file_name }} - {{ file_validation_report(validation_results, file_name) }} -{% endif %} -{% endfor %} +{% if run_status == 'success' %} + {% for file_name in vcf_files %} + {% if file_name != "pass"%} + {{ file_name }} + {{ file_validation_report(validation_results, file_name) }} + {% endif %} + {% endfor %} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('Vcf check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -58,8 +67,10 @@ SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. {% set run_status = validation_results.get('sample_check', {}).get('run_status', '') %} -{% if run_status %} +{% if run_status == 'success' %} {{ sample_name_check_report(validation_results) }} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('Sample name concordance') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -71,13 +82,15 @@ Checks that the reference sequences in the FASTA file used to call the variants Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. {% set run_status = validation_results.get('fasta_check', {}).get('run_status', '') %} -{% if run_status %} -{% for file_name in fasta_files %} -{% if file_name != "pass"%} - {{ file_name }} - {{ fasta_check_report(validation_results, file_name) }} -{% endif %} -{% endfor %} +{% if run_status == 'success' %} + {% for file_name in fasta_files %} + {% if file_name != "pass"%} + {{ file_name }} + {{ fasta_check_report(validation_results, file_name) }} + {% endif %} + {% endfor %} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('INSDC check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} diff --git a/eva_sub_cli/validators/validator.py b/eva_sub_cli/validators/validator.py index d3f962c..d74e913 100755 --- a/eva_sub_cli/validators/validator.py +++ b/eva_sub_cli/validators/validator.py @@ -41,7 +41,11 @@ METADATA_CHECK] SHALLOW_VALIDATION = 'shallow_validation' TRIM_DOWN = 'trim_down' +# Status of the validation for a specific task. It should be a string containing the value "success", "crashed" or "did not run". RUN_STATUS_KEY = 'run_status' +RUN_STATUS_SUCCESS = 'success' +RUN_STATUS_CRASHED = 'crashed' +RUN_STATUS_DID_NOT_RUN = 'did not run' PASS = 'pass' @@ -197,11 +201,14 @@ def _load_previous_validation_results(self): # update previous shallow validation format to new one, if applicable if 'version' in self.results: if version.parse(self.results['version']) < version.parse('v0.4.15'): - self.update_previous_version_results() + self.update_pre_0_4_15_version_results() + if version.parse(self.results['version']) < version.parse('v0.6.5'): + self.update_pre_0_6_5_version_results() else: - self.update_previous_version_results() + self.update_pre_0_4_15_version_results() + self.update_pre_0_6_5_version_results() - def update_previous_version_results(self): + def update_pre_0_4_15_version_results(self): if 'requested' in self.results.get(SHALLOW_VALIDATION, {}): if 'required' in self.results.get(SHALLOW_VALIDATION, {}): if self.results.get(SHALLOW_VALIDATION, {}).get('required', False): @@ -214,6 +221,15 @@ def update_previous_version_results(self): del self.results[SHALLOW_VALIDATION]['required'] del self.results[SHALLOW_VALIDATION]['requested'] + def update_pre_0_6_5_version_results(self): + for task in ALL_VALIDATION_TASKS_GRANULAR: + if task in self.results: + run_status = self.results[task].get(RUN_STATUS_KEY) + if run_status: + self.results[task][RUN_STATUS_KEY] = RUN_STATUS_SUCCESS + else: + self.results[task][RUN_STATUS_KEY] = RUN_STATUS_DID_NOT_RUN + def _collect_validation_workflow_results(self): # Collect information from the output and summarise in the config if self.shallow_validation: @@ -254,8 +270,8 @@ def _assess_validation_results(self): for v in self.results.get(EVIDENCE_TYPE_CHECK, {}).values() if isinstance(v, dict))) elif VCF_CHECK not in self.results: - self.results[VCF_CHECK] = {RUN_STATUS_KEY: False} - self.results[EVIDENCE_TYPE_CHECK] = {RUN_STATUS_KEY: False} + self.results[VCF_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} + self.results[EVIDENCE_TYPE_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} if ASSEMBLY_CHECK in self.tasks: # assembly_check result @@ -279,15 +295,15 @@ def _assess_validation_results(self): if isinstance(fa_file_check, dict)) self.results[FASTA_CHECK][PASS] = fasta_check_result and gca_check_result elif ASSEMBLY_CHECK not in self.results: - self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: False} - self.results[FASTA_CHECK] = {RUN_STATUS_KEY: False} + self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} + self.results[FASTA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} if SAMPLE_CHECK in self.tasks: # sample check result self.results[SAMPLE_CHECK][PASS] = self.results.get(SAMPLE_CHECK, {}).get('overall_differences', True) is False elif SAMPLE_CHECK not in self.results: - self.results[SAMPLE_CHECK] = {RUN_STATUS_KEY: False} + self.results[SAMPLE_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} if METADATA_CHECK in self.tasks: # metadata check result @@ -295,7 +311,7 @@ def _assess_validation_results(self): metadata_json_result = len(self.results.get(METADATA_CHECK, {}).get('json_errors', []) or []) == 0 self.results[METADATA_CHECK][PASS] = metadata_xlsx_result and metadata_json_result elif METADATA_CHECK not in self.results: - self.results[METADATA_CHECK] = {RUN_STATUS_KEY: False} + self.results[METADATA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} # update config based on the validation results self.sub_config.set(READY_FOR_SUBMISSION_TO_EVA, value=self.verify_ready_for_submission_to_eva()) @@ -351,7 +367,7 @@ def _evidence_type_check_yaml(self): def _collect_vcf_check_results(self): # detect output files for vcf check - self.results[VCF_CHECK] = {RUN_STATUS_KEY: True} + self.results[VCF_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[VCF_CHECK].update({TRIM_DOWN: True}) @@ -379,7 +395,7 @@ def _collect_vcf_check_results(self): def _collect_assembly_check_results(self): # detect output files for assembly check - self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: True} + self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[ASSEMBLY_CHECK].update({TRIM_DOWN: True}) @@ -409,7 +425,7 @@ def _collect_assembly_check_results(self): } def _load_fasta_check_results(self): - self.results[FASTA_CHECK] = {RUN_STATUS_KEY: True} + self.results[FASTA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[FASTA_CHECK].update({TRIM_DOWN: True}) @@ -433,10 +449,10 @@ def _load_sample_check_results(self): with open(self._sample_check_yaml) as open_yaml: self.results[SAMPLE_CHECK] = yaml.safe_load(open_yaml) self.results[SAMPLE_CHECK]['report_path'] = self._sample_check_yaml - self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: True}) + self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_SUCCESS}) else: self.error(f'Cannot locate sample check results. The process might have failed.') - self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: False}) + self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) def _load_evidence_check_results(self): self.results[EVIDENCE_TYPE_CHECK] = {} @@ -444,14 +460,14 @@ def _load_evidence_check_results(self): with open(self._evidence_type_check_yaml) as open_yaml: self.results[EVIDENCE_TYPE_CHECK] = yaml.safe_load(open_yaml) self.results[EVIDENCE_TYPE_CHECK]['report_path'] = self._evidence_type_check_yaml - self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: True}) + self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_SUCCESS}) else: self.error(f'Cannot locate evidence type check results. The process might have failed.') - self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: False}) + self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) self._update_metadata_with_evidence_type() def _collect_metadata_results(self): - self.results[METADATA_CHECK] = {RUN_STATUS_KEY: True} + self.results[METADATA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} self._load_spreadsheet_conversion_errors() self.collect_biovalidator_validation_results() self._collect_semantic_metadata_results() @@ -648,7 +664,7 @@ def _update_metadata_with_evidence_type(self): def _collect_trim_down_metrics(self): self.results[SHALLOW_VALIDATION] = {'metrics': {}} - shallow_validation_required = False + shallow_validation_required = False # Flag to indicate if shallow validation actually reduced the number of lines for vcf_file in self.vcf_files: basename = os.path.basename(vcf_file) vcf_name, _ = os.path.splitext(basename) @@ -656,6 +672,7 @@ def _collect_trim_down_metrics(self): f'{vcf_name}_trim_down.yml')) if not trimmed_down_metrics: self.error(f'Cannot locate trim down metrics for {vcf_name}. The process might have failed.') + shallow_validation_required = True # Assume that the shallow validation would have been required continue with open(trimmed_down_metrics) as open_file: metrics = yaml.safe_load(open_file) diff --git a/tests/resources/validation_reports/expected_metadata_json_report.txt b/tests/resources/validation_reports/expected_metadata_json_report.txt index 6ed2d15..58ac851 100644 --- a/tests/resources/validation_reports/expected_metadata_json_report.txt +++ b/tests/resources/validation_reports/expected_metadata_json_report.txt @@ -72,7 +72,7 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf + input_fail.vcf ❌ Assembly check: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report @@ -89,7 +89,7 @@ Also checks whether the variants' reference alleles match against the reference First 10 errors per category are below. Full report: /path/to/vcf_failed/report Critical error: Line 4: Error in meta-data section. Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf + input_passed.vcf ✔ Assembly check: 247/247 (100.0%) ✔ VCF check: 0 critical errors, 0 non-critical errors - @@ -108,16 +108,16 @@ Checks whether information in the metadata is concordant with that contained in REFERENCE GENOME INSDC CHECK Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. - metadata_asm_match.fa + metadata_asm_match.fa ✔ All sequences are INSDC accessioned. ✔ Analysis A: Assembly accession in metadata is compatible - metadata_asm_not_found.fa + metadata_asm_not_found.fa ✔ All sequences are INSDC accessioned. ❌ No assembly accession found in metadata Full report: /path/to/metadata_asm_not_found.yml Assembly accession found in metadata: Not found Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_gca.fa + metadata_asm_not_gca.fa Warning: Non-GCA reference found in metadata. Please provide the INSDC accession for your reference assembly. If you would like to submit using a non-GCA reference sequence, contact eva-helpdesk@ebi.ac.uk. ✔ All sequences are INSDC accessioned. @@ -125,19 +125,19 @@ Also checks if the reference assembly accession in the metadata matches the one Full report: /path/to/metadata_asm_not_gca.yml Assembly accession found in metadata: GCF_1 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_match.fa + metadata_asm_not_match.fa ✔ All sequences are INSDC accessioned. ❗ Analysis B: Assembly accession in metadata is not compatible Full report: /path/to/metadata_asm_not_match.yml Assembly accession found in metadata: GCA_2 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_error.fa + metadata_error.fa Warning: The following results may be incomplete due to problems with external services. Please try again later for complete results. Error message: 500 Server Error: Internal Server Error for url: https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum/hjfdoijsfc47hfg0gh9qwjrve ✔ All sequences are INSDC accessioned. ✔ Analysis C: Assembly accession in metadata is compatible - not_all_insdc.fa + not_all_insdc.fa ❌ Some sequences are not INSDC accessioned First 10 sequences not in INSDC. Full report: /path/to/not_all_insdc_check.yml Sequence name: 2 diff --git a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt index c3ee9b4..7360f62 100644 --- a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt @@ -66,7 +66,7 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf + input_fail.vcf ❌ Assembly check: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report @@ -83,7 +83,7 @@ Also checks whether the variants' reference alleles match against the reference First 10 errors per category are below. Full report: /path/to/vcf_failed/report Critical error: Line 4: Error in meta-data section. Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf + input_passed.vcf ✔ Assembly check: 247/247 (100.0%) ✔ VCF check: 0 critical errors, 0 non-critical errors - @@ -102,16 +102,16 @@ Checks whether information in the metadata is concordant with that contained in REFERENCE GENOME INSDC CHECK Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. - metadata_asm_match.fa + metadata_asm_match.fa ✔ All sequences are INSDC accessioned. ✔ Analysis A: Assembly accession in metadata is compatible - metadata_asm_not_found.fa + metadata_asm_not_found.fa ✔ All sequences are INSDC accessioned. ❌ No assembly accession found in metadata Full report: /path/to/metadata_asm_not_found.yml Assembly accession found in metadata: Not found Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_gca.fa + metadata_asm_not_gca.fa Warning: Non-GCA reference found in metadata. Please provide the INSDC accession for your reference assembly. If you would like to submit using a non-GCA reference sequence, contact eva-helpdesk@ebi.ac.uk. ✔ All sequences are INSDC accessioned. @@ -119,19 +119,19 @@ Also checks if the reference assembly accession in the metadata matches the one Full report: /path/to/metadata_asm_not_gca.yml Assembly accession found in metadata: GCF_1 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_match.fa + metadata_asm_not_match.fa ✔ All sequences are INSDC accessioned. ❗ Analysis B: Assembly accession in metadata is not compatible Full report: /path/to/metadata_asm_not_match.yml Assembly accession found in metadata: GCA_2 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_error.fa + metadata_error.fa Warning: The following results may be incomplete due to problems with external services. Please try again later for complete results. Error message: 500 Server Error: Internal Server Error for url: https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum/hjfdoijsfc47hfg0gh9qwjrve ✔ All sequences are INSDC accessioned. ✔ Analysis C: Assembly accession in metadata is compatible - not_all_insdc.fa + not_all_insdc.fa ❌ Some sequences are not INSDC accessioned First 10 sequences not in INSDC. Full report: /path/to/not_all_insdc_check.yml Sequence name: 2 diff --git a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt index 58cfe7f..dafc26e 100644 --- a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt @@ -75,7 +75,7 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf + input_fail.vcf ❌ Assembly check: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report @@ -92,7 +92,7 @@ Also checks whether the variants' reference alleles match against the reference First 10 errors per category are below. Full report: /path/to/vcf_failed/report Critical error: Line 4: Error in meta-data section. Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf + input_passed.vcf ✔ Assembly check: 247/247 (100.0%) ✔ VCF check: 0 critical errors, 0 non-critical errors - @@ -111,16 +111,16 @@ Checks whether information in the metadata is concordant with that contained in REFERENCE GENOME INSDC CHECK Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. - metadata_asm_match.fa + metadata_asm_match.fa ✔ All sequences are INSDC accessioned. ✔ Analysis A: Assembly accession in metadata is compatible - metadata_asm_not_found.fa + metadata_asm_not_found.fa ✔ All sequences are INSDC accessioned. ❌ No assembly accession found in metadata Full report: /path/to/metadata_asm_not_found.yml Assembly accession found in metadata: Not found Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_gca.fa + metadata_asm_not_gca.fa Warning: Non-GCA reference found in metadata. Please provide the INSDC accession for your reference assembly. If you would like to submit using a non-GCA reference sequence, contact eva-helpdesk@ebi.ac.uk. ✔ All sequences are INSDC accessioned. @@ -128,19 +128,19 @@ Also checks if the reference assembly accession in the metadata matches the one Full report: /path/to/metadata_asm_not_gca.yml Assembly accession found in metadata: GCF_1 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_match.fa + metadata_asm_not_match.fa ✔ All sequences are INSDC accessioned. ❗ Analysis B: Assembly accession in metadata is not compatible Full report: /path/to/metadata_asm_not_match.yml Assembly accession found in metadata: GCA_2 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_error.fa + metadata_error.fa Warning: The following results may be incomplete due to problems with external services. Please try again later for complete results. Error message: 500 Server Error: Internal Server Error for url: https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum/hjfdoijsfc47hfg0gh9qwjrve ✔ All sequences are INSDC accessioned. ✔ Analysis C: Assembly accession in metadata is compatible - not_all_insdc.fa + not_all_insdc.fa ❌ Some sequences are not INSDC accessioned First 10 sequences not in INSDC. Full report: /path/to/not_all_insdc_check.yml Sequence name: 2 diff --git a/tests/test_report.py b/tests/test_report.py index 50672d3..9dd8e73 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -6,14 +6,14 @@ import eva_sub_cli from eva_sub_cli.report import generate_html_report, generate_text_report -from eva_sub_cli.validators.validator import RUN_STATUS_KEY, TRIM_DOWN +from eva_sub_cli.validators.validator import RUN_STATUS_KEY, RUN_STATUS_SUCCESS, RUN_STATUS_DID_NOT_RUN, TRIM_DOWN common_validation_results = { "ready_for_submission_to_eva": False, "version": "0.5.1", "trim_down": False, "assembly_check": { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, "input_passed.vcf": { "report_path": "/path/to/assembly_passed/report", @@ -45,7 +45,7 @@ }, }, "vcf_check": { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, "input_passed.vcf": { 'report_path': '/path/to/vcf_passed/report', @@ -66,7 +66,7 @@ }, }, "sample_check": { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'report_path': '/path/to/sample/report', 'overall_differences': True, @@ -96,7 +96,7 @@ # NB. obviously this doesn't make sense for the number of analyses in this report, but demonstrates the possible # outputs for this check. "fasta_check": { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'not_all_insdc.fa': { 'report_path': '/path/to/not_all_insdc_check.yml', @@ -175,7 +175,7 @@ } }, 'evidence_type_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'report_path': '/path/to/evidence_type.yml', 'Analysis A': { @@ -191,7 +191,7 @@ validation_results_xlsx = deepcopy(common_validation_results) validation_results_xlsx['metadata_check'] = { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'spreadsheet_errors': [ {'sheet': 'Files', 'row': '', 'column': '', 'description': 'Sheet "Files" is missing'}, @@ -217,7 +217,7 @@ validation_results_json = deepcopy(common_validation_results) validation_results_json['metadata_check'] = { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'json_errors': [ {'property': '.files', 'description': "should have required property 'files'"}, @@ -304,12 +304,12 @@ def test_generate_html_report_metadata_json(self): def test_generate_html_report_metadata_json_metadata_report_not_run_yet(self): validation_result = { - 'vcf_check': {RUN_STATUS_KEY: False}, - 'evidence_type_check': {RUN_STATUS_KEY: False}, - 'assembly_check': {RUN_STATUS_KEY: False}, - 'fasta_check': {RUN_STATUS_KEY: False}, - 'metadata_check': {RUN_STATUS_KEY: False}, - 'sample_check': {RUN_STATUS_KEY: False} + 'vcf_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'evidence_type_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'assembly_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'fasta_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'metadata_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'sample_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} } self.check_report_vs_expected( @@ -352,12 +352,12 @@ def test_generate_text_report_metadata_json(self): def test_generate_text_report_metadata_json_report_not_run(self): validation_result = { - 'vcf_check': {RUN_STATUS_KEY: False}, - 'evidence_type_check': {RUN_STATUS_KEY: False}, - 'assembly_check': {RUN_STATUS_KEY: False}, - 'fasta_check': {RUN_STATUS_KEY: False}, - 'metadata_check': {RUN_STATUS_KEY: False}, - 'sample_check': {RUN_STATUS_KEY: False} + 'vcf_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'evidence_type_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'assembly_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'fasta_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'metadata_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'sample_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} } self.check_report_vs_expected( diff --git a/tests/test_validator.py b/tests/test_validator.py index 51ba52a..a3b0f69 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -9,23 +9,24 @@ import eva_sub_cli from eva_sub_cli.metadata import EvaMetadataJson from eva_sub_cli.validators.validator import (Validator, VALIDATION_OUTPUT_DIR, VCF_CHECK, READY_FOR_SUBMISSION_TO_EVA, - RUN_STATUS_KEY, METADATA_CHECK, PASS, TRIM_DOWN, SHALLOW_VALIDATION, - FASTA_CHECK, SAMPLE_CHECK, EVIDENCE_TYPE_CHECK) + RUN_STATUS_KEY, RUN_STATUS_SUCCESS, RUN_STATUS_CRASHED, + RUN_STATUS_DID_NOT_RUN, METADATA_CHECK, PASS, TRIM_DOWN, + SHALLOW_VALIDATION, FASTA_CHECK, SAMPLE_CHECK, EVIDENCE_TYPE_CHECK) from tests.test_utils import create_mapping_file expected_validation_results = { 'vcf_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'input_passed.vcf': {'valid': True, 'error_list': [], 'error_count': 0, 'warning_count': 0, 'critical_count': 0, 'critical_list': []} }, 'assembly_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'input_passed.vcf': {'error_list': [], 'mismatch_list': [], 'nb_mismatch': 0, 'nb_error': 0, 'match': 247, 'total': 247} }, 'sample_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'overall_differences': False, 'results_per_analysis': { 'AA': { @@ -37,7 +38,7 @@ } }, 'evidence_type_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'AA': { 'errors': None, 'evidence_type': 'allele_frequency' @@ -45,14 +46,14 @@ }, 'fasta_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'input_passed.fa': {'all_insdc': False, 'sequences': [ {'sequence_name': 1, 'insdc': True, 'sequence_md5': '6681ac2f62509cfc220d78751b8dc524'}, {'sequence_name': 2, 'insdc': False, 'sequence_md5': 'd2b3f22704d944f92a6bc45b6603ea2d'} ]}, }, 'metadata_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'json_errors': [ {'property': '/files', 'description': "should have required property 'files'"}, {'property': '/project/title', 'description': "should have required property 'title'"}, @@ -253,13 +254,13 @@ def test__collect_validation_workflow_results_for_validation_task_no_previous_re self.validator_json._assess_validation_results() # assert assessed results assert self.validator_json.results['vcf_check']['pass'] == True - assert self.validator_json.results['vcf_check'][RUN_STATUS_KEY] == True + assert self.validator_json.results['vcf_check'][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS assert self.validator_json.results['evidence_type_check']['pass'] == True - assert self.validator_json.results['evidence_type_check'][RUN_STATUS_KEY] == True - assert self.validator_json.results['assembly_check'][RUN_STATUS_KEY] == False - assert self.validator_json.results['fasta_check'][RUN_STATUS_KEY] == False - assert self.validator_json.results['sample_check'][RUN_STATUS_KEY] == False - assert self.validator_json.results['metadata_check'][RUN_STATUS_KEY] == False + assert self.validator_json.results['evidence_type_check'][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + assert self.validator_json.results['assembly_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert self.validator_json.results['fasta_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert self.validator_json.results['sample_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert self.validator_json.results['metadata_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN assert self.validator_json.sub_config.get(READY_FOR_SUBMISSION_TO_EVA) == False assert self.validator_json.results[READY_FOR_SUBMISSION_TO_EVA] == False @@ -272,10 +273,10 @@ def test__collect_validation_workflow_results_for_validation_task_no_previous_re expected_evidence_type_check['pass'] = True assert saved_results['vcf_check'] == expected_vcf_check assert saved_results['evidence_type_check'] == expected_evidence_type_check - assert saved_results['assembly_check'][RUN_STATUS_KEY] == False - assert saved_results['fasta_check'][RUN_STATUS_KEY] == False - assert saved_results['sample_check'][RUN_STATUS_KEY] == False - assert saved_results['metadata_check'][RUN_STATUS_KEY] == False + assert saved_results['assembly_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert saved_results['fasta_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert saved_results['sample_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert saved_results['metadata_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN assert saved_results[READY_FOR_SUBMISSION_TO_EVA] == False def test__collect_validation_workflow_results_for_validation_task_and_add_to_previous_results(self): @@ -730,7 +731,8 @@ def test__collect_trim_down_metrics_missing_yml(self): shallow_validation=True) os.remove(os.path.join(validator.output_dir, 'other_validations', 'input_passed_trim_down.yml')) validator._collect_validation_workflow_results() - assert validator.results[SHALLOW_VALIDATION][TRIM_DOWN] is False + # Missing metrics are conservatively assumed to require shallow validation + assert validator.results[SHALLOW_VALIDATION][TRIM_DOWN] is True assert validator.vcf_files[0] not in validator.results[SHALLOW_VALIDATION]['metrics'] def test__load_sample_check_results_missing_yaml(self): @@ -739,7 +741,7 @@ def test__load_sample_check_results_missing_yaml(self): validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) os.remove(os.path.join(validator.output_dir, 'other_validations', 'sample_checker.yml')) validator._load_sample_check_results() - assert validator.results[SAMPLE_CHECK][RUN_STATUS_KEY] is False + assert validator.results[SAMPLE_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED def test__load_evidence_check_results_missing_yaml(self): # Test for a nextflow run that did not complete and never produced evidence_type_checker.yml @@ -747,7 +749,7 @@ def test__load_evidence_check_results_missing_yaml(self): validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) os.remove(os.path.join(validator.output_dir, 'other_validations', 'evidence_type_checker.yml')) validator._load_evidence_check_results() - assert validator.results[EVIDENCE_TYPE_CHECK][RUN_STATUS_KEY] is False + assert validator.results[EVIDENCE_TYPE_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED def test__load_fasta_check_results_missing_yaml(self): # Test for a nextflow run that did not complete and never produced input_passed.fa_check.yml From b27d0486280ccb485780fad1cb3c59af61709c25 Mon Sep 17 00:00:00 2001 From: tcezard Date: Thu, 30 Jul 2026 10:13:41 +0100 Subject: [PATCH 3/8] Add support for handling `crashed` run_status in reports and update tests --- eva_sub_cli/jinja_templates/text/report.txt | 10 +- .../expected_metadata_json_report.txt | 16 +- .../expected_metadata_xlsx_report.txt | 16 +- ...expected_report_metadata_json_crashed.html | 144 ++++++++++++++++++ .../expected_report_metadata_json_crashed.txt | 44 ++++++ .../expected_shallow_metadata_xlsx_report.txt | 16 +- tests/test_report.py | 40 ++++- 7 files changed, 257 insertions(+), 29 deletions(-) create mode 100644 tests/resources/validation_reports/expected_report_metadata_json_crashed.html create mode 100644 tests/resources/validation_reports/expected_report_metadata_json_crashed.txt diff --git a/eva_sub_cli/jinja_templates/text/report.txt b/eva_sub_cli/jinja_templates/text/report.txt index 83ee50e..119af3c 100644 --- a/eva_sub_cli/jinja_templates/text/report.txt +++ b/eva_sub_cli/jinja_templates/text/report.txt @@ -51,8 +51,9 @@ Also checks whether the variants' reference alleles match against the reference {% if run_status == 'success' %} {% for file_name in vcf_files %} {% if file_name != "pass"%} - {{ file_name }} - {{ file_validation_report(validation_results, file_name) }} + {# Indentation is important as it impact the report's look#} + {{ file_name }} + {{ file_validation_report(validation_results, file_name) }} {% endif %} {% endfor %} {% elif run_status == 'crashed' %} @@ -85,8 +86,9 @@ Also checks if the reference assembly accession in the metadata matches the one {% if run_status == 'success' %} {% for file_name in fasta_files %} {% if file_name != "pass"%} - {{ file_name }} - {{ fasta_check_report(validation_results, file_name) }} + {# Indentation is important as it impact the report's look#} + {{ file_name }} + {{ fasta_check_report(validation_results, file_name) }} {% endif %} {% endfor %} {% elif run_status == 'crashed' %} diff --git a/tests/resources/validation_reports/expected_metadata_json_report.txt b/tests/resources/validation_reports/expected_metadata_json_report.txt index 58ac851..6ed2d15 100644 --- a/tests/resources/validation_reports/expected_metadata_json_report.txt +++ b/tests/resources/validation_reports/expected_metadata_json_report.txt @@ -72,7 +72,7 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf + input_fail.vcf ❌ Assembly check: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report @@ -89,7 +89,7 @@ Also checks whether the variants' reference alleles match against the reference First 10 errors per category are below. Full report: /path/to/vcf_failed/report Critical error: Line 4: Error in meta-data section. Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf + input_passed.vcf ✔ Assembly check: 247/247 (100.0%) ✔ VCF check: 0 critical errors, 0 non-critical errors - @@ -108,16 +108,16 @@ Checks whether information in the metadata is concordant with that contained in REFERENCE GENOME INSDC CHECK Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. - metadata_asm_match.fa + metadata_asm_match.fa ✔ All sequences are INSDC accessioned. ✔ Analysis A: Assembly accession in metadata is compatible - metadata_asm_not_found.fa + metadata_asm_not_found.fa ✔ All sequences are INSDC accessioned. ❌ No assembly accession found in metadata Full report: /path/to/metadata_asm_not_found.yml Assembly accession found in metadata: Not found Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_gca.fa + metadata_asm_not_gca.fa Warning: Non-GCA reference found in metadata. Please provide the INSDC accession for your reference assembly. If you would like to submit using a non-GCA reference sequence, contact eva-helpdesk@ebi.ac.uk. ✔ All sequences are INSDC accessioned. @@ -125,19 +125,19 @@ Also checks if the reference assembly accession in the metadata matches the one Full report: /path/to/metadata_asm_not_gca.yml Assembly accession found in metadata: GCF_1 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_match.fa + metadata_asm_not_match.fa ✔ All sequences are INSDC accessioned. ❗ Analysis B: Assembly accession in metadata is not compatible Full report: /path/to/metadata_asm_not_match.yml Assembly accession found in metadata: GCA_2 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_error.fa + metadata_error.fa Warning: The following results may be incomplete due to problems with external services. Please try again later for complete results. Error message: 500 Server Error: Internal Server Error for url: https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum/hjfdoijsfc47hfg0gh9qwjrve ✔ All sequences are INSDC accessioned. ✔ Analysis C: Assembly accession in metadata is compatible - not_all_insdc.fa + not_all_insdc.fa ❌ Some sequences are not INSDC accessioned First 10 sequences not in INSDC. Full report: /path/to/not_all_insdc_check.yml Sequence name: 2 diff --git a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt index 7360f62..c3ee9b4 100644 --- a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt @@ -66,7 +66,7 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf + input_fail.vcf ❌ Assembly check: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report @@ -83,7 +83,7 @@ Also checks whether the variants' reference alleles match against the reference First 10 errors per category are below. Full report: /path/to/vcf_failed/report Critical error: Line 4: Error in meta-data section. Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf + input_passed.vcf ✔ Assembly check: 247/247 (100.0%) ✔ VCF check: 0 critical errors, 0 non-critical errors - @@ -102,16 +102,16 @@ Checks whether information in the metadata is concordant with that contained in REFERENCE GENOME INSDC CHECK Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. - metadata_asm_match.fa + metadata_asm_match.fa ✔ All sequences are INSDC accessioned. ✔ Analysis A: Assembly accession in metadata is compatible - metadata_asm_not_found.fa + metadata_asm_not_found.fa ✔ All sequences are INSDC accessioned. ❌ No assembly accession found in metadata Full report: /path/to/metadata_asm_not_found.yml Assembly accession found in metadata: Not found Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_gca.fa + metadata_asm_not_gca.fa Warning: Non-GCA reference found in metadata. Please provide the INSDC accession for your reference assembly. If you would like to submit using a non-GCA reference sequence, contact eva-helpdesk@ebi.ac.uk. ✔ All sequences are INSDC accessioned. @@ -119,19 +119,19 @@ Also checks if the reference assembly accession in the metadata matches the one Full report: /path/to/metadata_asm_not_gca.yml Assembly accession found in metadata: GCF_1 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_match.fa + metadata_asm_not_match.fa ✔ All sequences are INSDC accessioned. ❗ Analysis B: Assembly accession in metadata is not compatible Full report: /path/to/metadata_asm_not_match.yml Assembly accession found in metadata: GCA_2 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_error.fa + metadata_error.fa Warning: The following results may be incomplete due to problems with external services. Please try again later for complete results. Error message: 500 Server Error: Internal Server Error for url: https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum/hjfdoijsfc47hfg0gh9qwjrve ✔ All sequences are INSDC accessioned. ✔ Analysis C: Assembly accession in metadata is compatible - not_all_insdc.fa + not_all_insdc.fa ❌ Some sequences are not INSDC accessioned First 10 sequences not in INSDC. Full report: /path/to/not_all_insdc_check.yml Sequence name: 2 diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.html b/tests/resources/validation_reports/expected_report_metadata_json_crashed.html new file mode 100644 index 0000000..52931e3 --- /dev/null +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.html @@ -0,0 +1,144 @@ + + + + + Validation Report + + + +
+ +
+

Validation Report

+
eva-sub-cli vcligeneratedversion
+
+
+
+

Project Summary

+
+ General details about the project +
+
+

Project Title: My cool project

+

Validation Date: 2023-08-31 12:34:56

+

Submission Directory: /test/submission/dir

+
+

Note for human genotype data:

You will need to provide a signed copy of our Consent Statement via email to eva-helpdesk@ebi.ac.uk after submission. + Your submission will only be processed after receiving the consent statement. For more info, please see our help section.

+
+
▶ Files mapping
+
+ + + + + + + + + + + + + + + + + + + + + +
VCF FileFasta FileAnalysis
input_fail.vcfinput_fail.faA
input_pass.vcfinput_pass.faB
input_test.vcfinput_test.facould not be linked
+
+ +
+
+

Metadata validation results

+
+ Ensures that required fields are present and values are formatted correctly. + For requirements, please refer to the EVA website. +
+
❗ Metadata check did not complete successfully. Check the logs for more information.
+
+
+

VCF validation results

+
+ Checks whether each file is compliant with the VCF specification. + Also checks whether the variants' reference alleles match against the reference assembly. +
+
❗ Vcf check did not complete successfully. Check the logs for more information.
+
+
+

Sample name concordance check

+
+ Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. +
+
❗ Sample name concordance did not complete successfully. Check the logs for more information.
+
+
+

Reference genome INSDC check

+
+ Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. + Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. +
+
❗ INSDC check did not complete successfully. Check the logs for more information.
+
+ + + \ No newline at end of file diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt new file mode 100644 index 0000000..8035e90 --- /dev/null +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt @@ -0,0 +1,44 @@ + +VALIDATION REPORT +eva-sub-cli vcligeneratedversion +- +PROJECT SUMMARY +General details about the project + Project Title: My cool project + Validation Date: 2023-08-31 12:34:56 + Submission Directory: /test/submission/dir + Note for human genotype data: + You will need to provide a signed copy of our Consent Statement (https://docs.google.com/document/d/1UaRmimAe919IZYIH55mDXMe-4sHsRVqaP4moqx2IYE4) via email to eva-helpdesk@ebi.ac.uk after submission. + Your submission will only be processed after receiving the consent statement.For more info, please see our help section (https://www.ebi.ac.uk/eva/?Help#submissionPanel&link=consent-statement-for-human-genotype-data). + Files mapping: + --- + VCF File: input_fail.vcf + Fasta File: input_fail.fa + Analysis: A + --- + VCF File: input_pass.vcf + Fasta File: input_pass.fa + Analysis: B + --- + VCF File: input_test.vcf + Fasta File: input_test.fa + Analysis: could not be linked +- +METADATA VALIDATION RESULTS +Ensures that required fields are present and values are formatted correctly. +For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data). +❗ Metadata check did not complete successfully. Check the logs for more information. +- +VCF VALIDATION RESULTS +Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). +Also checks whether the variants' reference alleles match against the reference assembly. +❗ Vcf check did not complete successfully. Check the logs for more information. +- +SAMPLE NAME CONCORDANCE CHECK +Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. +❗ Sample name concordance did not complete successfully. Check the logs for more information. +- +REFERENCE GENOME INSDC CHECK +Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. +Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. +❗ INSDC check did not complete successfully. Check the logs for more information. diff --git a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt index dafc26e..58cfe7f 100644 --- a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt @@ -75,7 +75,7 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf + input_fail.vcf ❌ Assembly check: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report @@ -92,7 +92,7 @@ Also checks whether the variants' reference alleles match against the reference First 10 errors per category are below. Full report: /path/to/vcf_failed/report Critical error: Line 4: Error in meta-data section. Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf + input_passed.vcf ✔ Assembly check: 247/247 (100.0%) ✔ VCF check: 0 critical errors, 0 non-critical errors - @@ -111,16 +111,16 @@ Checks whether information in the metadata is concordant with that contained in REFERENCE GENOME INSDC CHECK Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. - metadata_asm_match.fa + metadata_asm_match.fa ✔ All sequences are INSDC accessioned. ✔ Analysis A: Assembly accession in metadata is compatible - metadata_asm_not_found.fa + metadata_asm_not_found.fa ✔ All sequences are INSDC accessioned. ❌ No assembly accession found in metadata Full report: /path/to/metadata_asm_not_found.yml Assembly accession found in metadata: Not found Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_gca.fa + metadata_asm_not_gca.fa Warning: Non-GCA reference found in metadata. Please provide the INSDC accession for your reference assembly. If you would like to submit using a non-GCA reference sequence, contact eva-helpdesk@ebi.ac.uk. ✔ All sequences are INSDC accessioned. @@ -128,19 +128,19 @@ Also checks if the reference assembly accession in the metadata matches the one Full report: /path/to/metadata_asm_not_gca.yml Assembly accession found in metadata: GCF_1 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_asm_not_match.fa + metadata_asm_not_match.fa ✔ All sequences are INSDC accessioned. ❗ Analysis B: Assembly accession in metadata is not compatible Full report: /path/to/metadata_asm_not_match.yml Assembly accession found in metadata: GCA_2 Assembly accession(s) compatible with FASTA: GCA_1 - metadata_error.fa + metadata_error.fa Warning: The following results may be incomplete due to problems with external services. Please try again later for complete results. Error message: 500 Server Error: Internal Server Error for url: https://www.ebi.ac.uk/eva/webservices/contig-alias/v1/chromosomes/md5checksum/hjfdoijsfc47hfg0gh9qwjrve ✔ All sequences are INSDC accessioned. ✔ Analysis C: Assembly accession in metadata is compatible - not_all_insdc.fa + not_all_insdc.fa ❌ Some sequences are not INSDC accessioned First 10 sequences not in INSDC. Full report: /path/to/not_all_insdc_check.yml Sequence name: 2 diff --git a/tests/test_report.py b/tests/test_report.py index 9dd8e73..8b5c8ef 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -6,7 +6,8 @@ import eva_sub_cli from eva_sub_cli.report import generate_html_report, generate_text_report -from eva_sub_cli.validators.validator import RUN_STATUS_KEY, RUN_STATUS_SUCCESS, RUN_STATUS_DID_NOT_RUN, TRIM_DOWN +from eva_sub_cli.validators.validator import (RUN_STATUS_KEY, RUN_STATUS_SUCCESS, RUN_STATUS_CRASHED, + RUN_STATUS_DID_NOT_RUN, TRIM_DOWN) common_validation_results = { "ready_for_submission_to_eva": False, @@ -247,6 +248,8 @@ class TestReport(TestCase): 'expected_metadata_json_report.html') expected_report_metadata_json_process_not_run = os.path.join(resource_dir, 'validation_reports', 'expected_report_metadata_json_process_not_run.html') + expected_report_metadata_json_crashed = os.path.join(resource_dir, 'validation_reports', + 'expected_report_metadata_json_crashed.html') expected_report_metadata_xlsx_shallow = os.path.join(resource_dir, 'validation_reports', 'expected_shallow_metadata_xlsx_report.html') expected_text_report_metadata_xlsx = os.path.join(resource_dir, 'validation_reports', @@ -255,6 +258,8 @@ class TestReport(TestCase): 'expected_metadata_json_report.txt') expected_text_report_metadata_json_process_not_run = os.path.join(resource_dir, 'validation_reports', 'expected_report_metadata_json_process_not_run.txt') + expected_text_report_metadata_json_crashed = os.path.join(resource_dir, 'validation_reports', + 'expected_report_metadata_json_crashed.txt') expected_text_report_metadata_xlsx_shallow = os.path.join(resource_dir, 'validation_reports', 'expected_shallow_metadata_xlsx_report.txt') test_project_name = "My cool project" @@ -318,6 +323,22 @@ def test_generate_html_report_metadata_json_metadata_report_not_run_yet(self): self.expected_report_metadata_json_process_not_run ) + def test_generate_html_report_metadata_json_crashed(self): + validation_result = { + 'vcf_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'evidence_type_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'assembly_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'fasta_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'metadata_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'sample_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED} + } + + self.check_report_vs_expected( + validation_result, + 'metadata_json_report.html', + self.expected_report_metadata_json_crashed + ) + def test_generate_html_report_metadata_xlsx_shallow(self): shallow_validation_results_xlsx = copy.deepcopy(validation_results_xlsx) shallow_validation_results_xlsx[TRIM_DOWN] = True @@ -367,6 +388,23 @@ def test_generate_text_report_metadata_json_report_not_run(self): html=False ) + def test_generate_text_report_metadata_json_crashed(self): + validation_result = { + 'vcf_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'evidence_type_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'assembly_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'fasta_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'metadata_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'sample_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED} + } + + self.check_report_vs_expected( + validation_result, + 'metadata_json_report.txt', + self.expected_text_report_metadata_json_crashed, + html=False + ) + def test_generate_text_report_metadata_xlsx_shallow(self): shallow_validation_results_xlsx = copy.deepcopy(validation_results_xlsx) shallow_validation_results_xlsx[TRIM_DOWN] = True From 7a323474d946d8046fbebc96da4e1ee2a71b6f21 Mon Sep 17 00:00:00 2001 From: Timothee Cezard Date: Thu, 30 Jul 2026 13:44:33 +0100 Subject: [PATCH 4/8] Apply suggestions from code review Co-authored-by: April Shen --- eva_sub_cli/jinja_templates/html/report.html | 2 +- eva_sub_cli/jinja_templates/text/report.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eva_sub_cli/jinja_templates/html/report.html b/eva_sub_cli/jinja_templates/html/report.html index d48d2f6..1c38d3d 100644 --- a/eva_sub_cli/jinja_templates/html/report.html +++ b/eva_sub_cli/jinja_templates/html/report.html @@ -104,7 +104,7 @@

{{ file_name }}

{% endif %} {% endfor %} {% elif run_status == 'crashed' %} - {{ report_crashed_task('Vcf check') }} + {{ report_crashed_task('VCF check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} diff --git a/eva_sub_cli/jinja_templates/text/report.txt b/eva_sub_cli/jinja_templates/text/report.txt index 119af3c..1d27e9f 100644 --- a/eva_sub_cli/jinja_templates/text/report.txt +++ b/eva_sub_cli/jinja_templates/text/report.txt @@ -57,7 +57,7 @@ Also checks whether the variants' reference alleles match against the reference {% endif %} {% endfor %} {% elif run_status == 'crashed' %} - {{ report_crashed_task('Vcf check') }} + {{ report_crashed_task('VCF check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} From aba120917b1d7045a6bd4c89745417372118c0a0 Mon Sep 17 00:00:00 2001 From: tcezard Date: Thu, 30 Jul 2026 16:53:06 +0100 Subject: [PATCH 5/8] Replace mocked up validation error when nextflow failed with report of crash --- eva_sub_cli/validators/validator.py | 121 ++++++++++-------- ...expected_report_metadata_json_crashed.html | 2 +- .../expected_report_metadata_json_crashed.txt | 2 +- tests/test_validator.py | 50 +++++--- 4 files changed, 108 insertions(+), 67 deletions(-) diff --git a/eva_sub_cli/validators/validator.py b/eva_sub_cli/validators/validator.py index d74e913..e6b1db7 100755 --- a/eva_sub_cli/validators/validator.py +++ b/eva_sub_cli/validators/validator.py @@ -255,13 +255,15 @@ def _assess_validation_results(self): It assumes all validation have been parsed already. """ - if VCF_CHECK in self.tasks: + if VCF_CHECK in self.tasks and self.results[VCF_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # vcf_check result - vcf_check_result = all((vcf_check.get('critical_count', 1) == 0 - for vcf_name, vcf_check in self.results.get(VCF_CHECK, {}).items() - if isinstance(vcf_check, dict))) - self.results[VCF_CHECK][PASS] = vcf_check_result + self.results[VCF_CHECK][PASS] = all((vcf_check.get('critical_count', 1) == 0 + for vcf_name, vcf_check in self.results.get(VCF_CHECK, {}).items() + if isinstance(vcf_check, dict))) + elif VCF_CHECK in self.tasks: + self.results[VCF_CHECK][PASS] = False + if VCF_CHECK in self.tasks and self.results[EVIDENCE_TYPE_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # evidence type check result self.results[EVIDENCE_TYPE_CHECK][PASS] = ( any(isinstance(v, dict) for v in self.results.get(EVIDENCE_TYPE_CHECK, {}).values()) @@ -269,11 +271,14 @@ def _assess_validation_results(self): all('evidence_type' in v and v['evidence_type'] is not None for v in self.results.get(EVIDENCE_TYPE_CHECK, {}).values() if isinstance(v, dict))) - elif VCF_CHECK not in self.results: + elif VCF_CHECK in self.tasks: + self.results[EVIDENCE_TYPE_CHECK][PASS] = False + + if VCF_CHECK not in self.results: self.results[VCF_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} self.results[EVIDENCE_TYPE_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} - if ASSEMBLY_CHECK in self.tasks: + if ASSEMBLY_CHECK in self.tasks and self.results[ASSEMBLY_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # assembly_check result asm_nb_mismatch_result = all((asm_check.get('nb_mismatch', 1) == 0 for vcf_name, asm_check in self.results.get(ASSEMBLY_CHECK, {}).items() @@ -282,7 +287,10 @@ def _assess_validation_results(self): for vcf_name, asm_check in self.results.get(ASSEMBLY_CHECK, {}).items() if isinstance(asm_check, dict))) self.results[ASSEMBLY_CHECK][PASS] = asm_nb_mismatch_result and asm_nb_error_result + elif ASSEMBLY_CHECK in self.tasks: + self.results[ASSEMBLY_CHECK][PASS] = False + if ASSEMBLY_CHECK in self.tasks and self.results[FASTA_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # fasta_check result # Note this fails only if the FASTA file is not INSDC or the reference in the metadata is not a GCA. # It does not fail if the metadata assembly is not compatible with the FASTA, even though this is reported @@ -294,22 +302,29 @@ def _assess_validation_results(self): for fa_file, fa_file_check in self.results.get(FASTA_CHECK, {}).items() if isinstance(fa_file_check, dict)) self.results[FASTA_CHECK][PASS] = fasta_check_result and gca_check_result - elif ASSEMBLY_CHECK not in self.results: + elif ASSEMBLY_CHECK in self.tasks: + self.results[FASTA_CHECK][PASS] = False + + if ASSEMBLY_CHECK not in self.results: self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} self.results[FASTA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} - if SAMPLE_CHECK in self.tasks: + if SAMPLE_CHECK in self.tasks and self.results[SAMPLE_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # sample check result self.results[SAMPLE_CHECK][PASS] = self.results.get(SAMPLE_CHECK, {}).get('overall_differences', True) is False + elif SAMPLE_CHECK in self.tasks: + self.results[SAMPLE_CHECK][PASS] = False elif SAMPLE_CHECK not in self.results: self.results[SAMPLE_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} - if METADATA_CHECK in self.tasks: + if METADATA_CHECK in self.tasks and self.results[METADATA_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # metadata check result metadata_xlsx_result = len(self.results.get(METADATA_CHECK, {}).get('spreadsheet_errors', []) or []) == 0 metadata_json_result = len(self.results.get(METADATA_CHECK, {}).get('json_errors', []) or []) == 0 self.results[METADATA_CHECK][PASS] = metadata_xlsx_result and metadata_json_result + elif METADATA_CHECK in self.tasks: + self.results[METADATA_CHECK][PASS] = False elif METADATA_CHECK not in self.results: self.results[METADATA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} @@ -380,18 +395,19 @@ def _collect_vcf_check_results(self): if vcf_check_log and vcf_check_text_report: valid, warning_count, error_count, critical_count, error_list, critical_list = parse_vcf_check_report( vcf_check_text_report) + self.results[VCF_CHECK][vcf_name] = { + 'report_path': vcf_check_text_report, + 'valid': valid, + 'error_list': error_list, + 'error_count': error_count, + 'warning_count': warning_count, + 'critical_count': critical_count, + 'critical_list': critical_list + } else: - valid, warning_count, error_count, critical_count, error_list, critical_list = (False, 0, 0, 1, [], - ['Process failed']) - self.results[VCF_CHECK][vcf_name] = { - 'report_path': vcf_check_text_report, - 'valid': valid, - 'error_list': error_list, - 'error_count': error_count, - 'warning_count': warning_count, - 'critical_count': critical_count, - 'critical_list': critical_list - } + error_txt = f'Cannot locate vcf check results for {vcf_name}. The process might have failed.' + self.error(error_txt) + self.results[VCF_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) def _collect_assembly_check_results(self): # detect output files for assembly check @@ -412,17 +428,19 @@ def _collect_assembly_check_results(self): parse_assembly_check_report(assembly_check_text_report) nb_error = nb_error_from_log + nb_error_from_report error_list = error_list_from_log + error_list_from_report + self.results[ASSEMBLY_CHECK][vcf_name] = { + 'report_path': assembly_check_text_report, + 'error_list': error_list, + 'mismatch_list': mismatch_list, + 'nb_mismatch': nb_mismatch, + 'nb_error': nb_error, + 'match': match, + 'total': total + } else: - error_list, mismatch_list, nb_mismatch, nb_error, match, total = (['Process failed'], [], 0, 1, 0, 0) - self.results[ASSEMBLY_CHECK][vcf_name] = { - 'report_path': assembly_check_text_report, - 'error_list': error_list, - 'mismatch_list': mismatch_list, - 'nb_mismatch': nb_mismatch, - 'nb_error': nb_error, - 'match': match, - 'total': total - } + error_txt = f'Cannot locate assembly check results for {vcf_name}. The process might have failed.' + self.error(error_txt) + self.results[ASSEMBLY_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) def _load_fasta_check_results(self): self.results[FASTA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} @@ -434,11 +452,9 @@ def _load_fasta_check_results(self): fasta_check = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', f'{fasta_file_name}_check.yml')) if not fasta_check: - error_txt = f'Cannot locate sequence check results for {fasta_file_name}. The process might have failed.' + error_txt = f'Cannot locate fasta check results for {fasta_file_name}. The process might have failed.' self.error(error_txt) - self.results[FASTA_CHECK][fasta_file_name] = { - 'all_insdc': False, 'sequences': [], 'connection_error': error_txt - } + self.results[FASTA_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) continue with open(fasta_check) as open_yaml: self.results[FASTA_CHECK][fasta_file_name] = yaml.safe_load(open_yaml) @@ -493,14 +509,14 @@ def collect_biovalidator_validation_results(self): 'metadata_validation.txt')) if metadata_check_file: errors = parse_biovalidator_validation_results(metadata_check_file) + self.results[METADATA_CHECK].update({ + 'json_report_path': metadata_check_file, + 'json_errors': errors + }) else: - error_txt = (f"Cannot locate metadata check file. The process might have failed.") + error_txt = f"Cannot locate metadata_validation.txt. The process might have failed." self.error(error_txt) - errors = [{'property': '/', 'description': error_txt}] - self.results[METADATA_CHECK].update({ - 'json_report_path': metadata_check_file, - 'json_errors': errors - }) + self.results[METADATA_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED, 'json_errors': []}) def _collect_semantic_metadata_results(self): errors_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', @@ -587,6 +603,7 @@ def _collect_file_info_to_metadata(self): else: error_txt = f"Cannot locate file_info.txt. The process might have failed." self.error(error_txt) + self.results[METADATA_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) if self.metadata_json_post_validation: metadata = EvaMetadataJson(self.metadata_json_post_validation) @@ -601,14 +618,18 @@ def _collect_file_info_to_metadata(self): file_dict['fileSize'] = file_path_2_file_size.get(file_path) or \ file_name_2_file_size.get(file_dict.get('fileName')) or '' - if not file_dict.get('fileSize'): - error_txt = f"File size is not available for {file_dict.get('fileName')}" - self.error(error_txt) - errors.append({'property': f'/files/{file_count}.fileSize', 'description': error_txt}) - if not file_dict.get('md5'): - error_txt = f"md5 is not available for {file_dict.get('fileName')}" - self.error(error_txt) - errors.append({'property': f'/files/{file_count}.md5', 'description': error_txt}) + # Only report a per-file discrepancy if file_info.txt was found but was missing + # this specific file's entry. If the whole file is missing, the process-level + # crash reported above already covers it. + if md5sum_file: + if not file_dict.get('fileSize'): + error_txt = f"File size is not available for {file_dict.get('fileName')}" + self.error(error_txt) + errors.append({'property': f'/files/{file_count}.fileSize', 'description': error_txt}) + if not file_dict.get('md5'): + error_txt = f"md5 is not available for {file_dict.get('fileName')}" + self.error(error_txt) + errors.append({'property': f'/files/{file_count}.md5', 'description': error_txt}) file_rows.append(file_dict) file_count += 1 else: @@ -625,7 +646,7 @@ def _collect_file_info_to_metadata(self): else: error_txt = f'Cannot locate the metadata in JSON format. The process might have failed.' self.error(error_txt) - errors.append({'property': '/', 'description': error_txt}) + self.results[METADATA_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) if errors: if 'json_errors' in self.results[METADATA_CHECK]: self.results[METADATA_CHECK]['json_errors'].extend(errors) diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.html b/tests/resources/validation_reports/expected_report_metadata_json_crashed.html index 52931e3..cf55559 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_crashed.html +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.html @@ -90,7 +90,7 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. -
❗ Vcf check did not complete successfully. Check the logs for more information.
+
❗ VCF check did not complete successfully. Check the logs for more information.

Sample name concordance check

diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt index 8035e90..3fe4978 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt @@ -32,7 +32,7 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. -❗ Vcf check did not complete successfully. Check the logs for more information. +❗ VCF check did not complete successfully. Check the logs for more information. - SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. diff --git a/tests/test_validator.py b/tests/test_validator.py index a3b0f69..6139d89 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -8,7 +8,8 @@ import eva_sub_cli from eva_sub_cli.metadata import EvaMetadataJson -from eva_sub_cli.validators.validator import (Validator, VALIDATION_OUTPUT_DIR, VCF_CHECK, READY_FOR_SUBMISSION_TO_EVA, +from eva_sub_cli.validators.validator import (Validator, VALIDATION_OUTPUT_DIR, VCF_CHECK, ASSEMBLY_CHECK, + READY_FOR_SUBMISSION_TO_EVA, RUN_STATUS_KEY, RUN_STATUS_SUCCESS, RUN_STATUS_CRASHED, RUN_STATUS_DID_NOT_RUN, METADATA_CHECK, PASS, TRIM_DOWN, SHALLOW_VALIDATION, FASTA_CHECK, SAMPLE_CHECK, EVIDENCE_TYPE_CHECK) @@ -693,7 +694,13 @@ def test__collect_file_info_to_metadata_missing_file_info_txt(self): validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) os.remove(os.path.join(validator.output_dir, 'other_validations', 'file_info.txt')) self.run_collect_results(validator) - assert validator.results == self.format_data_structure(expected_validation_results) + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + # The per-file size/md5 unavailable messages are only added when file_info.txt was found but + # incomplete; when the whole file is missing the high-level crash status covers it instead. + assert not any('is not available for' in error['description'] + for error in validator.results[METADATA_CHECK]['json_errors']) + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False def test__collect_file_info_to_metadata_missing_metadata_json(self): # Test for a nextflow run that did not complete and never produced metadata.json @@ -701,11 +708,7 @@ def test__collect_file_info_to_metadata_missing_metadata_json(self): validator = self.create_validator_with_copied_output(submission_dir) os.remove(os.path.join(validator.output_dir, 'metadata.json')) validator._collect_validation_workflow_results() - expected_error = { - 'property': '/', - 'description': f'Cannot locate the metadata in JSON format. The process might have failed.' - } - assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED validator._assess_validation_results() assert validator.results[METADATA_CHECK][PASS] is False @@ -716,11 +719,7 @@ def test_collect_biovalidator_validation_results_missing_report(self): validation_tasks=[METADATA_CHECK]) os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_validation.txt')) validator._collect_validation_workflow_results() - expected_error = { - 'property': '/', - 'description': f'Cannot locate metadata check file. The process might have failed.' - } - assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED validator._assess_validation_results() assert validator.results[METADATA_CHECK][PASS] is False @@ -757,8 +756,29 @@ def test__load_fasta_check_results_missing_yaml(self): validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) os.remove(os.path.join(validator.output_dir, 'other_validations', 'input_passed.fa_check.yml')) validator._collect_validation_workflow_results() - fasta_result = validator.results[FASTA_CHECK]['input_passed.fa'] - assert fasta_result['all_insdc'] is False - assert 'connection_error' in fasta_result + assert validator.results[FASTA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + assert 'input_passed.fa' not in validator.results[FASTA_CHECK] validator._assess_validation_results() assert validator.results[FASTA_CHECK][PASS] is False + + def test__collect_vcf_check_results_missing_output(self): + # Test for a nextflow run that did not complete and never produced the vcf_format check output + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + shutil.rmtree(os.path.join(validator.output_dir, 'vcf_format')) + validator._collect_validation_workflow_results() + assert validator.results[VCF_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + assert 'input_passed.vcf' not in validator.results[VCF_CHECK] + validator._assess_validation_results() + assert validator.results[VCF_CHECK][PASS] is False + + def test__collect_assembly_check_results_missing_output(self): + # Test for a nextflow run that did not complete and never produced the assembly_check output + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + shutil.rmtree(os.path.join(validator.output_dir, 'assembly_check')) + validator._collect_validation_workflow_results() + assert validator.results[ASSEMBLY_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + assert 'input_passed.vcf' not in validator.results[ASSEMBLY_CHECK] + validator._assess_validation_results() + assert validator.results[ASSEMBLY_CHECK][PASS] is False From 0ddcf0fabe84559c36eed34f39ce80fb6163ec5e Mon Sep 17 00:00:00 2001 From: tcezard Date: Fri, 31 Jul 2026 12:54:03 +0100 Subject: [PATCH 6/8] Split the error handling between all files missing (report crash) and some files missing some present (report dummy validation errors) Split the VCF and assembly check validation report --- .../jinja_templates/html/file_validation.html | 144 ++++++++--------- eva_sub_cli/jinja_templates/html/report.html | 19 ++- .../jinja_templates/text/file_validation.txt | 101 ++++++------ eva_sub_cli/jinja_templates/text/report.txt | 24 +-- eva_sub_cli/validators/validator.py | 151 +++++++++++------- .../expected_metadata_json_report.html | 98 ++++++------ .../expected_metadata_json_report.txt | 41 +++-- .../expected_metadata_xlsx_report.html | 98 ++++++------ .../expected_metadata_xlsx_report.txt | 41 +++-- ...expected_report_metadata_json_crashed.html | 3 + .../expected_report_metadata_json_crashed.txt | 4 +- ..._report_metadata_json_process_not_run.html | 3 + ...d_report_metadata_json_process_not_run.txt | 4 +- ...expected_shallow_metadata_xlsx_report.html | 98 ++++++------ .../expected_shallow_metadata_xlsx_report.txt | 41 +++-- tests/test_validator.py | 125 +++++++++++++-- 16 files changed, 556 insertions(+), 439 deletions(-) diff --git a/eva_sub_cli/jinja_templates/html/file_validation.html b/eva_sub_cli/jinja_templates/html/file_validation.html index 9382c28..d4c7e82 100644 --- a/eva_sub_cli/jinja_templates/html/file_validation.html +++ b/eva_sub_cli/jinja_templates/html/file_validation.html @@ -1,93 +1,89 @@ -{% macro file_validation_report(validation_results, file_name) -%} - {% for check_type, check_per_file in validation_results.items() if check_type not in ["trim_down", "version"] %} - {% set result = check_per_file.get(file_name, {}) %} - {% if check_type == "assembly_check" %} - {{ assembly_check(result) }} - {% elif check_type == "vcf_check" %} - {{ vcf_check(result) }} + +{% macro vcf_check_report(vcf_check_result_per_file, vcf_file_names) -%} + {% for file_name in vcf_file_names %} + {% set vcf_check_result = vcf_check_result_per_file.get(file_name, {}) %} + {% set critical_count = vcf_check_result.get("critical_count", 0) %} + {% set error_count = vcf_check_result.get("error_count", 0) %} + {% set expand_icon = "" %} + {% if critical_count > 0 %} + {% set expand_icon = "▶" %} + {% set icon = "❌" %} + {% set row_class = "report-section fail collapsible" %} + {% elif error_count > 0 %} + {% set expand_icon = "▶" %} + {% set icon = "❗" %} + {% set row_class = "report-section warn collapsible" %} + {% else %} + {% set icon = "✔" %} + {% set row_class = "report-section pass" %} + {% endif %} +
{{ expand_icon }} {{ icon }} {{ file_name }}: {{ critical_count }} critical errors, {{ error_count }} non-critical errors
+ {% set critical_list = vcf_check_result.get("critical_list") %} + {% set error_list = vcf_check_result.get("error_list") %} + + {% if critical_list or error_list%} +
+
First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }}
+ + + + + {% for error in critical_list[:10] %} + + + + {% endfor %} + {% for error in error_list[:10] %} + + + + {% endfor %} +
CategoryError
critical error {{ error }}
non-critical error {{ error }}
+
{% endif %} {% endfor %} {%- endmacro %} -{% macro vcf_check(vcf_check_result) %} - {% set critical_count = vcf_check_result.get("critical_count", 0) %} - {% set error_count = vcf_check_result.get("error_count", 0) %} - {% set expand_icon = "" %} - {% if critical_count > 0 %} - {% set expand_icon = "▶" %} - {% set icon = "❌" %} - {% set row_class = "report-section fail collapsible" %} - {% elif error_count > 0 %} - {% set expand_icon = "▶" %} - {% set icon = "❗" %} - {% set row_class = "report-section warn collapsible" %} - {% else %} - {% set icon = "✔" %} - {% set row_class = "report-section pass" %} - {% endif %} -
{{ expand_icon }} {{ icon }} VCF check: {{ critical_count }} critical errors, {{ error_count }} non-critical errors
- {% set critical_list = vcf_check_result.get("critical_list") %} - {% set error_list = vcf_check_result.get("error_list") %} - - {% if critical_list or error_list%} -
-
First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }}
+{% macro assembly_check_report(assembly_check_result_per_file, vcf_file_names) -%} + {% for file_name in vcf_file_names %} + {% set assembly_check_result = assembly_check_result_per_file.get(file_name, {}) %} + {% set nb_match = assembly_check_result.get("match", 0) %} + {% set nb_total = assembly_check_result.get("total", 0) %} + {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %} + {% set expand_icon = "" %} + {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %} + {% set expand_icon = "▶" %} + {% set icon = "❌" %} + {% set row_class = "report-section fail collapsible" %} + {% else %} + {% set icon = "✔" %} + {% set row_class = "report-section pass" %} + {% endif %} +
{{ expand_icon }} {{ icon }} {{ file_name }}: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%)
+ {% set mismatch_list = assembly_check_result.get("mismatch_list") %} + {% set error_list = assembly_check_result.get("error_list") %} + {% if mismatch_list or error_list %} +
+
First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }}
- {% for error in critical_list[:10] %} + {% for error in error_list[:10] %} - + {% endfor %} - {% for error in error_list[:10] %} + {% for error in mismatch_list[:10] %} - + {% endfor %}
CategoryError
critical error {{ error }}Parsing Error {{ error }}
non-critical error {{ error }}mismatch error {{ error }}
-
- {% endif %} -{%- endmacro %} - -{% macro assembly_check(assembly_check_result) %} - {% set nb_match = assembly_check_result.get("match", 0) %} - {% set nb_total = assembly_check_result.get("total", 0) %} - {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %} - {% set expand_icon = "" %} - {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %} - {% set expand_icon = "▶" %} - {% set icon = "❌" %} - {% set row_class = "report-section fail collapsible" %} - {% else %} - {% set icon = "✔" %} - {% set row_class = "report-section pass" %} - {% endif %} -
{{ expand_icon }} {{ icon }} Assembly check: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%)
- {% set mismatch_list = assembly_check_result.get("mismatch_list") %} - {% set error_list = assembly_check_result.get("error_list") %} - {% if mismatch_list or error_list %} -
-
First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }}
- - - - - {% for error in error_list[:10] %} - - - - {% endfor %} - {% for error in mismatch_list[:10] %} - - - - {% endfor %} -
CategoryError
Parsing Error {{ error }}
mismatch error {{ error }}
-
- {% endif %} +
+ {% endif %} + {% endfor %} {%- endmacro %} diff --git a/eva_sub_cli/jinja_templates/html/report.html b/eva_sub_cli/jinja_templates/html/report.html index 1c38d3d..2f03d5b 100644 --- a/eva_sub_cli/jinja_templates/html/report.html +++ b/eva_sub_cli/jinja_templates/html/report.html @@ -1,6 +1,6 @@ {% from 'project_details.html' import project_details_report %} -{% from 'file_validation.html' import file_validation_report %} +{% from 'file_validation.html' import assembly_check_report, vcf_check_report %} {% from 'sample_name_check.html' import sample_name_check_report %} {% from 'fasta_check.html' import fasta_check_report %} {% from 'metadata_validation.html' import metadata_validation_report %} @@ -95,14 +95,19 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. +

Assembly check

+ {% set run_status = validation_results.get('assembly_check', {}).get('run_status', '') %} + {% if run_status == 'success' %} + {{ assembly_check_report(validation_results.get('assembly_check', {}), vcf_files) }} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('Assembly check') }} + {% else %} + {{ validation_not_run_yet_message() }} + {% endif %} +

VCF check

{% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %} {% if run_status == 'success' %} - {% for file_name in vcf_files %} - {% if file_name != "pass"%} -

{{ file_name }}

- {{ file_validation_report(validation_results, file_name) }} - {% endif %} - {% endfor %} + {{ vcf_check_report(validation_results.get('vcf_check', {}), vcf_files) }} {% elif run_status == 'crashed' %} {{ report_crashed_task('VCF check') }} {% else %} diff --git a/eva_sub_cli/jinja_templates/text/file_validation.txt b/eva_sub_cli/jinja_templates/text/file_validation.txt index f6c9632..1838429 100644 --- a/eva_sub_cli/jinja_templates/text/file_validation.txt +++ b/eva_sub_cli/jinja_templates/text/file_validation.txt @@ -1,60 +1,55 @@ -{% macro file_validation_report(validation_results, file_name) -%} - {% for check_type, check_per_file in validation_results.items() if check_type not in ["trim_down", "version"] %} - {% set result = check_per_file.get(file_name, {}) %} - {% if check_type == "assembly_check" %} - {{ assembly_check(result) }} - {% elif check_type == "vcf_check" %} - {{ vcf_check(result) }} +{% macro vcf_check_report(vcf_check_result_per_file, vcf_file_names) -%} + {% for file_name in vcf_file_names %} + {% set vcf_check_result = vcf_check_result_per_file.get(file_name, {}) %} + {% set critical_count = vcf_check_result.get("critical_count", 0) %} + {% set error_count = vcf_check_result.get("error_count", 0) %} + {% if critical_count > 0 %} + {% set icon = "\u274C" %} + {% elif error_count > 0 %} + {% set icon = "\u26A0" %} + {% else %} + {% set icon = "\u2714" %} {% endif %} - {% endfor %} -{%- endmacro %} + {{ icon }} {{ file_name }}: {{ critical_count }} critical errors, {{ error_count }} non-critical errors -{% macro vcf_check(vcf_check_result) %} - {% set critical_count = vcf_check_result.get("critical_count", 0) %} - {% set error_count = vcf_check_result.get("error_count", 0) %} - {% if critical_count > 0 %} - {% set icon = "\u274C" %} - {% elif error_count > 0 %} - {% set icon = "\u26A0" %} - {% else %} - {% set icon = "\u2714" %} - {% endif %} - {{ icon }} VCF check: {{ critical_count }} critical errors, {{ error_count }} non-critical errors - - {% set critical_list = vcf_check_result.get("critical_list") %} - {% set error_list = vcf_check_result.get("error_list") %} - {% if critical_list or error_list%} - First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }} - {% for error in critical_list[:10] %} - Critical error: {{ error }} - {% endfor %} - {% for error in error_list[:10] %} - Non-critical error: {{ error }} - {% endfor %} - {% endif %} + {% set critical_list = vcf_check_result.get("critical_list") %} + {% set error_list = vcf_check_result.get("error_list") %} + {% if critical_list or error_list%} + First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }} + {% for error in critical_list[:10] %} + Critical error: {{ error }} + {% endfor %} + {% for error in error_list[:10] %} + Non-critical error: {{ error }} + {% endfor %} + {% endif %} + {% endfor %} {%- endmacro %} -{% macro assembly_check(assembly_check_result) %} - {% set nb_match = assembly_check_result.get("match", 0) %} - {% set nb_total = assembly_check_result.get("total", 0) %} - {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %} - {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %} - {% set icon = "\u274C" %} - {% else %} - {% set icon = "\u2714" %} - {% endif %} - {{ icon }} Assembly check: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%) +{% macro assembly_check_report(assembly_check_result_per_file, vcf_file_names) -%} + {% for file_name in vcf_file_names %} + {% set assembly_check_result = assembly_check_result_per_file.get(file_name, {}) %} + {% set nb_match = assembly_check_result.get("match", 0) %} + {% set nb_total = assembly_check_result.get("total", 0) %} + {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %} + {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %} + {% set icon = "\u274C" %} + {% else %} + {% set icon = "\u2714" %} + {% endif %} + {{ icon }} {{ file_name }}: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%) - {% set mismatch_list = assembly_check_result.get("mismatch_list") %} - {% set error_list = assembly_check_result.get("error_list") %} - {% if mismatch_list or error_list %} - First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }} - {% for error in error_list[:10] %} - Parsing error: {{ error }} - {% endfor %} - {% for error in mismatch_list[:10] %} - Mismatch error: {{ error }} - {% endfor %} - {% endif %} + {% set mismatch_list = assembly_check_result.get("mismatch_list") %} + {% set error_list = assembly_check_result.get("error_list") %} + {% if mismatch_list or error_list %} + First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }} + {% for error in error_list[:10] %} + Parsing error: {{ error }} + {% endfor %} + {% for error in mismatch_list[:10] %} + Mismatch error: {{ error }} + {% endfor %} + {% endif %} + {% endfor %} {%- endmacro %} diff --git a/eva_sub_cli/jinja_templates/text/report.txt b/eva_sub_cli/jinja_templates/text/report.txt index 1d27e9f..53481bf 100644 --- a/eva_sub_cli/jinja_templates/text/report.txt +++ b/eva_sub_cli/jinja_templates/text/report.txt @@ -1,5 +1,5 @@ {% from 'project_details.txt' import project_details_report %} -{% from 'file_validation.txt' import file_validation_report %} +{% from 'file_validation.txt' import assembly_check_report, vcf_check_report %} {% from 'sample_name_check.txt' import sample_name_check_report %} {% from 'fasta_check.txt' import fasta_check_report %} {% from 'metadata_validation.txt' import metadata_validation_report %} @@ -41,21 +41,23 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su {% else %} {{ validation_not_run_yet_message() }} {% endif %} + - -VCF VALIDATION RESULTS -Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). -Also checks whether the variants' reference alleles match against the reference assembly. +ASSEMBLY CHECK RESULTS +{% set run_status = validation_results.get('assembly_check', {}).get('run_status', '') %} +{% if run_status == 'success' %} + {{ assembly_check_report(validation_results.get('assembly_check', {}), vcf_files) }} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('Assembly check') }} +{% else %} + {{ validation_not_run_yet_message() }} +{% endif %} +VCF VALIDATION RESULTS {% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %} {% if run_status == 'success' %} - {% for file_name in vcf_files %} - {% if file_name != "pass"%} - {# Indentation is important as it impact the report's look#} - {{ file_name }} - {{ file_validation_report(validation_results, file_name) }} - {% endif %} - {% endfor %} + {{ vcf_check_report(validation_results.get('vcf_check', {}), vcf_files) }} {% elif run_status == 'crashed' %} {{ report_crashed_task('VCF check') }} {% else %} diff --git a/eva_sub_cli/validators/validator.py b/eva_sub_cli/validators/validator.py index e6b1db7..02bc2a8 100755 --- a/eva_sub_cli/validators/validator.py +++ b/eva_sub_cli/validators/validator.py @@ -386,28 +386,32 @@ def _collect_vcf_check_results(self): if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[VCF_CHECK].update({TRIM_DOWN: True}) - for vcf_file in self.vcf_files: - vcf_name = os.path.basename(vcf_file) - - vcf_check_log = self._vcf_check_log(vcf_name) - vcf_check_text_report = self._vcf_check_text_report(vcf_name) + vcf_names = [os.path.basename(vcf_file) for vcf_file in self.vcf_files] + per_vcf_paths = {vcf_name: (self._vcf_check_log(vcf_name), self._vcf_check_text_report(vcf_name)) + for vcf_name in vcf_names} + if not any(log and report for log, report in per_vcf_paths.values()): + self.error('Cannot locate any vcf check results. The process might have failed.') + self.results[VCF_CHECK][RUN_STATUS_KEY] = RUN_STATUS_CRASHED + return + for vcf_name, (vcf_check_log, vcf_check_text_report) in per_vcf_paths.items(): if vcf_check_log and vcf_check_text_report: valid, warning_count, error_count, critical_count, error_list, critical_list = parse_vcf_check_report( vcf_check_text_report) - self.results[VCF_CHECK][vcf_name] = { - 'report_path': vcf_check_text_report, - 'valid': valid, - 'error_list': error_list, - 'error_count': error_count, - 'warning_count': warning_count, - 'critical_count': critical_count, - 'critical_list': critical_list - } else: error_txt = f'Cannot locate vcf check results for {vcf_name}. The process might have failed.' self.error(error_txt) - self.results[VCF_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) + valid, warning_count, error_count, critical_count, error_list, critical_list = (False, 0, 0, 1, [], + ['Process failed']) + self.results[VCF_CHECK][vcf_name] = { + 'report_path': vcf_check_text_report, + 'valid': valid, + 'error_list': error_list, + 'error_count': error_count, + 'warning_count': warning_count, + 'critical_count': critical_count, + 'critical_list': critical_list + } def _collect_assembly_check_results(self): # detect output files for assembly check @@ -415,12 +419,15 @@ def _collect_assembly_check_results(self): if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[ASSEMBLY_CHECK].update({TRIM_DOWN: True}) - for vcf_file in self.vcf_files: - vcf_name = os.path.basename(vcf_file) - - assembly_check_log = self._assembly_check_log(vcf_name) - assembly_check_text_report = self._assembly_check_text_report(vcf_name) + vcf_names = [os.path.basename(vcf_file) for vcf_file in self.vcf_files] + per_vcf_paths = {vcf_name: (self._assembly_check_log(vcf_name), self._assembly_check_text_report(vcf_name)) + for vcf_name in vcf_names} + if not any(log and report for log, report in per_vcf_paths.values()): + self.error('Cannot locate any assembly check results. The process might have failed.') + self.results[ASSEMBLY_CHECK][RUN_STATUS_KEY] = RUN_STATUS_CRASHED + return + for vcf_name, (assembly_check_log, assembly_check_text_report) in per_vcf_paths.items(): if assembly_check_log and assembly_check_text_report: error_list_from_log, nb_error_from_log, match, total = \ parse_assembly_check_log(assembly_check_log) @@ -428,33 +435,39 @@ def _collect_assembly_check_results(self): parse_assembly_check_report(assembly_check_text_report) nb_error = nb_error_from_log + nb_error_from_report error_list = error_list_from_log + error_list_from_report - self.results[ASSEMBLY_CHECK][vcf_name] = { - 'report_path': assembly_check_text_report, - 'error_list': error_list, - 'mismatch_list': mismatch_list, - 'nb_mismatch': nb_mismatch, - 'nb_error': nb_error, - 'match': match, - 'total': total - } else: - error_txt = f'Cannot locate assembly check results for {vcf_name}. The process might have failed.' - self.error(error_txt) - self.results[ASSEMBLY_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) + self.error(f'Cannot locate assembly check results for {vcf_name}. The process might have failed.') + error_list, mismatch_list, nb_mismatch, nb_error, match, total = (['Process failed'], [], 0, 1, 0, 0) + self.results[ASSEMBLY_CHECK][vcf_name] = { + 'report_path': assembly_check_text_report, + 'error_list': error_list, + 'mismatch_list': mismatch_list, + 'nb_mismatch': nb_mismatch, + 'nb_error': nb_error, + 'match': match, + 'total': total + } def _load_fasta_check_results(self): self.results[FASTA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[FASTA_CHECK].update({TRIM_DOWN: True}) - for fasta_file in self.fasta_files: - fasta_file_name = os.path.basename(fasta_file) - fasta_check = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', - f'{fasta_file_name}_check.yml')) + fasta_check_paths = {os.path.basename(fasta_file): resolve_single_file_path( + os.path.join(self.output_dir, 'other_validations', f'{os.path.basename(fasta_file)}_check.yml')) + for fasta_file in self.fasta_files} + if not any(fasta_check_paths.values()): + self.error('Cannot locate any fasta check results. The process might have failed.') + self.results[FASTA_CHECK][RUN_STATUS_KEY] = RUN_STATUS_CRASHED + return + + for fasta_file_name, fasta_check in fasta_check_paths.items(): if not fasta_check: error_txt = f'Cannot locate fasta check results for {fasta_file_name}. The process might have failed.' self.error(error_txt) - self.results[FASTA_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) + self.results[FASTA_CHECK][fasta_file_name] = { + 'all_insdc': False, 'sequences': [], 'connection_error': error_txt + } continue with open(fasta_check) as open_yaml: self.results[FASTA_CHECK][fasta_file_name] = yaml.safe_load(open_yaml) @@ -484,15 +497,21 @@ def _load_evidence_check_results(self): def _collect_metadata_results(self): self.results[METADATA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} + self._load_spreadsheet_conversion_errors() - self.collect_biovalidator_validation_results() - self._collect_semantic_metadata_results() + metadata_validation_file = self.collect_biovalidator_validation_results() + metadata_semantic_file = self._collect_semantic_metadata_results() if self.metadata_xlsx: self._convert_biovalidator_validation_to_spreadsheet() self._write_spreadsheet_validation_results() - self._collect_file_info_to_metadata() + file_info_file = self._collect_file_info_to_metadata() self._add_schema_to_metadata() + if not any([metadata_validation_file, metadata_semantic_file, file_info_file, + self.metadata_json_post_validation]): + self.error('Cannot locate any metadata check output. The process might have failed.') + self.results[METADATA_CHECK][RUN_STATUS_KEY] = RUN_STATUS_CRASHED + def _load_spreadsheet_conversion_errors(self): errors_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'metadata_conversion_errors.yml')) @@ -503,31 +522,41 @@ def _load_spreadsheet_conversion_errors(self): def collect_biovalidator_validation_results(self): """ - Read the biovalidator's report and extract the list of validation errors + Read the biovalidator's report and extract the list of validation errors. + Returns the resolved report path, or None if it could not be located. """ metadata_check_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'metadata_validation.txt')) if metadata_check_file: errors = parse_biovalidator_validation_results(metadata_check_file) - self.results[METADATA_CHECK].update({ - 'json_report_path': metadata_check_file, - 'json_errors': errors - }) else: error_txt = f"Cannot locate metadata_validation.txt. The process might have failed." self.error(error_txt) - self.results[METADATA_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED, 'json_errors': []}) + errors = [{'property': '/', 'description': error_txt}] + self.results[METADATA_CHECK].update({ + 'json_report_path': metadata_check_file, + 'json_errors': errors + }) + return metadata_check_file def _collect_semantic_metadata_results(self): + """ + Read the semantic check's report and merge its errors into the biovalidator's. + Returns the resolved report path, or None if it could not be located. + """ errors_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'metadata_semantic_check.yml')) if not errors_file: - return + error_txt = 'Cannot locate metadata_semantic_check.yml. The process might have failed.' + self.error(error_txt) + self.results[METADATA_CHECK]['json_errors'].append({'property': '/', 'description': error_txt}) + return None with open(errors_file) as open_yaml: # errors is a list of dicts matching format of biovalidator errors errors = yaml.safe_load(open_yaml) # biovalidator error parsing always places a list here, even if no errors self.results[METADATA_CHECK]['json_errors'] += errors + return errors_file def _convert_biovalidator_validation_to_spreadsheet(self): config_file = os.path.join(ETC_DIR, "spreadsheet2json_conf.yaml") @@ -583,6 +612,10 @@ def _write_spreadsheet_validation_results(self): self.results[METADATA_CHECK]['spreadsheet_report_path'] = spreadsheet_report_file def _collect_file_info_to_metadata(self): + """ + Attach md5/file size info to each file in the metadata. Returns the resolved file_info.txt + path, or None if it could not be located. + """ errors = [] md5sum_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'file_info.txt')) file_path_2_md5 = {} @@ -603,7 +636,6 @@ def _collect_file_info_to_metadata(self): else: error_txt = f"Cannot locate file_info.txt. The process might have failed." self.error(error_txt) - self.results[METADATA_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) if self.metadata_json_post_validation: metadata = EvaMetadataJson(self.metadata_json_post_validation) @@ -618,18 +650,14 @@ def _collect_file_info_to_metadata(self): file_dict['fileSize'] = file_path_2_file_size.get(file_path) or \ file_name_2_file_size.get(file_dict.get('fileName')) or '' - # Only report a per-file discrepancy if file_info.txt was found but was missing - # this specific file's entry. If the whole file is missing, the process-level - # crash reported above already covers it. - if md5sum_file: - if not file_dict.get('fileSize'): - error_txt = f"File size is not available for {file_dict.get('fileName')}" - self.error(error_txt) - errors.append({'property': f'/files/{file_count}.fileSize', 'description': error_txt}) - if not file_dict.get('md5'): - error_txt = f"md5 is not available for {file_dict.get('fileName')}" - self.error(error_txt) - errors.append({'property': f'/files/{file_count}.md5', 'description': error_txt}) + if not file_dict.get('fileSize'): + error_txt = f"File size is not available for {file_dict.get('fileName')}" + self.error(error_txt) + errors.append({'property': f'/files/{file_count}.fileSize', 'description': error_txt}) + if not file_dict.get('md5'): + error_txt = f"md5 is not available for {file_dict.get('fileName')}" + self.error(error_txt) + errors.append({'property': f'/files/{file_count}.md5', 'description': error_txt}) file_rows.append(file_dict) file_count += 1 else: @@ -646,12 +674,13 @@ def _collect_file_info_to_metadata(self): else: error_txt = f'Cannot locate the metadata in JSON format. The process might have failed.' self.error(error_txt) - self.results[METADATA_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) + errors.append({'property': '/', 'description': error_txt}) if errors: if 'json_errors' in self.results[METADATA_CHECK]: self.results[METADATA_CHECK]['json_errors'].extend(errors) else: self.results[METADATA_CHECK]['json_errors'] = errors + return md5sum_file def _add_schema_to_metadata(self): if self.metadata_json_post_validation: diff --git a/tests/resources/validation_reports/expected_metadata_json_report.html b/tests/resources/validation_reports/expected_metadata_json_report.html index faa3393..8ed34e9 100644 --- a/tests/resources/validation_reports/expected_metadata_json_report.html +++ b/tests/resources/validation_reports/expected_metadata_json_report.html @@ -162,64 +162,64 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. -

input_fail.vcf

-
▶ ❌ Assembly check: 26/36 (72.22%)
-
-
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CategoryError
Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-
▶ ❌ VCF check: 1 critical errors, 1 non-critical errors
-
-
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+

Assembly check

+
▶ ❌ input_fail.vcf: 26/36 (72.22%)
+
+
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- + - + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-

input_passed.vcf

-
✔ Assembly check: 247/247 (100.0%)
-
✔ VCF check: 0 critical errors, 0 non-critical errors
+
+
✔ input_passed.vcf: 247/247 (100.0%)
+

VCF check

+
▶ ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors
+
+
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+ + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..
+
+
✔ input_passed.vcf: 0 critical errors, 0 non-critical errors

Sample name concordance check

diff --git a/tests/resources/validation_reports/expected_metadata_json_report.txt b/tests/resources/validation_reports/expected_metadata_json_report.txt index 6ed2d15..97b68e5 100644 --- a/tests/resources/validation_reports/expected_metadata_json_report.txt +++ b/tests/resources/validation_reports/expected_metadata_json_report.txt @@ -69,29 +69,26 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su Error: should match exactly one schema in oneOf --- - +ASSEMBLY CHECK RESULTS + ❌ input_fail.vcf: 26/36 (72.22%) + First 10 errors per category are below. Full report: /path/to/assembly_failed/report + Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report + Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' + ✔ input_passed.vcf: 247/247 (100.0%) VCF VALIDATION RESULTS -Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). -Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf - ❌ Assembly check: 26/36 (72.22%) - First 10 errors per category are below. Full report: /path/to/assembly_failed/report - Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report - Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' - ❌ VCF check: 1 critical errors, 1 non-critical errors - First 10 errors per category are below. Full report: /path/to/vcf_failed/report - Critical error: Line 4: Error in meta-data section. - Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf - ✔ Assembly check: 247/247 (100.0%) - ✔ VCF check: 0 critical errors, 0 non-critical errors + ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors + First 10 errors per category are below. Full report: /path/to/vcf_failed/report + Critical error: Line 4: Error in meta-data section. + Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. + ✔ input_passed.vcf: 0 critical errors, 0 non-critical errors - SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. diff --git a/tests/resources/validation_reports/expected_metadata_xlsx_report.html b/tests/resources/validation_reports/expected_metadata_xlsx_report.html index 9c2da12..16f5cfc 100644 --- a/tests/resources/validation_reports/expected_metadata_xlsx_report.html +++ b/tests/resources/validation_reports/expected_metadata_xlsx_report.html @@ -174,64 +174,64 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. -

input_fail.vcf

-
▶ ❌ Assembly check: 26/36 (72.22%)
-
-
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CategoryError
Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-
▶ ❌ VCF check: 1 critical errors, 1 non-critical errors
-
-
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+

Assembly check

+
▶ ❌ input_fail.vcf: 26/36 (72.22%)
+
+
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- + - + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-

input_passed.vcf

-
✔ Assembly check: 247/247 (100.0%)
-
✔ VCF check: 0 critical errors, 0 non-critical errors
+
+
✔ input_passed.vcf: 247/247 (100.0%)
+

VCF check

+
▶ ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors
+
+
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+ + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..
+
+
✔ input_passed.vcf: 0 critical errors, 0 non-critical errors

Sample name concordance check

diff --git a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt index c3ee9b4..0e5b510 100644 --- a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt @@ -63,29 +63,26 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su Error: Column "Sample Accession" is not populated --- - +ASSEMBLY CHECK RESULTS + ❌ input_fail.vcf: 26/36 (72.22%) + First 10 errors per category are below. Full report: /path/to/assembly_failed/report + Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report + Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' + ✔ input_passed.vcf: 247/247 (100.0%) VCF VALIDATION RESULTS -Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). -Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf - ❌ Assembly check: 26/36 (72.22%) - First 10 errors per category are below. Full report: /path/to/assembly_failed/report - Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report - Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' - ❌ VCF check: 1 critical errors, 1 non-critical errors - First 10 errors per category are below. Full report: /path/to/vcf_failed/report - Critical error: Line 4: Error in meta-data section. - Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf - ✔ Assembly check: 247/247 (100.0%) - ✔ VCF check: 0 critical errors, 0 non-critical errors + ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors + First 10 errors per category are below. Full report: /path/to/vcf_failed/report + Critical error: Line 4: Error in meta-data section. + Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. + ✔ input_passed.vcf: 0 critical errors, 0 non-critical errors - SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.html b/tests/resources/validation_reports/expected_report_metadata_json_crashed.html index cf55559..69de034 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_crashed.html +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.html @@ -90,6 +90,9 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. +

Assembly check

+
❗ Assembly check did not complete successfully. Check the logs for more information.
+

VCF check

❗ VCF check did not complete successfully. Check the logs for more information.
diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt index 3fe4978..7933afe 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt @@ -29,9 +29,9 @@ Ensures that required fields are present and values are formatted correctly. For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data). ❗ Metadata check did not complete successfully. Check the logs for more information. - +ASSEMBLY CHECK RESULTS +❗ Assembly check did not complete successfully. Check the logs for more information. VCF VALIDATION RESULTS -Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). -Also checks whether the variants' reference alleles match against the reference assembly. ❗ VCF check did not complete successfully. Check the logs for more information. - SAMPLE NAME CONCORDANCE CHECK diff --git a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html index e9546f8..e3ad393 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html +++ b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html @@ -90,6 +90,9 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. +

Assembly check

+
⏸ Process not run yet
+

VCF check

⏸ Process not run yet
diff --git a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt index 82cb4d5..dee5643 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt +++ b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt @@ -29,9 +29,9 @@ Ensures that required fields are present and values are formatted correctly. For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data). ⏸ Process not run yet - +ASSEMBLY CHECK RESULTS +⏸ Process not run yet VCF VALIDATION RESULTS -Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). -Also checks whether the variants' reference alleles match against the reference assembly. ⏸ Process not run yet - SAMPLE NAME CONCORDANCE CHECK diff --git a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html index 031bcc2..e62e1a9 100644 --- a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html +++ b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html @@ -198,64 +198,64 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. -

input_fail.vcf

-
▶ ❌ Assembly check: 26/36 (72.22%)
-
-
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CategoryError
Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-
▶ ❌ VCF check: 1 critical errors, 1 non-critical errors
-
-
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+

Assembly check

+
▶ ❌ input_fail.vcf: 26/36 (72.22%)
+
+
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- + + + + + + + + + + + + + + + + + + + - + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-

input_passed.vcf

-
✔ Assembly check: 247/247 (100.0%)
-
✔ VCF check: 0 critical errors, 0 non-critical errors
+
+
✔ input_passed.vcf: 247/247 (100.0%)
+

VCF check

+
▶ ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors
+
+
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+ + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..
+
+
✔ input_passed.vcf: 0 critical errors, 0 non-critical errors

Sample name concordance check

diff --git a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt index 58cfe7f..c90c2d4 100644 --- a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt @@ -72,29 +72,26 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su Error: Column "Sample Accession" is not populated --- - +ASSEMBLY CHECK RESULTS + ❌ input_fail.vcf: 26/36 (72.22%) + First 10 errors per category are below. Full report: /path/to/assembly_failed/report + Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report + Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' + ✔ input_passed.vcf: 247/247 (100.0%) VCF VALIDATION RESULTS -Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). -Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf - ❌ Assembly check: 26/36 (72.22%) - First 10 errors per category are below. Full report: /path/to/assembly_failed/report - Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report - Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' - ❌ VCF check: 1 critical errors, 1 non-critical errors - First 10 errors per category are below. Full report: /path/to/vcf_failed/report - Critical error: Line 4: Error in meta-data section. - Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf - ✔ Assembly check: 247/247 (100.0%) - ✔ VCF check: 0 critical errors, 0 non-critical errors + ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors + First 10 errors per category are below. Full report: /path/to/vcf_failed/report + Critical error: Line 4: Error in meta-data section. + Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. + ✔ input_passed.vcf: 0 critical errors, 0 non-critical errors - SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. diff --git a/tests/test_validator.py b/tests/test_validator.py index 6139d89..aeebf13 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -188,18 +188,28 @@ def create_validator(self, submission_dir): return Validator(mapping_file, submission_dir, metadata_json=self.metadata_json_file) def create_validator_with_copied_output(self, submission_dir, metadata_json=None, metadata_xlsx=None, - shallow_validation=False, validation_tasks=None): + shallow_validation=False, validation_tasks=None, + include_second_vcf_with_no_output=False): """ Copy the shared validation_output fixture tree into a fresh submission_dir so a test can delete individual output files to simulate an incomplete Nextflow run without affecting other tests. + + If include_second_vcf_with_no_output is True, a second VCF/FASTA entry ("missing_output.*") + is added to the mapping file with no corresponding check output on disk, so a test can exercise + the "some files present, some missing" partial scenario. The extra input files don't need to + exist on disk since these tests call the collection methods directly, not verify_files_present(). """ shutil.copytree(os.path.join(self.output_dir, VALIDATION_OUTPUT_DIR), os.path.join(submission_dir, VALIDATION_OUTPUT_DIR)) mapping_file = os.path.join(submission_dir, 'vcf_files_mapping.csv') - create_mapping_file(mapping_file, - [os.path.join(self.vcf_files, 'input_passed.vcf')], - [os.path.join(self.fasta_files, 'input_passed.fa')], - [os.path.join(self.assembly_reports, 'input_passed.txt')]) + vcf_files = [os.path.join(self.vcf_files, 'input_passed.vcf')] + fasta_files = [os.path.join(self.fasta_files, 'input_passed.fa')] + assembly_reports = [os.path.join(self.assembly_reports, 'input_passed.txt')] + if include_second_vcf_with_no_output: + vcf_files.append(os.path.join(self.vcf_files, 'missing_output.vcf')) + fasta_files.append(os.path.join(self.fasta_files, 'missing_output.fa')) + assembly_reports.append('') + create_mapping_file(mapping_file, vcf_files, fasta_files, assembly_reports) kwargs = {} if validation_tasks is not None: kwargs['validation_tasks'] = validation_tasks @@ -689,40 +699,83 @@ def test__check_consent_statement_is_needed_for_submission(self): assert self.validator_json._check_consent_statement_is_needed_for_submission() is True def test__collect_file_info_to_metadata_missing_file_info_txt(self): - # Test for a nextflow run that did not complete and never produced file_info.txt + # Test for a nextflow run that did not complete and never produced file_info.txt, but other + # metadata check outputs are present -> task should not crash, only the per-file md5/size + # entries should be reported as missing (same as when file_info.txt exists but is empty). with TemporaryDirectory() as submission_dir: validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) os.remove(os.path.join(validator.output_dir, 'other_validations', 'file_info.txt')) self.run_collect_results(validator) - assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED - # The per-file size/md5 unavailable messages are only added when file_info.txt was found but - # incomplete; when the whole file is missing the high-level crash status covers it instead. - assert not any('is not available for' in error['description'] - for error in validator.results[METADATA_CHECK]['json_errors']) - validator._assess_validation_results() - assert validator.results[METADATA_CHECK][PASS] is False + assert validator.results == self.format_data_structure(expected_validation_results) def test__collect_file_info_to_metadata_missing_metadata_json(self): - # Test for a nextflow run that did not complete and never produced metadata.json + # Test for a nextflow run that did not complete and never produced metadata.json, but other + # metadata check outputs are present -> task should not crash, only a dummy error for the + # missing metadata.json should be added. with TemporaryDirectory() as submission_dir: validator = self.create_validator_with_copied_output(submission_dir) os.remove(os.path.join(validator.output_dir, 'metadata.json')) validator._collect_validation_workflow_results() - assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + expected_error = { + 'property': '/', + 'description': 'Cannot locate the metadata in JSON format. The process might have failed.' + } + assert expected_error in validator.results[METADATA_CHECK]['json_errors'] validator._assess_validation_results() assert validator.results[METADATA_CHECK][PASS] is False def test_collect_biovalidator_validation_results_missing_report(self): - # Test for a nextflow run that did not complete and never produced metadata_validation.txt + # Test for a nextflow run that did not complete and never produced metadata_validation.txt, + # but other metadata check outputs are present -> task should not crash, only a dummy error + # for the missing biovalidator report should be added. with TemporaryDirectory() as submission_dir: validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file, validation_tasks=[METADATA_CHECK]) os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_validation.txt')) validator._collect_validation_workflow_results() + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + expected_error = { + 'property': '/', + 'description': 'Cannot locate metadata_validation.txt. The process might have failed.' + } + assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False + + def test__collect_metadata_results_all_files_missing(self): + # Test for a nextflow run that never completed any part of the metadata check -> the whole + # task should be reported as crashed, even though each sub-collector still records its own + # diagnostic dummy error for whichever specific file it was looking for. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, validation_tasks=[METADATA_CHECK]) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'file_info.txt')) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_validation.txt')) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_semantic_check.yml')) + os.remove(os.path.join(validator.output_dir, 'metadata.json')) + validator._collect_validation_workflow_results() assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED validator._assess_validation_results() assert validator.results[METADATA_CHECK][PASS] is False + def test__collect_semantic_metadata_results_missing_yaml(self): + # Test for a nextflow run that did not complete and never produced metadata_semantic_check.yml, + # but other metadata check outputs are present -> task should not crash, only a dummy error + # for the missing semantic check should be added. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file, + validation_tasks=[METADATA_CHECK]) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_semantic_check.yml')) + validator._collect_validation_workflow_results() + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + expected_error = { + 'property': '/', + 'description': 'Cannot locate metadata_semantic_check.yml. The process might have failed.' + } + assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False + def test__collect_trim_down_metrics_missing_yml(self): # Test for a nextflow run that did not complete and never produced input_passed_trim_down.yml with TemporaryDirectory() as submission_dir: @@ -782,3 +835,43 @@ def test__collect_assembly_check_results_missing_output(self): assert 'input_passed.vcf' not in validator.results[ASSEMBLY_CHECK] validator._assess_validation_results() assert validator.results[ASSEMBLY_CHECK][PASS] is False + + def test__collect_vcf_check_results_partial_output(self): + # One of two VCF files has no vcf_format output while the other succeeds -> the task should + # not crash; only the missing file gets a synthetic failing entry. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output( + submission_dir, metadata_json=self.metadata_json_file, include_second_vcf_with_no_output=True) + validator._collect_validation_workflow_results() + assert validator.results[VCF_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + assert validator.results[VCF_CHECK]['input_passed.vcf']['valid'] is True + assert validator.results[VCF_CHECK]['missing_output.vcf']['critical_list'] == ['Process failed'] + validator._assess_validation_results() + assert validator.results[VCF_CHECK][PASS] is False + + def test__collect_assembly_check_results_partial_output(self): + # One of two VCF files has no assembly_check output while the other succeeds -> the task + # should not crash; only the missing file gets a synthetic failing entry. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output( + submission_dir, metadata_json=self.metadata_json_file, include_second_vcf_with_no_output=True) + validator._collect_validation_workflow_results() + assert validator.results[ASSEMBLY_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + assert validator.results[ASSEMBLY_CHECK]['input_passed.vcf']['nb_error'] == 0 + assert validator.results[ASSEMBLY_CHECK]['missing_output.vcf']['error_list'] == ['Process failed'] + validator._assess_validation_results() + assert validator.results[ASSEMBLY_CHECK][PASS] is False + + def test__load_fasta_check_results_partial_output(self): + # One of two FASTA files has no fasta check output while the other succeeds -> the task + # should not crash; only the missing file gets a synthetic failing entry. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output( + submission_dir, metadata_json=self.metadata_json_file, include_second_vcf_with_no_output=True) + validator._collect_validation_workflow_results() + assert validator.results[FASTA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + assert 'input_passed.fa' in validator.results[FASTA_CHECK] + assert validator.results[FASTA_CHECK]['missing_output.fa']['all_insdc'] is False + assert 'connection_error' in validator.results[FASTA_CHECK]['missing_output.fa'] + validator._assess_validation_results() + assert validator.results[FASTA_CHECK][PASS] is False From 0312c6f6141d1690e09e0b832da9504d8e919cf3 Mon Sep 17 00:00:00 2001 From: tcezard Date: Fri, 31 Jul 2026 13:03:16 +0100 Subject: [PATCH 7/8] Add back removed text --- eva_sub_cli/jinja_templates/text/report.txt | 4 ++++ .../validation_reports/expected_metadata_json_report.txt | 3 +++ .../validation_reports/expected_metadata_xlsx_report.txt | 3 +++ .../expected_report_metadata_json_crashed.txt | 3 +++ .../expected_report_metadata_json_process_not_run.txt | 3 +++ .../expected_shallow_metadata_xlsx_report.txt | 3 +++ tests/test_validator.py | 3 +-- 7 files changed, 20 insertions(+), 2 deletions(-) diff --git a/eva_sub_cli/jinja_templates/text/report.txt b/eva_sub_cli/jinja_templates/text/report.txt index 53481bf..2c01803 100644 --- a/eva_sub_cli/jinja_templates/text/report.txt +++ b/eva_sub_cli/jinja_templates/text/report.txt @@ -44,6 +44,10 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su - +VCF VALIDATION RESULTS +Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). +Also checks whether the variants' reference alleles match against the reference assembly. + ASSEMBLY CHECK RESULTS {% set run_status = validation_results.get('assembly_check', {}).get('run_status', '') %} {% if run_status == 'success' %} diff --git a/tests/resources/validation_reports/expected_metadata_json_report.txt b/tests/resources/validation_reports/expected_metadata_json_report.txt index 97b68e5..896d99d 100644 --- a/tests/resources/validation_reports/expected_metadata_json_report.txt +++ b/tests/resources/validation_reports/expected_metadata_json_report.txt @@ -69,6 +69,9 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su Error: should match exactly one schema in oneOf --- - +VCF VALIDATION RESULTS +Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). +Also checks whether the variants' reference alleles match against the reference assembly. ASSEMBLY CHECK RESULTS ❌ input_fail.vcf: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report diff --git a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt index 0e5b510..bb04f4d 100644 --- a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt @@ -63,6 +63,9 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su Error: Column "Sample Accession" is not populated --- - +VCF VALIDATION RESULTS +Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). +Also checks whether the variants' reference alleles match against the reference assembly. ASSEMBLY CHECK RESULTS ❌ input_fail.vcf: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt index 7933afe..638b338 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt @@ -29,6 +29,9 @@ Ensures that required fields are present and values are formatted correctly. For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data). ❗ Metadata check did not complete successfully. Check the logs for more information. - +VCF VALIDATION RESULTS +Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). +Also checks whether the variants' reference alleles match against the reference assembly. ASSEMBLY CHECK RESULTS ❗ Assembly check did not complete successfully. Check the logs for more information. VCF VALIDATION RESULTS diff --git a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt index dee5643..d468bde 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt +++ b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt @@ -29,6 +29,9 @@ Ensures that required fields are present and values are formatted correctly. For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data). ⏸ Process not run yet - +VCF VALIDATION RESULTS +Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). +Also checks whether the variants' reference alleles match against the reference assembly. ASSEMBLY CHECK RESULTS ⏸ Process not run yet VCF VALIDATION RESULTS diff --git a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt index c90c2d4..62aa7c4 100644 --- a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt @@ -72,6 +72,9 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su Error: Column "Sample Accession" is not populated --- - +VCF VALIDATION RESULTS +Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). +Also checks whether the variants' reference alleles match against the reference assembly. ASSEMBLY CHECK RESULTS ❌ input_fail.vcf: 26/36 (72.22%) First 10 errors per category are below. Full report: /path/to/assembly_failed/report diff --git a/tests/test_validator.py b/tests/test_validator.py index aeebf13..f818bfc 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -196,8 +196,7 @@ def create_validator_with_copied_output(self, submission_dir, metadata_json=None If include_second_vcf_with_no_output is True, a second VCF/FASTA entry ("missing_output.*") is added to the mapping file with no corresponding check output on disk, so a test can exercise - the "some files present, some missing" partial scenario. The extra input files don't need to - exist on disk since these tests call the collection methods directly, not verify_files_present(). + the "some files present, some missing" partial scenario. """ shutil.copytree(os.path.join(self.output_dir, VALIDATION_OUTPUT_DIR), os.path.join(submission_dir, VALIDATION_OUTPUT_DIR)) From 5644fecfea14b1d83433d544672a02817172b0ad Mon Sep 17 00:00:00 2001 From: Timothee Cezard Date: Mon, 3 Aug 2026 15:33:43 +0100 Subject: [PATCH 8/8] Apply suggestions from code review Co-authored-by: April Shen --- eva_sub_cli/jinja_templates/text/report.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eva_sub_cli/jinja_templates/text/report.txt b/eva_sub_cli/jinja_templates/text/report.txt index 2c01803..f97ad0d 100644 --- a/eva_sub_cli/jinja_templates/text/report.txt +++ b/eva_sub_cli/jinja_templates/text/report.txt @@ -48,7 +48,7 @@ VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. -ASSEMBLY CHECK RESULTS + ASSEMBLY CHECK {% set run_status = validation_results.get('assembly_check', {}).get('run_status', '') %} {% if run_status == 'success' %} {{ assembly_check_report(validation_results.get('assembly_check', {}), vcf_files) }} @@ -58,7 +58,7 @@ ASSEMBLY CHECK RESULTS {{ validation_not_run_yet_message() }} {% endif %} -VCF VALIDATION RESULTS + VCF CHECK {% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %} {% if run_status == 'success' %} {{ vcf_check_report(validation_results.get('vcf_check', {}), vcf_files) }}