diff --git a/eva_sub_cli/jinja_templates/html/file_validation.html b/eva_sub_cli/jinja_templates/html/file_validation.html index 9382c28..d4c7e82 100644 --- a/eva_sub_cli/jinja_templates/html/file_validation.html +++ b/eva_sub_cli/jinja_templates/html/file_validation.html @@ -1,93 +1,89 @@ -{% macro file_validation_report(validation_results, file_name) -%} - {% for check_type, check_per_file in validation_results.items() if check_type not in ["trim_down", "version"] %} - {% set result = check_per_file.get(file_name, {}) %} - {% if check_type == "assembly_check" %} - {{ assembly_check(result) }} - {% elif check_type == "vcf_check" %} - {{ vcf_check(result) }} + +{% macro vcf_check_report(vcf_check_result_per_file, vcf_file_names) -%} + {% for file_name in vcf_file_names %} + {% set vcf_check_result = vcf_check_result_per_file.get(file_name, {}) %} + {% set critical_count = vcf_check_result.get("critical_count", 0) %} + {% set error_count = vcf_check_result.get("error_count", 0) %} + {% set expand_icon = "" %} + {% if critical_count > 0 %} + {% set expand_icon = "▶" %} + {% set icon = "❌" %} + {% set row_class = "report-section fail collapsible" %} + {% elif error_count > 0 %} + {% set expand_icon = "▶" %} + {% set icon = "❗" %} + {% set row_class = "report-section warn collapsible" %} + {% else %} + {% set icon = "✔" %} + {% set row_class = "report-section pass" %} + {% endif %} +
{{ expand_icon }} {{ icon }} {{ file_name }}: {{ critical_count }} critical errors, {{ error_count }} non-critical errors
+ {% set critical_list = vcf_check_result.get("critical_list") %} + {% set error_list = vcf_check_result.get("error_list") %} + + {% if critical_list or error_list%} +
+
First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }}
+ + + + + {% for error in critical_list[:10] %} + + + + {% endfor %} + {% for error in error_list[:10] %} + + + + {% endfor %} +
CategoryError
critical error {{ error }}
non-critical error {{ error }}
+
{% endif %} {% endfor %} {%- endmacro %} -{% macro vcf_check(vcf_check_result) %} - {% set critical_count = vcf_check_result.get("critical_count", 0) %} - {% set error_count = vcf_check_result.get("error_count", 0) %} - {% set expand_icon = "" %} - {% if critical_count > 0 %} - {% set expand_icon = "▶" %} - {% set icon = "❌" %} - {% set row_class = "report-section fail collapsible" %} - {% elif error_count > 0 %} - {% set expand_icon = "▶" %} - {% set icon = "❗" %} - {% set row_class = "report-section warn collapsible" %} - {% else %} - {% set icon = "✔" %} - {% set row_class = "report-section pass" %} - {% endif %} -
{{ expand_icon }} {{ icon }} VCF check: {{ critical_count }} critical errors, {{ error_count }} non-critical errors
- {% set critical_list = vcf_check_result.get("critical_list") %} - {% set error_list = vcf_check_result.get("error_list") %} - - {% if critical_list or error_list%} -
-
First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }}
+{% macro assembly_check_report(assembly_check_result_per_file, vcf_file_names) -%} + {% for file_name in vcf_file_names %} + {% set assembly_check_result = assembly_check_result_per_file.get(file_name, {}) %} + {% set nb_match = assembly_check_result.get("match", 0) %} + {% set nb_total = assembly_check_result.get("total", 0) %} + {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %} + {% set expand_icon = "" %} + {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %} + {% set expand_icon = "▶" %} + {% set icon = "❌" %} + {% set row_class = "report-section fail collapsible" %} + {% else %} + {% set icon = "✔" %} + {% set row_class = "report-section pass" %} + {% endif %} +
{{ expand_icon }} {{ icon }} {{ file_name }}: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%)
+ {% set mismatch_list = assembly_check_result.get("mismatch_list") %} + {% set error_list = assembly_check_result.get("error_list") %} + {% if mismatch_list or error_list %} +
+
First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }}
- {% for error in critical_list[:10] %} + {% for error in error_list[:10] %} - + {% endfor %} - {% for error in error_list[:10] %} + {% for error in mismatch_list[:10] %} - + {% endfor %}
CategoryError
critical error {{ error }}Parsing Error {{ error }}
non-critical error {{ error }}mismatch error {{ error }}
-
- {% endif %} -{%- endmacro %} - -{% macro assembly_check(assembly_check_result) %} - {% set nb_match = assembly_check_result.get("match", 0) %} - {% set nb_total = assembly_check_result.get("total", 0) %} - {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %} - {% set expand_icon = "" %} - {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %} - {% set expand_icon = "▶" %} - {% set icon = "❌" %} - {% set row_class = "report-section fail collapsible" %} - {% else %} - {% set icon = "✔" %} - {% set row_class = "report-section pass" %} - {% endif %} -
{{ expand_icon }} {{ icon }} Assembly check: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%)
- {% set mismatch_list = assembly_check_result.get("mismatch_list") %} - {% set error_list = assembly_check_result.get("error_list") %} - {% if mismatch_list or error_list %} -
-
First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }}
- - - - - {% for error in error_list[:10] %} - - - - {% endfor %} - {% for error in mismatch_list[:10] %} - - - - {% endfor %} -
CategoryError
Parsing Error {{ error }}
mismatch error {{ error }}
-
- {% endif %} +
+ {% endif %} + {% endfor %} {%- endmacro %} diff --git a/eva_sub_cli/jinja_templates/html/report.html b/eva_sub_cli/jinja_templates/html/report.html index 8d71fa8..2f03d5b 100644 --- a/eva_sub_cli/jinja_templates/html/report.html +++ b/eva_sub_cli/jinja_templates/html/report.html @@ -1,6 +1,6 @@ {% from 'project_details.html' import project_details_report %} -{% from 'file_validation.html' import file_validation_report %} +{% from 'file_validation.html' import assembly_check_report, vcf_check_report %} {% from 'sample_name_check.html' import sample_name_check_report %} {% from 'fasta_check.html' import fasta_check_report %} {% from 'metadata_validation.html' import metadata_validation_report %} @@ -13,6 +13,13 @@
{{ expand_icon }} {{ icon }} {{ text }}
{% endmacro %} +{% macro report_crashed_task(task_name) %} + {% set icon = "❗" %} {# exclamation point ❗#} + {% set row_class = "report-section info" %} + {% set expand_icon = "" %} +
{{ expand_icon }} {{ icon }} {{ task_name }} did not complete successfully. Check the logs for more information.
+{% endmacro %} + @@ -73,8 +80,10 @@

Metadata validation results

For requirements, please refer to the EVA website. {% set run_status = validation_results.get('metadata_check', {}).get('run_status', '') %} - {% if run_status %} + {% if run_status == 'success' %} {{ metadata_validation_report(validation_results) }} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('Metadata check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -86,14 +95,21 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. +

Assembly check

+ {% set run_status = validation_results.get('assembly_check', {}).get('run_status', '') %} + {% if run_status == 'success' %} + {{ assembly_check_report(validation_results.get('assembly_check', {}), vcf_files) }} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('Assembly check') }} + {% else %} + {{ validation_not_run_yet_message() }} + {% endif %} +

VCF check

{% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %} - {% if run_status %} - {% for file_name in vcf_files %} - {% if file_name != "pass"%} -

{{ file_name }}

- {{ file_validation_report(validation_results, file_name) }} - {% endif %} - {% endfor %} + {% if run_status == 'success' %} + {{ vcf_check_report(validation_results.get('vcf_check', {}), vcf_files) }} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('VCF check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -105,8 +121,10 @@

Sample name concordance check

Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. {% set run_status = validation_results.get('sample_check', {}).get('run_status', '') %} - {% if run_status %} + {% if run_status == 'success' %} {{ sample_name_check_report(validation_results)}} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('Sample name concordance') }} {% else %} {{ validation_not_run_yet_message() }} @@ -120,13 +138,15 @@

Reference genome INSDC check

Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. {% set run_status = validation_results.get('fasta_check', {}).get('run_status', '') %} - {% if run_status %} + {% if run_status == 'success' %} {% for file_name in fasta_files %} {% if file_name != "pass"%}

{{ file_name }}

{{ fasta_check_report(validation_results, file_name) }} {% endif %} {% endfor %} + {% elif run_status == 'crashed' %} + {{ report_crashed_task('INSDC check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} diff --git a/eva_sub_cli/jinja_templates/text/file_validation.txt b/eva_sub_cli/jinja_templates/text/file_validation.txt index f6c9632..1838429 100644 --- a/eva_sub_cli/jinja_templates/text/file_validation.txt +++ b/eva_sub_cli/jinja_templates/text/file_validation.txt @@ -1,60 +1,55 @@ -{% macro file_validation_report(validation_results, file_name) -%} - {% for check_type, check_per_file in validation_results.items() if check_type not in ["trim_down", "version"] %} - {% set result = check_per_file.get(file_name, {}) %} - {% if check_type == "assembly_check" %} - {{ assembly_check(result) }} - {% elif check_type == "vcf_check" %} - {{ vcf_check(result) }} +{% macro vcf_check_report(vcf_check_result_per_file, vcf_file_names) -%} + {% for file_name in vcf_file_names %} + {% set vcf_check_result = vcf_check_result_per_file.get(file_name, {}) %} + {% set critical_count = vcf_check_result.get("critical_count", 0) %} + {% set error_count = vcf_check_result.get("error_count", 0) %} + {% if critical_count > 0 %} + {% set icon = "\u274C" %} + {% elif error_count > 0 %} + {% set icon = "\u26A0" %} + {% else %} + {% set icon = "\u2714" %} {% endif %} - {% endfor %} -{%- endmacro %} + {{ icon }} {{ file_name }}: {{ critical_count }} critical errors, {{ error_count }} non-critical errors -{% macro vcf_check(vcf_check_result) %} - {% set critical_count = vcf_check_result.get("critical_count", 0) %} - {% set error_count = vcf_check_result.get("error_count", 0) %} - {% if critical_count > 0 %} - {% set icon = "\u274C" %} - {% elif error_count > 0 %} - {% set icon = "\u26A0" %} - {% else %} - {% set icon = "\u2714" %} - {% endif %} - {{ icon }} VCF check: {{ critical_count }} critical errors, {{ error_count }} non-critical errors - - {% set critical_list = vcf_check_result.get("critical_list") %} - {% set error_list = vcf_check_result.get("error_list") %} - {% if critical_list or error_list%} - First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }} - {% for error in critical_list[:10] %} - Critical error: {{ error }} - {% endfor %} - {% for error in error_list[:10] %} - Non-critical error: {{ error }} - {% endfor %} - {% endif %} + {% set critical_list = vcf_check_result.get("critical_list") %} + {% set error_list = vcf_check_result.get("error_list") %} + {% if critical_list or error_list%} + First 10 errors per category are below. Full report: {{ vcf_check_result.get('report_path', '') }} + {% for error in critical_list[:10] %} + Critical error: {{ error }} + {% endfor %} + {% for error in error_list[:10] %} + Non-critical error: {{ error }} + {% endfor %} + {% endif %} + {% endfor %} {%- endmacro %} -{% macro assembly_check(assembly_check_result) %} - {% set nb_match = assembly_check_result.get("match", 0) %} - {% set nb_total = assembly_check_result.get("total", 0) %} - {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %} - {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %} - {% set icon = "\u274C" %} - {% else %} - {% set icon = "\u2714" %} - {% endif %} - {{ icon }} Assembly check: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%) +{% macro assembly_check_report(assembly_check_result_per_file, vcf_file_names) -%} + {% for file_name in vcf_file_names %} + {% set assembly_check_result = assembly_check_result_per_file.get(file_name, {}) %} + {% set nb_match = assembly_check_result.get("match", 0) %} + {% set nb_total = assembly_check_result.get("total", 0) %} + {% set match_percentage = nb_match / nb_total * 100 if nb_total else 0 %} + {% if assembly_check_result.get("nb_mismatch", 0) > 0 or nb_total == 0 %} + {% set icon = "\u274C" %} + {% else %} + {% set icon = "\u2714" %} + {% endif %} + {{ icon }} {{ file_name }}: {{ nb_match }}/{{ nb_total }} ({{ match_percentage|round(2) }}%) - {% set mismatch_list = assembly_check_result.get("mismatch_list") %} - {% set error_list = assembly_check_result.get("error_list") %} - {% if mismatch_list or error_list %} - First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }} - {% for error in error_list[:10] %} - Parsing error: {{ error }} - {% endfor %} - {% for error in mismatch_list[:10] %} - Mismatch error: {{ error }} - {% endfor %} - {% endif %} + {% set mismatch_list = assembly_check_result.get("mismatch_list") %} + {% set error_list = assembly_check_result.get("error_list") %} + {% if mismatch_list or error_list %} + First 10 errors per category are below. Full report: {{ assembly_check_result.get('report_path', '') }} + {% for error in error_list[:10] %} + Parsing error: {{ error }} + {% endfor %} + {% for error in mismatch_list[:10] %} + Mismatch error: {{ error }} + {% endfor %} + {% endif %} + {% endfor %} {%- endmacro %} diff --git a/eva_sub_cli/jinja_templates/text/report.txt b/eva_sub_cli/jinja_templates/text/report.txt index d087b12..f97ad0d 100644 --- a/eva_sub_cli/jinja_templates/text/report.txt +++ b/eva_sub_cli/jinja_templates/text/report.txt @@ -1,5 +1,5 @@ {% from 'project_details.txt' import project_details_report %} -{% from 'file_validation.txt' import file_validation_report %} +{% from 'file_validation.txt' import assembly_check_report, vcf_check_report %} {% from 'sample_name_check.txt' import sample_name_check_report %} {% from 'fasta_check.txt' import fasta_check_report %} {% from 'metadata_validation.txt' import metadata_validation_report %} @@ -10,6 +10,11 @@ {{ icon }} {{ text }} {%- endmacro %} +{% macro report_crashed_task(task_name) -%} +{% set icon = "\u2757" %} {# exclamation point ❗#} +{{ icon }} {{ task_name }} did not complete successfully. Check the logs for more information. +{%- endmacro %} + VALIDATION REPORT eva-sub-cli v{{cli_version}} @@ -29,25 +34,36 @@ Ensures that required fields are present and values are formatted correctly. For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data). {% set run_status = validation_results.get('metadata_check', {}).get('run_status', '') %} -{% if run_status %} +{% if run_status == 'success' %} {{ metadata_validation_report(validation_results) }} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('Metadata check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} + - VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. -{% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %} -{% if run_status %} -{% for file_name in vcf_files %} -{% if file_name != "pass"%} - {{ file_name }} - {{ file_validation_report(validation_results, file_name) }} + ASSEMBLY CHECK +{% set run_status = validation_results.get('assembly_check', {}).get('run_status', '') %} +{% if run_status == 'success' %} + {{ assembly_check_report(validation_results.get('assembly_check', {}), vcf_files) }} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('Assembly check') }} +{% else %} + {{ validation_not_run_yet_message() }} {% endif %} -{% endfor %} + + VCF CHECK +{% set run_status = validation_results.get('vcf_check', {}).get('run_status', '') %} +{% if run_status == 'success' %} + {{ vcf_check_report(validation_results.get('vcf_check', {}), vcf_files) }} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('VCF check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -58,8 +74,10 @@ SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. {% set run_status = validation_results.get('sample_check', {}).get('run_status', '') %} -{% if run_status %} +{% if run_status == 'success' %} {{ sample_name_check_report(validation_results) }} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('Sample name concordance') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} @@ -71,13 +89,16 @@ Checks that the reference sequences in the FASTA file used to call the variants Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. {% set run_status = validation_results.get('fasta_check', {}).get('run_status', '') %} -{% if run_status %} -{% for file_name in fasta_files %} -{% if file_name != "pass"%} +{% if run_status == 'success' %} + {% for file_name in fasta_files %} + {% if file_name != "pass"%} + {# Indentation is important as it impact the report's look#} {{ file_name }} {{ fasta_check_report(validation_results, file_name) }} -{% endif %} -{% endfor %} + {% endif %} + {% endfor %} +{% elif run_status == 'crashed' %} + {{ report_crashed_task('INSDC check') }} {% else %} {{ validation_not_run_yet_message() }} {% endif %} diff --git a/eva_sub_cli/validators/validator.py b/eva_sub_cli/validators/validator.py index 57ee390..02bc2a8 100755 --- a/eva_sub_cli/validators/validator.py +++ b/eva_sub_cli/validators/validator.py @@ -41,7 +41,11 @@ METADATA_CHECK] SHALLOW_VALIDATION = 'shallow_validation' TRIM_DOWN = 'trim_down' +# Status of the validation for a specific task. It should be a string containing the value "success", "crashed" or "did not run". RUN_STATUS_KEY = 'run_status' +RUN_STATUS_SUCCESS = 'success' +RUN_STATUS_CRASHED = 'crashed' +RUN_STATUS_DID_NOT_RUN = 'did not run' PASS = 'pass' @@ -197,11 +201,14 @@ def _load_previous_validation_results(self): # update previous shallow validation format to new one, if applicable if 'version' in self.results: if version.parse(self.results['version']) < version.parse('v0.4.15'): - self.update_previous_version_results() + self.update_pre_0_4_15_version_results() + if version.parse(self.results['version']) < version.parse('v0.6.5'): + self.update_pre_0_6_5_version_results() else: - self.update_previous_version_results() + self.update_pre_0_4_15_version_results() + self.update_pre_0_6_5_version_results() - def update_previous_version_results(self): + def update_pre_0_4_15_version_results(self): if 'requested' in self.results.get(SHALLOW_VALIDATION, {}): if 'required' in self.results.get(SHALLOW_VALIDATION, {}): if self.results.get(SHALLOW_VALIDATION, {}).get('required', False): @@ -214,6 +221,15 @@ def update_previous_version_results(self): del self.results[SHALLOW_VALIDATION]['required'] del self.results[SHALLOW_VALIDATION]['requested'] + def update_pre_0_6_5_version_results(self): + for task in ALL_VALIDATION_TASKS_GRANULAR: + if task in self.results: + run_status = self.results[task].get(RUN_STATUS_KEY) + if run_status: + self.results[task][RUN_STATUS_KEY] = RUN_STATUS_SUCCESS + else: + self.results[task][RUN_STATUS_KEY] = RUN_STATUS_DID_NOT_RUN + def _collect_validation_workflow_results(self): # Collect information from the output and summarise in the config if self.shallow_validation: @@ -239,13 +255,15 @@ def _assess_validation_results(self): It assumes all validation have been parsed already. """ - if VCF_CHECK in self.tasks: + if VCF_CHECK in self.tasks and self.results[VCF_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # vcf_check result - vcf_check_result = all((vcf_check.get('critical_count', 1) == 0 - for vcf_name, vcf_check in self.results.get(VCF_CHECK, {}).items() - if isinstance(vcf_check, dict))) - self.results[VCF_CHECK][PASS] = vcf_check_result + self.results[VCF_CHECK][PASS] = all((vcf_check.get('critical_count', 1) == 0 + for vcf_name, vcf_check in self.results.get(VCF_CHECK, {}).items() + if isinstance(vcf_check, dict))) + elif VCF_CHECK in self.tasks: + self.results[VCF_CHECK][PASS] = False + if VCF_CHECK in self.tasks and self.results[EVIDENCE_TYPE_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # evidence type check result self.results[EVIDENCE_TYPE_CHECK][PASS] = ( any(isinstance(v, dict) for v in self.results.get(EVIDENCE_TYPE_CHECK, {}).values()) @@ -253,11 +271,14 @@ def _assess_validation_results(self): all('evidence_type' in v and v['evidence_type'] is not None for v in self.results.get(EVIDENCE_TYPE_CHECK, {}).values() if isinstance(v, dict))) - elif VCF_CHECK not in self.results: - self.results[VCF_CHECK] = {RUN_STATUS_KEY: False} - self.results[EVIDENCE_TYPE_CHECK] = {RUN_STATUS_KEY: False} + elif VCF_CHECK in self.tasks: + self.results[EVIDENCE_TYPE_CHECK][PASS] = False - if ASSEMBLY_CHECK in self.tasks: + if VCF_CHECK not in self.results: + self.results[VCF_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} + self.results[EVIDENCE_TYPE_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} + + if ASSEMBLY_CHECK in self.tasks and self.results[ASSEMBLY_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # assembly_check result asm_nb_mismatch_result = all((asm_check.get('nb_mismatch', 1) == 0 for vcf_name, asm_check in self.results.get(ASSEMBLY_CHECK, {}).items() @@ -266,7 +287,10 @@ def _assess_validation_results(self): for vcf_name, asm_check in self.results.get(ASSEMBLY_CHECK, {}).items() if isinstance(asm_check, dict))) self.results[ASSEMBLY_CHECK][PASS] = asm_nb_mismatch_result and asm_nb_error_result + elif ASSEMBLY_CHECK in self.tasks: + self.results[ASSEMBLY_CHECK][PASS] = False + if ASSEMBLY_CHECK in self.tasks and self.results[FASTA_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # fasta_check result # Note this fails only if the FASTA file is not INSDC or the reference in the metadata is not a GCA. # It does not fail if the metadata assembly is not compatible with the FASTA, even though this is reported @@ -278,24 +302,31 @@ def _assess_validation_results(self): for fa_file, fa_file_check in self.results.get(FASTA_CHECK, {}).items() if isinstance(fa_file_check, dict)) self.results[FASTA_CHECK][PASS] = fasta_check_result and gca_check_result - elif ASSEMBLY_CHECK not in self.results: - self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: False} - self.results[FASTA_CHECK] = {RUN_STATUS_KEY: False} + elif ASSEMBLY_CHECK in self.tasks: + self.results[FASTA_CHECK][PASS] = False - if SAMPLE_CHECK in self.tasks: + if ASSEMBLY_CHECK not in self.results: + self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} + self.results[FASTA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} + + if SAMPLE_CHECK in self.tasks and self.results[SAMPLE_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # sample check result self.results[SAMPLE_CHECK][PASS] = self.results.get(SAMPLE_CHECK, {}).get('overall_differences', True) is False + elif SAMPLE_CHECK in self.tasks: + self.results[SAMPLE_CHECK][PASS] = False elif SAMPLE_CHECK not in self.results: - self.results[SAMPLE_CHECK] = {RUN_STATUS_KEY: False} + self.results[SAMPLE_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} - if METADATA_CHECK in self.tasks: + if METADATA_CHECK in self.tasks and self.results[METADATA_CHECK].get(RUN_STATUS_KEY) == RUN_STATUS_SUCCESS: # metadata check result metadata_xlsx_result = len(self.results.get(METADATA_CHECK, {}).get('spreadsheet_errors', []) or []) == 0 metadata_json_result = len(self.results.get(METADATA_CHECK, {}).get('json_errors', []) or []) == 0 self.results[METADATA_CHECK][PASS] = metadata_xlsx_result and metadata_json_result + elif METADATA_CHECK in self.tasks: + self.results[METADATA_CHECK][PASS] = False elif METADATA_CHECK not in self.results: - self.results[METADATA_CHECK] = {RUN_STATUS_KEY: False} + self.results[METADATA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} # update config based on the validation results self.sub_config.set(READY_FOR_SUBMISSION_TO_EVA, value=self.verify_ready_for_submission_to_eva()) @@ -351,20 +382,25 @@ def _evidence_type_check_yaml(self): def _collect_vcf_check_results(self): # detect output files for vcf check - self.results[VCF_CHECK] = {RUN_STATUS_KEY: True} + self.results[VCF_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[VCF_CHECK].update({TRIM_DOWN: True}) - for vcf_file in self.vcf_files: - vcf_name = os.path.basename(vcf_file) - - vcf_check_log = self._vcf_check_log(vcf_name) - vcf_check_text_report = self._vcf_check_text_report(vcf_name) + vcf_names = [os.path.basename(vcf_file) for vcf_file in self.vcf_files] + per_vcf_paths = {vcf_name: (self._vcf_check_log(vcf_name), self._vcf_check_text_report(vcf_name)) + for vcf_name in vcf_names} + if not any(log and report for log, report in per_vcf_paths.values()): + self.error('Cannot locate any vcf check results. The process might have failed.') + self.results[VCF_CHECK][RUN_STATUS_KEY] = RUN_STATUS_CRASHED + return + for vcf_name, (vcf_check_log, vcf_check_text_report) in per_vcf_paths.items(): if vcf_check_log and vcf_check_text_report: valid, warning_count, error_count, critical_count, error_list, critical_list = parse_vcf_check_report( vcf_check_text_report) else: + error_txt = f'Cannot locate vcf check results for {vcf_name}. The process might have failed.' + self.error(error_txt) valid, warning_count, error_count, critical_count, error_list, critical_list = (False, 0, 0, 1, [], ['Process failed']) self.results[VCF_CHECK][vcf_name] = { @@ -379,16 +415,19 @@ def _collect_vcf_check_results(self): def _collect_assembly_check_results(self): # detect output files for assembly check - self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: True} + self.results[ASSEMBLY_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[ASSEMBLY_CHECK].update({TRIM_DOWN: True}) - for vcf_file in self.vcf_files: - vcf_name = os.path.basename(vcf_file) - - assembly_check_log = self._assembly_check_log(vcf_name) - assembly_check_text_report = self._assembly_check_text_report(vcf_name) + vcf_names = [os.path.basename(vcf_file) for vcf_file in self.vcf_files] + per_vcf_paths = {vcf_name: (self._assembly_check_log(vcf_name), self._assembly_check_text_report(vcf_name)) + for vcf_name in vcf_names} + if not any(log and report for log, report in per_vcf_paths.values()): + self.error('Cannot locate any assembly check results. The process might have failed.') + self.results[ASSEMBLY_CHECK][RUN_STATUS_KEY] = RUN_STATUS_CRASHED + return + for vcf_name, (assembly_check_log, assembly_check_text_report) in per_vcf_paths.items(): if assembly_check_log and assembly_check_text_report: error_list_from_log, nb_error_from_log, match, total = \ parse_assembly_check_log(assembly_check_log) @@ -397,6 +436,7 @@ def _collect_assembly_check_results(self): nb_error = nb_error_from_log + nb_error_from_report error_list = error_list_from_log + error_list_from_report else: + self.error(f'Cannot locate assembly check results for {vcf_name}. The process might have failed.') error_list, mismatch_list, nb_mismatch, nb_error, match, total = (['Process failed'], [], 0, 1, 0, 0) self.results[ASSEMBLY_CHECK][vcf_name] = { 'report_path': assembly_check_text_report, @@ -409,15 +449,25 @@ def _collect_assembly_check_results(self): } def _load_fasta_check_results(self): - self.results[FASTA_CHECK] = {RUN_STATUS_KEY: True} + self.results[FASTA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} if self.shallow_validation and self.results[SHALLOW_VALIDATION][TRIM_DOWN] is True: self.results[FASTA_CHECK].update({TRIM_DOWN: True}) - for fasta_file in self.fasta_files: - fasta_file_name = os.path.basename(fasta_file) - fasta_check = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', - f'{fasta_file_name}_check.yml')) + fasta_check_paths = {os.path.basename(fasta_file): resolve_single_file_path( + os.path.join(self.output_dir, 'other_validations', f'{os.path.basename(fasta_file)}_check.yml')) + for fasta_file in self.fasta_files} + if not any(fasta_check_paths.values()): + self.error('Cannot locate any fasta check results. The process might have failed.') + self.results[FASTA_CHECK][RUN_STATUS_KEY] = RUN_STATUS_CRASHED + return + + for fasta_file_name, fasta_check in fasta_check_paths.items(): if not fasta_check: + error_txt = f'Cannot locate fasta check results for {fasta_file_name}. The process might have failed.' + self.error(error_txt) + self.results[FASTA_CHECK][fasta_file_name] = { + 'all_insdc': False, 'sequences': [], 'connection_error': error_txt + } continue with open(fasta_check) as open_yaml: self.results[FASTA_CHECK][fasta_file_name] = yaml.safe_load(open_yaml) @@ -428,8 +478,10 @@ def _load_sample_check_results(self): with open(self._sample_check_yaml) as open_yaml: self.results[SAMPLE_CHECK] = yaml.safe_load(open_yaml) self.results[SAMPLE_CHECK]['report_path'] = self._sample_check_yaml - - self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: True}) + self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_SUCCESS}) + else: + self.error(f'Cannot locate sample check results. The process might have failed.') + self.results[SAMPLE_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) def _load_evidence_check_results(self): self.results[EVIDENCE_TYPE_CHECK] = {} @@ -437,20 +489,29 @@ def _load_evidence_check_results(self): with open(self._evidence_type_check_yaml) as open_yaml: self.results[EVIDENCE_TYPE_CHECK] = yaml.safe_load(open_yaml) self.results[EVIDENCE_TYPE_CHECK]['report_path'] = self._evidence_type_check_yaml - self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: True}) + self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_SUCCESS}) + else: + self.error(f'Cannot locate evidence type check results. The process might have failed.') + self.results[EVIDENCE_TYPE_CHECK].update({RUN_STATUS_KEY: RUN_STATUS_CRASHED}) self._update_metadata_with_evidence_type() def _collect_metadata_results(self): - self.results[METADATA_CHECK] = {RUN_STATUS_KEY: True} + self.results[METADATA_CHECK] = {RUN_STATUS_KEY: RUN_STATUS_SUCCESS} + self._load_spreadsheet_conversion_errors() - self.collect_biovalidator_validation_results() - self._collect_semantic_metadata_results() + metadata_validation_file = self.collect_biovalidator_validation_results() + metadata_semantic_file = self._collect_semantic_metadata_results() if self.metadata_xlsx: self._convert_biovalidator_validation_to_spreadsheet() self._write_spreadsheet_validation_results() - self._collect_file_info_to_metadata() + file_info_file = self._collect_file_info_to_metadata() self._add_schema_to_metadata() + if not any([metadata_validation_file, metadata_semantic_file, file_info_file, + self.metadata_json_post_validation]): + self.error('Cannot locate any metadata check output. The process might have failed.') + self.results[METADATA_CHECK][RUN_STATUS_KEY] = RUN_STATUS_CRASHED + def _load_spreadsheet_conversion_errors(self): errors_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'metadata_conversion_errors.yml')) @@ -461,26 +522,41 @@ def _load_spreadsheet_conversion_errors(self): def collect_biovalidator_validation_results(self): """ - Read the biovalidator's report and extract the list of validation errors + Read the biovalidator's report and extract the list of validation errors. + Returns the resolved report path, or None if it could not be located. """ metadata_check_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'metadata_validation.txt')) - errors = parse_biovalidator_validation_results(metadata_check_file) + if metadata_check_file: + errors = parse_biovalidator_validation_results(metadata_check_file) + else: + error_txt = f"Cannot locate metadata_validation.txt. The process might have failed." + self.error(error_txt) + errors = [{'property': '/', 'description': error_txt}] self.results[METADATA_CHECK].update({ 'json_report_path': metadata_check_file, 'json_errors': errors }) + return metadata_check_file def _collect_semantic_metadata_results(self): + """ + Read the semantic check's report and merge its errors into the biovalidator's. + Returns the resolved report path, or None if it could not be located. + """ errors_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'metadata_semantic_check.yml')) if not errors_file: - return + error_txt = 'Cannot locate metadata_semantic_check.yml. The process might have failed.' + self.error(error_txt) + self.results[METADATA_CHECK]['json_errors'].append({'property': '/', 'description': error_txt}) + return None with open(errors_file) as open_yaml: # errors is a list of dicts matching format of biovalidator errors errors = yaml.safe_load(open_yaml) # biovalidator error parsing always places a list here, even if no errors self.results[METADATA_CHECK]['json_errors'] += errors + return errors_file def _convert_biovalidator_validation_to_spreadsheet(self): config_file = os.path.join(ETC_DIR, "spreadsheet2json_conf.yaml") @@ -536,6 +612,10 @@ def _write_spreadsheet_validation_results(self): self.results[METADATA_CHECK]['spreadsheet_report_path'] = spreadsheet_report_file def _collect_file_info_to_metadata(self): + """ + Attach md5/file size info to each file in the metadata. Returns the resolved file_info.txt + path, or None if it could not be located. + """ errors = [] md5sum_file = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', 'file_info.txt')) file_path_2_md5 = {} @@ -554,9 +634,8 @@ def _collect_file_info_to_metadata(self): file_path_2_file_size[vcf_file] = file_size file_name_2_file_size[os.path.basename(vcf_file)] = file_size else: - error_txt = f"Cannot locate file_info.txt at {os.path.join(self.output_dir, 'other_validations', 'file_info.txt')}" + error_txt = f"Cannot locate file_info.txt. The process might have failed." self.error(error_txt) - raise FileNotFoundError(error_txt) if self.metadata_json_post_validation: metadata = EvaMetadataJson(self.metadata_json_post_validation) @@ -593,14 +672,15 @@ def _collect_file_info_to_metadata(self): errors.append({'property': '/', 'description': error_txt}) metadata.write(self.metadata_json_post_validation) else: - error_txt = f'Cannot locate the metadata in JSON format in {os.path.join(self.output_dir, "metadata.json")}' + error_txt = f'Cannot locate the metadata in JSON format. The process might have failed.' self.error(error_txt) - raise FileNotFoundError(error_txt) + errors.append({'property': '/', 'description': error_txt}) if errors: if 'json_errors' in self.results[METADATA_CHECK]: self.results[METADATA_CHECK]['json_errors'].extend(errors) else: self.results[METADATA_CHECK]['json_errors'] = errors + return md5sum_file def _add_schema_to_metadata(self): if self.metadata_json_post_validation: @@ -634,12 +714,16 @@ def _update_metadata_with_evidence_type(self): def _collect_trim_down_metrics(self): self.results[SHALLOW_VALIDATION] = {'metrics': {}} - shallow_validation_required = False + shallow_validation_required = False # Flag to indicate if shallow validation actually reduced the number of lines for vcf_file in self.vcf_files: basename = os.path.basename(vcf_file) vcf_name, _ = os.path.splitext(basename) trimmed_down_metrics = resolve_single_file_path(os.path.join(self.output_dir, 'other_validations', f'{vcf_name}_trim_down.yml')) + if not trimmed_down_metrics: + self.error(f'Cannot locate trim down metrics for {vcf_name}. The process might have failed.') + shallow_validation_required = True # Assume that the shallow validation would have been required + continue with open(trimmed_down_metrics) as open_file: metrics = yaml.safe_load(open_file) shallow_validation_required = shallow_validation_required or metrics['trim_down_required'] diff --git a/tests/resources/validation_reports/expected_metadata_json_report.html b/tests/resources/validation_reports/expected_metadata_json_report.html index faa3393..8ed34e9 100644 --- a/tests/resources/validation_reports/expected_metadata_json_report.html +++ b/tests/resources/validation_reports/expected_metadata_json_report.html @@ -162,64 +162,64 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. -

input_fail.vcf

-
▶ ❌ Assembly check: 26/36 (72.22%)
-
-
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CategoryError
Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-
▶ ❌ VCF check: 1 critical errors, 1 non-critical errors
-
-
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+

Assembly check

+
▶ ❌ input_fail.vcf: 26/36 (72.22%)
+
+
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- + - + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-

input_passed.vcf

-
✔ Assembly check: 247/247 (100.0%)
-
✔ VCF check: 0 critical errors, 0 non-critical errors
+
+
✔ input_passed.vcf: 247/247 (100.0%)
+

VCF check

+
▶ ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors
+
+
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+ + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..
+
+
✔ input_passed.vcf: 0 critical errors, 0 non-critical errors

Sample name concordance check

diff --git a/tests/resources/validation_reports/expected_metadata_json_report.txt b/tests/resources/validation_reports/expected_metadata_json_report.txt index 6ed2d15..896d99d 100644 --- a/tests/resources/validation_reports/expected_metadata_json_report.txt +++ b/tests/resources/validation_reports/expected_metadata_json_report.txt @@ -72,26 +72,26 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf - ❌ Assembly check: 26/36 (72.22%) - First 10 errors per category are below. Full report: /path/to/assembly_failed/report - Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report - Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' - ❌ VCF check: 1 critical errors, 1 non-critical errors - First 10 errors per category are below. Full report: /path/to/vcf_failed/report - Critical error: Line 4: Error in meta-data section. - Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf - ✔ Assembly check: 247/247 (100.0%) - ✔ VCF check: 0 critical errors, 0 non-critical errors +ASSEMBLY CHECK RESULTS + ❌ input_fail.vcf: 26/36 (72.22%) + First 10 errors per category are below. Full report: /path/to/assembly_failed/report + Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report + Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' + ✔ input_passed.vcf: 247/247 (100.0%) +VCF VALIDATION RESULTS + ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors + First 10 errors per category are below. Full report: /path/to/vcf_failed/report + Critical error: Line 4: Error in meta-data section. + Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. + ✔ input_passed.vcf: 0 critical errors, 0 non-critical errors - SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. diff --git a/tests/resources/validation_reports/expected_metadata_xlsx_report.html b/tests/resources/validation_reports/expected_metadata_xlsx_report.html index 9c2da12..16f5cfc 100644 --- a/tests/resources/validation_reports/expected_metadata_xlsx_report.html +++ b/tests/resources/validation_reports/expected_metadata_xlsx_report.html @@ -174,64 +174,64 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. -

input_fail.vcf

-
▶ ❌ Assembly check: 26/36 (72.22%)
-
-
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CategoryError
Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-
▶ ❌ VCF check: 1 critical errors, 1 non-critical errors
-
-
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+

Assembly check

+
▶ ❌ input_fail.vcf: 26/36 (72.22%)
+
+
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- + - + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-

input_passed.vcf

-
✔ Assembly check: 247/247 (100.0%)
-
✔ VCF check: 0 critical errors, 0 non-critical errors
+
+
✔ input_passed.vcf: 247/247 (100.0%)
+

VCF check

+
▶ ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors
+
+
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+ + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..
+
+
✔ input_passed.vcf: 0 critical errors, 0 non-critical errors

Sample name concordance check

diff --git a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt index c3ee9b4..bb04f4d 100644 --- a/tests/resources/validation_reports/expected_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_metadata_xlsx_report.txt @@ -66,26 +66,26 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf - ❌ Assembly check: 26/36 (72.22%) - First 10 errors per category are below. Full report: /path/to/assembly_failed/report - Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report - Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' - ❌ VCF check: 1 critical errors, 1 non-critical errors - First 10 errors per category are below. Full report: /path/to/vcf_failed/report - Critical error: Line 4: Error in meta-data section. - Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf - ✔ Assembly check: 247/247 (100.0%) - ✔ VCF check: 0 critical errors, 0 non-critical errors +ASSEMBLY CHECK RESULTS + ❌ input_fail.vcf: 26/36 (72.22%) + First 10 errors per category are below. Full report: /path/to/assembly_failed/report + Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report + Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' + ✔ input_passed.vcf: 247/247 (100.0%) +VCF VALIDATION RESULTS + ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors + First 10 errors per category are below. Full report: /path/to/vcf_failed/report + Critical error: Line 4: Error in meta-data section. + Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. + ✔ input_passed.vcf: 0 critical errors, 0 non-critical errors - SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.html b/tests/resources/validation_reports/expected_report_metadata_json_crashed.html new file mode 100644 index 0000000..69de034 --- /dev/null +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.html @@ -0,0 +1,147 @@ + + + + + Validation Report + + + +
+ +
+

Validation Report

+
eva-sub-cli vcligeneratedversion
+
+
+
+

Project Summary

+
+ General details about the project +
+
+

Project Title: My cool project

+

Validation Date: 2023-08-31 12:34:56

+

Submission Directory: /test/submission/dir

+
+

Note for human genotype data:

You will need to provide a signed copy of our Consent Statement via email to eva-helpdesk@ebi.ac.uk after submission. + Your submission will only be processed after receiving the consent statement. For more info, please see our help section.

+
+
▶ Files mapping
+
+ + + + + + + + + + + + + + + + + + + + + +
VCF FileFasta FileAnalysis
input_fail.vcfinput_fail.faA
input_pass.vcfinput_pass.faB
input_test.vcfinput_test.facould not be linked
+
+ +
+
+

Metadata validation results

+
+ Ensures that required fields are present and values are formatted correctly. + For requirements, please refer to the EVA website. +
+
❗ Metadata check did not complete successfully. Check the logs for more information.
+
+
+

VCF validation results

+
+ Checks whether each file is compliant with the VCF specification. + Also checks whether the variants' reference alleles match against the reference assembly. +
+

Assembly check

+
❗ Assembly check did not complete successfully. Check the logs for more information.
+

VCF check

+
❗ VCF check did not complete successfully. Check the logs for more information.
+
+
+

Sample name concordance check

+
+ Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. +
+
❗ Sample name concordance did not complete successfully. Check the logs for more information.
+
+
+

Reference genome INSDC check

+
+ Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. + Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. +
+
❗ INSDC check did not complete successfully. Check the logs for more information.
+
+ + + \ No newline at end of file diff --git a/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt new file mode 100644 index 0000000..638b338 --- /dev/null +++ b/tests/resources/validation_reports/expected_report_metadata_json_crashed.txt @@ -0,0 +1,47 @@ + +VALIDATION REPORT +eva-sub-cli vcligeneratedversion +- +PROJECT SUMMARY +General details about the project + Project Title: My cool project + Validation Date: 2023-08-31 12:34:56 + Submission Directory: /test/submission/dir + Note for human genotype data: + You will need to provide a signed copy of our Consent Statement (https://docs.google.com/document/d/1UaRmimAe919IZYIH55mDXMe-4sHsRVqaP4moqx2IYE4) via email to eva-helpdesk@ebi.ac.uk after submission. + Your submission will only be processed after receiving the consent statement.For more info, please see our help section (https://www.ebi.ac.uk/eva/?Help#submissionPanel&link=consent-statement-for-human-genotype-data). + Files mapping: + --- + VCF File: input_fail.vcf + Fasta File: input_fail.fa + Analysis: A + --- + VCF File: input_pass.vcf + Fasta File: input_pass.fa + Analysis: B + --- + VCF File: input_test.vcf + Fasta File: input_test.fa + Analysis: could not be linked +- +METADATA VALIDATION RESULTS +Ensures that required fields are present and values are formatted correctly. +For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Submit-Data). +❗ Metadata check did not complete successfully. Check the logs for more information. +- +VCF VALIDATION RESULTS +Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). +Also checks whether the variants' reference alleles match against the reference assembly. +ASSEMBLY CHECK RESULTS +❗ Assembly check did not complete successfully. Check the logs for more information. +VCF VALIDATION RESULTS +❗ VCF check did not complete successfully. Check the logs for more information. +- +SAMPLE NAME CONCORDANCE CHECK +Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. +❗ Sample name concordance did not complete successfully. Check the logs for more information. +- +REFERENCE GENOME INSDC CHECK +Checks that the reference sequences in the FASTA file used to call the variants are accessioned in INSDC. +Also checks if the reference assembly accession in the metadata matches the one determined from the FASTA file. +❗ INSDC check did not complete successfully. Check the logs for more information. diff --git a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html index e9546f8..e3ad393 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html +++ b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.html @@ -90,6 +90,9 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. +

Assembly check

+
⏸ Process not run yet
+

VCF check

⏸ Process not run yet
diff --git a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt index 82cb4d5..d468bde 100644 --- a/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt +++ b/tests/resources/validation_reports/expected_report_metadata_json_process_not_run.txt @@ -32,6 +32,9 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. +ASSEMBLY CHECK RESULTS +⏸ Process not run yet +VCF VALIDATION RESULTS ⏸ Process not run yet - SAMPLE NAME CONCORDANCE CHECK diff --git a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html index 031bcc2..e62e1a9 100644 --- a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html +++ b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.html @@ -198,64 +198,64 @@

VCF validation results

Checks whether each file is compliant with the VCF specification. Also checks whether the variants' reference alleles match against the reference assembly. -

input_fail.vcf

-
▶ ❌ Assembly check: 26/36 (72.22%)
-
-
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CategoryError
Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-
▶ ❌ VCF check: 1 critical errors, 1 non-critical errors
-
-
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+

Assembly check

+
▶ ❌ input_fail.vcf: 26/36 (72.22%)
+
+
First 10 errors per category are below. Full report: /path/to/assembly_failed/report
- + + + + + + + + + + + + + + + + + + + - + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.Parsing Error The assembly checking could not be completed: Contig 'chr23' not found in assembly report
mismatch error Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c'
mismatch error Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g'
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..mismatch error Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g'
mismatch error Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a'
mismatch error Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c'
-
-

input_passed.vcf

-
✔ Assembly check: 247/247 (100.0%)
-
✔ VCF check: 0 critical errors, 0 non-critical errors
+
+
✔ input_passed.vcf: 247/247 (100.0%)
+

VCF check

+
▶ ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors
+
+
First 10 errors per category are below. Full report: /path/to/vcf_failed/report
+ + + + + + + + + + +
CategoryError
critical error Line 4: Error in meta-data section.
non-critical error Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=..
+
+
✔ input_passed.vcf: 0 critical errors, 0 non-critical errors

Sample name concordance check

diff --git a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt index 58cfe7f..62aa7c4 100644 --- a/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt +++ b/tests/resources/validation_reports/expected_shallow_metadata_xlsx_report.txt @@ -75,26 +75,26 @@ For requirements, please refer to the EVA website (https://www.ebi.ac.uk/eva/?Su VCF VALIDATION RESULTS Checks whether each file is compliant with the VCF specification (http://samtools.github.io/hts-specs/VCFv4.4.pdf). Also checks whether the variants' reference alleles match against the reference assembly. - input_fail.vcf - ❌ Assembly check: 26/36 (72.22%) - First 10 errors per category are below. Full report: /path/to/assembly_failed/report - Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report - Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' - Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' - Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' - Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' - ❌ VCF check: 1 critical errors, 1 non-critical errors - First 10 errors per category are below. Full report: /path/to/vcf_failed/report - Critical error: Line 4: Error in meta-data section. - Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. - input_passed.vcf - ✔ Assembly check: 247/247 (100.0%) - ✔ VCF check: 0 critical errors, 0 non-critical errors +ASSEMBLY CHECK RESULTS + ❌ input_fail.vcf: 26/36 (72.22%) + First 10 errors per category are below. Full report: /path/to/assembly_failed/report + Parsing error: The assembly checking could not be completed: Contig 'chr23' not found in assembly report + Mismatch error: Chromosome 1, position 35549, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35595, reference allele 'G' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35618, reference allele 'G' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35626, reference allele 'A' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35639, reference allele 'T' does not match the reference sequence, expected 'c' + Mismatch error: Chromosome 1, position 35643, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35717, reference allele 'T' does not match the reference sequence, expected 'g' + Mismatch error: Chromosome 1, position 35819, reference allele 'T' does not match the reference sequence, expected 'a' + Mismatch error: Chromosome 1, position 35822, reference allele 'T' does not match the reference sequence, expected 'c' + ✔ input_passed.vcf: 247/247 (100.0%) +VCF VALIDATION RESULTS + ❌ input_fail.vcf: 1 critical errors, 1 non-critical errors + First 10 errors per category are below. Full report: /path/to/vcf_failed/report + Critical error: Line 4: Error in meta-data section. + Non-critical error: Sample #11, field AD does not match the meta specification Number=R (expected 2 value(s)). AD=.. + ✔ input_passed.vcf: 0 critical errors, 0 non-critical errors - SAMPLE NAME CONCORDANCE CHECK Checks whether information in the metadata is concordant with that contained in the VCF files, in particular sample names. diff --git a/tests/test_report.py b/tests/test_report.py index 50672d3..8b5c8ef 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -6,14 +6,15 @@ import eva_sub_cli from eva_sub_cli.report import generate_html_report, generate_text_report -from eva_sub_cli.validators.validator import RUN_STATUS_KEY, TRIM_DOWN +from eva_sub_cli.validators.validator import (RUN_STATUS_KEY, RUN_STATUS_SUCCESS, RUN_STATUS_CRASHED, + RUN_STATUS_DID_NOT_RUN, TRIM_DOWN) common_validation_results = { "ready_for_submission_to_eva": False, "version": "0.5.1", "trim_down": False, "assembly_check": { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, "input_passed.vcf": { "report_path": "/path/to/assembly_passed/report", @@ -45,7 +46,7 @@ }, }, "vcf_check": { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, "input_passed.vcf": { 'report_path': '/path/to/vcf_passed/report', @@ -66,7 +67,7 @@ }, }, "sample_check": { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'report_path': '/path/to/sample/report', 'overall_differences': True, @@ -96,7 +97,7 @@ # NB. obviously this doesn't make sense for the number of analyses in this report, but demonstrates the possible # outputs for this check. "fasta_check": { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'not_all_insdc.fa': { 'report_path': '/path/to/not_all_insdc_check.yml', @@ -175,7 +176,7 @@ } }, 'evidence_type_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'report_path': '/path/to/evidence_type.yml', 'Analysis A': { @@ -191,7 +192,7 @@ validation_results_xlsx = deepcopy(common_validation_results) validation_results_xlsx['metadata_check'] = { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'spreadsheet_errors': [ {'sheet': 'Files', 'row': '', 'column': '', 'description': 'Sheet "Files" is missing'}, @@ -217,7 +218,7 @@ validation_results_json = deepcopy(common_validation_results) validation_results_json['metadata_check'] = { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'pass': False, 'json_errors': [ {'property': '.files', 'description': "should have required property 'files'"}, @@ -247,6 +248,8 @@ class TestReport(TestCase): 'expected_metadata_json_report.html') expected_report_metadata_json_process_not_run = os.path.join(resource_dir, 'validation_reports', 'expected_report_metadata_json_process_not_run.html') + expected_report_metadata_json_crashed = os.path.join(resource_dir, 'validation_reports', + 'expected_report_metadata_json_crashed.html') expected_report_metadata_xlsx_shallow = os.path.join(resource_dir, 'validation_reports', 'expected_shallow_metadata_xlsx_report.html') expected_text_report_metadata_xlsx = os.path.join(resource_dir, 'validation_reports', @@ -255,6 +258,8 @@ class TestReport(TestCase): 'expected_metadata_json_report.txt') expected_text_report_metadata_json_process_not_run = os.path.join(resource_dir, 'validation_reports', 'expected_report_metadata_json_process_not_run.txt') + expected_text_report_metadata_json_crashed = os.path.join(resource_dir, 'validation_reports', + 'expected_report_metadata_json_crashed.txt') expected_text_report_metadata_xlsx_shallow = os.path.join(resource_dir, 'validation_reports', 'expected_shallow_metadata_xlsx_report.txt') test_project_name = "My cool project" @@ -304,12 +309,12 @@ def test_generate_html_report_metadata_json(self): def test_generate_html_report_metadata_json_metadata_report_not_run_yet(self): validation_result = { - 'vcf_check': {RUN_STATUS_KEY: False}, - 'evidence_type_check': {RUN_STATUS_KEY: False}, - 'assembly_check': {RUN_STATUS_KEY: False}, - 'fasta_check': {RUN_STATUS_KEY: False}, - 'metadata_check': {RUN_STATUS_KEY: False}, - 'sample_check': {RUN_STATUS_KEY: False} + 'vcf_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'evidence_type_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'assembly_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'fasta_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'metadata_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'sample_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} } self.check_report_vs_expected( @@ -318,6 +323,22 @@ def test_generate_html_report_metadata_json_metadata_report_not_run_yet(self): self.expected_report_metadata_json_process_not_run ) + def test_generate_html_report_metadata_json_crashed(self): + validation_result = { + 'vcf_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'evidence_type_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'assembly_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'fasta_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'metadata_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'sample_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED} + } + + self.check_report_vs_expected( + validation_result, + 'metadata_json_report.html', + self.expected_report_metadata_json_crashed + ) + def test_generate_html_report_metadata_xlsx_shallow(self): shallow_validation_results_xlsx = copy.deepcopy(validation_results_xlsx) shallow_validation_results_xlsx[TRIM_DOWN] = True @@ -352,12 +373,12 @@ def test_generate_text_report_metadata_json(self): def test_generate_text_report_metadata_json_report_not_run(self): validation_result = { - 'vcf_check': {RUN_STATUS_KEY: False}, - 'evidence_type_check': {RUN_STATUS_KEY: False}, - 'assembly_check': {RUN_STATUS_KEY: False}, - 'fasta_check': {RUN_STATUS_KEY: False}, - 'metadata_check': {RUN_STATUS_KEY: False}, - 'sample_check': {RUN_STATUS_KEY: False} + 'vcf_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'evidence_type_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'assembly_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'fasta_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'metadata_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN}, + 'sample_check': {RUN_STATUS_KEY: RUN_STATUS_DID_NOT_RUN} } self.check_report_vs_expected( @@ -367,6 +388,23 @@ def test_generate_text_report_metadata_json_report_not_run(self): html=False ) + def test_generate_text_report_metadata_json_crashed(self): + validation_result = { + 'vcf_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'evidence_type_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'assembly_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'fasta_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'metadata_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED}, + 'sample_check': {RUN_STATUS_KEY: RUN_STATUS_CRASHED} + } + + self.check_report_vs_expected( + validation_result, + 'metadata_json_report.txt', + self.expected_text_report_metadata_json_crashed, + html=False + ) + def test_generate_text_report_metadata_xlsx_shallow(self): shallow_validation_results_xlsx = copy.deepcopy(validation_results_xlsx) shallow_validation_results_xlsx[TRIM_DOWN] = True diff --git a/tests/test_validator.py b/tests/test_validator.py index 6e0dd37..f818bfc 100644 --- a/tests/test_validator.py +++ b/tests/test_validator.py @@ -8,23 +8,26 @@ import eva_sub_cli from eva_sub_cli.metadata import EvaMetadataJson -from eva_sub_cli.validators.validator import (Validator, VALIDATION_OUTPUT_DIR, VCF_CHECK, READY_FOR_SUBMISSION_TO_EVA, - RUN_STATUS_KEY, METADATA_CHECK, PASS, TRIM_DOWN, SHALLOW_VALIDATION) +from eva_sub_cli.validators.validator import (Validator, VALIDATION_OUTPUT_DIR, VCF_CHECK, ASSEMBLY_CHECK, + READY_FOR_SUBMISSION_TO_EVA, + RUN_STATUS_KEY, RUN_STATUS_SUCCESS, RUN_STATUS_CRASHED, + RUN_STATUS_DID_NOT_RUN, METADATA_CHECK, PASS, TRIM_DOWN, + SHALLOW_VALIDATION, FASTA_CHECK, SAMPLE_CHECK, EVIDENCE_TYPE_CHECK) from tests.test_utils import create_mapping_file expected_validation_results = { 'vcf_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'input_passed.vcf': {'valid': True, 'error_list': [], 'error_count': 0, 'warning_count': 0, 'critical_count': 0, 'critical_list': []} }, 'assembly_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'input_passed.vcf': {'error_list': [], 'mismatch_list': [], 'nb_mismatch': 0, 'nb_error': 0, 'match': 247, 'total': 247} }, 'sample_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'overall_differences': False, 'results_per_analysis': { 'AA': { @@ -36,7 +39,7 @@ } }, 'evidence_type_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'AA': { 'errors': None, 'evidence_type': 'allele_frequency' @@ -44,14 +47,14 @@ }, 'fasta_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'input_passed.fa': {'all_insdc': False, 'sequences': [ {'sequence_name': 1, 'insdc': True, 'sequence_md5': '6681ac2f62509cfc220d78751b8dc524'}, {'sequence_name': 2, 'insdc': False, 'sequence_md5': 'd2b3f22704d944f92a6bc45b6603ea2d'} ]}, }, 'metadata_check': { - 'run_status': True, + 'run_status': RUN_STATUS_SUCCESS, 'json_errors': [ {'property': '/files', 'description': "should have required property 'files'"}, {'property': '/project/title', 'description': "should have required property 'title'"}, @@ -184,6 +187,34 @@ def create_validator(self, submission_dir): ) return Validator(mapping_file, submission_dir, metadata_json=self.metadata_json_file) + def create_validator_with_copied_output(self, submission_dir, metadata_json=None, metadata_xlsx=None, + shallow_validation=False, validation_tasks=None, + include_second_vcf_with_no_output=False): + """ + Copy the shared validation_output fixture tree into a fresh submission_dir so a test can delete + individual output files to simulate an incomplete Nextflow run without affecting other tests. + + If include_second_vcf_with_no_output is True, a second VCF/FASTA entry ("missing_output.*") + is added to the mapping file with no corresponding check output on disk, so a test can exercise + the "some files present, some missing" partial scenario. + """ + shutil.copytree(os.path.join(self.output_dir, VALIDATION_OUTPUT_DIR), + os.path.join(submission_dir, VALIDATION_OUTPUT_DIR)) + mapping_file = os.path.join(submission_dir, 'vcf_files_mapping.csv') + vcf_files = [os.path.join(self.vcf_files, 'input_passed.vcf')] + fasta_files = [os.path.join(self.fasta_files, 'input_passed.fa')] + assembly_reports = [os.path.join(self.assembly_reports, 'input_passed.txt')] + if include_second_vcf_with_no_output: + vcf_files.append(os.path.join(self.vcf_files, 'missing_output.vcf')) + fasta_files.append(os.path.join(self.fasta_files, 'missing_output.fa')) + assembly_reports.append('') + create_mapping_file(mapping_file, vcf_files, fasta_files, assembly_reports) + kwargs = {} + if validation_tasks is not None: + kwargs['validation_tasks'] = validation_tasks + return Validator(mapping_file, submission_dir, metadata_json=metadata_json, metadata_xlsx=metadata_xlsx, + shallow_validation=shallow_validation, **kwargs) + def test_clean_up_output_dir_moves_intermediate_files_and_removes_nextflow_work_dir(self): with TemporaryDirectory() as submission_dir: validator = self.create_validator(submission_dir) @@ -233,13 +264,13 @@ def test__collect_validation_workflow_results_for_validation_task_no_previous_re self.validator_json._assess_validation_results() # assert assessed results assert self.validator_json.results['vcf_check']['pass'] == True - assert self.validator_json.results['vcf_check'][RUN_STATUS_KEY] == True + assert self.validator_json.results['vcf_check'][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS assert self.validator_json.results['evidence_type_check']['pass'] == True - assert self.validator_json.results['evidence_type_check'][RUN_STATUS_KEY] == True - assert self.validator_json.results['assembly_check'][RUN_STATUS_KEY] == False - assert self.validator_json.results['fasta_check'][RUN_STATUS_KEY] == False - assert self.validator_json.results['sample_check'][RUN_STATUS_KEY] == False - assert self.validator_json.results['metadata_check'][RUN_STATUS_KEY] == False + assert self.validator_json.results['evidence_type_check'][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + assert self.validator_json.results['assembly_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert self.validator_json.results['fasta_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert self.validator_json.results['sample_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert self.validator_json.results['metadata_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN assert self.validator_json.sub_config.get(READY_FOR_SUBMISSION_TO_EVA) == False assert self.validator_json.results[READY_FOR_SUBMISSION_TO_EVA] == False @@ -252,10 +283,10 @@ def test__collect_validation_workflow_results_for_validation_task_no_previous_re expected_evidence_type_check['pass'] = True assert saved_results['vcf_check'] == expected_vcf_check assert saved_results['evidence_type_check'] == expected_evidence_type_check - assert saved_results['assembly_check'][RUN_STATUS_KEY] == False - assert saved_results['fasta_check'][RUN_STATUS_KEY] == False - assert saved_results['sample_check'][RUN_STATUS_KEY] == False - assert saved_results['metadata_check'][RUN_STATUS_KEY] == False + assert saved_results['assembly_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert saved_results['fasta_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert saved_results['sample_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN + assert saved_results['metadata_check'][RUN_STATUS_KEY] == RUN_STATUS_DID_NOT_RUN assert saved_results[READY_FOR_SUBMISSION_TO_EVA] == False def test__collect_validation_workflow_results_for_validation_task_and_add_to_previous_results(self): @@ -666,6 +697,180 @@ def test__check_consent_statement_is_needed_for_submission(self): } assert self.validator_json._check_consent_statement_is_needed_for_submission() is True + def test__collect_file_info_to_metadata_missing_file_info_txt(self): + # Test for a nextflow run that did not complete and never produced file_info.txt, but other + # metadata check outputs are present -> task should not crash, only the per-file md5/size + # entries should be reported as missing (same as when file_info.txt exists but is empty). + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'file_info.txt')) + self.run_collect_results(validator) + assert validator.results == self.format_data_structure(expected_validation_results) + + def test__collect_file_info_to_metadata_missing_metadata_json(self): + # Test for a nextflow run that did not complete and never produced metadata.json, but other + # metadata check outputs are present -> task should not crash, only a dummy error for the + # missing metadata.json should be added. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir) + os.remove(os.path.join(validator.output_dir, 'metadata.json')) + validator._collect_validation_workflow_results() + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + expected_error = { + 'property': '/', + 'description': 'Cannot locate the metadata in JSON format. The process might have failed.' + } + assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False + + def test_collect_biovalidator_validation_results_missing_report(self): + # Test for a nextflow run that did not complete and never produced metadata_validation.txt, + # but other metadata check outputs are present -> task should not crash, only a dummy error + # for the missing biovalidator report should be added. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file, + validation_tasks=[METADATA_CHECK]) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_validation.txt')) + validator._collect_validation_workflow_results() + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + expected_error = { + 'property': '/', + 'description': 'Cannot locate metadata_validation.txt. The process might have failed.' + } + assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False + + def test__collect_metadata_results_all_files_missing(self): + # Test for a nextflow run that never completed any part of the metadata check -> the whole + # task should be reported as crashed, even though each sub-collector still records its own + # diagnostic dummy error for whichever specific file it was looking for. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, validation_tasks=[METADATA_CHECK]) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'file_info.txt')) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_validation.txt')) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_semantic_check.yml')) + os.remove(os.path.join(validator.output_dir, 'metadata.json')) + validator._collect_validation_workflow_results() + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False + + def test__collect_semantic_metadata_results_missing_yaml(self): + # Test for a nextflow run that did not complete and never produced metadata_semantic_check.yml, + # but other metadata check outputs are present -> task should not crash, only a dummy error + # for the missing semantic check should be added. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file, + validation_tasks=[METADATA_CHECK]) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'metadata_semantic_check.yml')) + validator._collect_validation_workflow_results() + assert validator.results[METADATA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + expected_error = { + 'property': '/', + 'description': 'Cannot locate metadata_semantic_check.yml. The process might have failed.' + } + assert expected_error in validator.results[METADATA_CHECK]['json_errors'] + validator._assess_validation_results() + assert validator.results[METADATA_CHECK][PASS] is False + def test__collect_trim_down_metrics_missing_yml(self): + # Test for a nextflow run that did not complete and never produced input_passed_trim_down.yml + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file, + shallow_validation=True) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'input_passed_trim_down.yml')) + validator._collect_validation_workflow_results() + # Missing metrics are conservatively assumed to require shallow validation + assert validator.results[SHALLOW_VALIDATION][TRIM_DOWN] is True + assert validator.vcf_files[0] not in validator.results[SHALLOW_VALIDATION]['metrics'] + + def test__load_sample_check_results_missing_yaml(self): + # Test for a nextflow run that did not complete and never produced sample_checker.yml + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'sample_checker.yml')) + validator._load_sample_check_results() + assert validator.results[SAMPLE_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + def test__load_evidence_check_results_missing_yaml(self): + # Test for a nextflow run that did not complete and never produced evidence_type_checker.yml + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'evidence_type_checker.yml')) + validator._load_evidence_check_results() + assert validator.results[EVIDENCE_TYPE_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + def test__load_fasta_check_results_missing_yaml(self): + # Test for a nextflow run that did not complete and never produced input_passed.fa_check.yml + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + os.remove(os.path.join(validator.output_dir, 'other_validations', 'input_passed.fa_check.yml')) + validator._collect_validation_workflow_results() + assert validator.results[FASTA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + assert 'input_passed.fa' not in validator.results[FASTA_CHECK] + validator._assess_validation_results() + assert validator.results[FASTA_CHECK][PASS] is False + + def test__collect_vcf_check_results_missing_output(self): + # Test for a nextflow run that did not complete and never produced the vcf_format check output + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + shutil.rmtree(os.path.join(validator.output_dir, 'vcf_format')) + validator._collect_validation_workflow_results() + assert validator.results[VCF_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + assert 'input_passed.vcf' not in validator.results[VCF_CHECK] + validator._assess_validation_results() + assert validator.results[VCF_CHECK][PASS] is False + + def test__collect_assembly_check_results_missing_output(self): + # Test for a nextflow run that did not complete and never produced the assembly_check output + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output(submission_dir, metadata_json=self.metadata_json_file) + shutil.rmtree(os.path.join(validator.output_dir, 'assembly_check')) + validator._collect_validation_workflow_results() + assert validator.results[ASSEMBLY_CHECK][RUN_STATUS_KEY] == RUN_STATUS_CRASHED + assert 'input_passed.vcf' not in validator.results[ASSEMBLY_CHECK] + validator._assess_validation_results() + assert validator.results[ASSEMBLY_CHECK][PASS] is False + + def test__collect_vcf_check_results_partial_output(self): + # One of two VCF files has no vcf_format output while the other succeeds -> the task should + # not crash; only the missing file gets a synthetic failing entry. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output( + submission_dir, metadata_json=self.metadata_json_file, include_second_vcf_with_no_output=True) + validator._collect_validation_workflow_results() + assert validator.results[VCF_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + assert validator.results[VCF_CHECK]['input_passed.vcf']['valid'] is True + assert validator.results[VCF_CHECK]['missing_output.vcf']['critical_list'] == ['Process failed'] + validator._assess_validation_results() + assert validator.results[VCF_CHECK][PASS] is False + + def test__collect_assembly_check_results_partial_output(self): + # One of two VCF files has no assembly_check output while the other succeeds -> the task + # should not crash; only the missing file gets a synthetic failing entry. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output( + submission_dir, metadata_json=self.metadata_json_file, include_second_vcf_with_no_output=True) + validator._collect_validation_workflow_results() + assert validator.results[ASSEMBLY_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + assert validator.results[ASSEMBLY_CHECK]['input_passed.vcf']['nb_error'] == 0 + assert validator.results[ASSEMBLY_CHECK]['missing_output.vcf']['error_list'] == ['Process failed'] + validator._assess_validation_results() + assert validator.results[ASSEMBLY_CHECK][PASS] is False + + def test__load_fasta_check_results_partial_output(self): + # One of two FASTA files has no fasta check output while the other succeeds -> the task + # should not crash; only the missing file gets a synthetic failing entry. + with TemporaryDirectory() as submission_dir: + validator = self.create_validator_with_copied_output( + submission_dir, metadata_json=self.metadata_json_file, include_second_vcf_with_no_output=True) + validator._collect_validation_workflow_results() + assert validator.results[FASTA_CHECK][RUN_STATUS_KEY] == RUN_STATUS_SUCCESS + assert 'input_passed.fa' in validator.results[FASTA_CHECK] + assert validator.results[FASTA_CHECK]['missing_output.fa']['all_insdc'] is False + assert 'connection_error' in validator.results[FASTA_CHECK]['missing_output.fa'] + validator._assess_validation_results() + assert validator.results[FASTA_CHECK][PASS] is False