diff --git a/.gitignore b/.gitignore index b7e83e9..8e696a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# Nextflow runtime +.nextflow/ +.nextflow.log* +work/ + # Python cache __pycache__/ *.pyc diff --git a/bin/compare_same_species_models.py b/bin/compare_same_species_models.py old mode 100644 new mode 100755 diff --git a/bin/encode_split.py b/bin/encode_split.py old mode 100644 new mode 100755 diff --git a/bin/label.py b/bin/label.py old mode 100644 new mode 100755 diff --git a/bin/model_processing.py b/bin/model_processing.py old mode 100644 new mode 100755 diff --git a/bin/normalize.py b/bin/normalize.py old mode 100644 new mode 100755 diff --git a/bin/preprocess.py b/bin/preprocess.py old mode 100644 new mode 100755 diff --git a/bin/profiling.py b/bin/profiling.py old mode 100644 new mode 100755 diff --git a/main.nf b/main.nf index 23ab463..b11f1b0 100644 --- a/main.nf +++ b/main.nf @@ -1,7 +1,5 @@ #!/usr/bin/env nextflow -nextflow.enable.dsl = 2 - include { PROMOTER_SEQUENCE_PIPELINE } from './workflows/main.nf' workflow { diff --git a/modules/local/encode_and_split_pairs.nf b/modules/local/encode_and_split_pairs.nf index 64665e8..2f3a4b3 100644 --- a/modules/local/encode_and_split_pairs.nf +++ b/modules/local/encode_and_split_pairs.nf @@ -1,20 +1,23 @@ process ENCODE_AND_SPLIT_PAIRS { tag { dataset_name } - publishDir { "${params.outdir}/${dataset_name}/Encoded" }, mode: params.publish_mode + publishDir "${params.outdir}/${dataset_name}/Encoded", mode: params.publish_mode input: tuple val(dataset_name), path(labeled_pairs) path human_fasta path mouse_fasta path original_gene_list + path project_lib output: tuple val(dataset_name), path("results/${dataset_name}/splits/${params.split_mode}"), emit: split_directory script: + def random_seqs_flag = params.random_seqs ? '--random-seqs' : '' """ - python ${projectDir}/bin/encode_split.py \ + export PYTHONPATH="${project_lib}/..:${project_lib}" + encode_split.py \ --input ${labeled_pairs} \ --name ${dataset_name} \ --human-fasta ${human_fasta} \ @@ -24,6 +27,7 @@ process ENCODE_AND_SPLIT_PAIRS { --gene-list ${original_gene_list} \ --val-frac ${params.val_frac} \ --test-frac ${params.test_frac} \ - --seed ${params.seed} + --seed ${params.seed} \ + ${random_seqs_flag} """ } diff --git a/modules/local/label_gene_pairs.nf b/modules/local/label_gene_pairs.nf index 85785f8..2b10421 100644 --- a/modules/local/label_gene_pairs.nf +++ b/modules/local/label_gene_pairs.nf @@ -1,17 +1,19 @@ process LABEL_GENE_PAIRS { tag { dataset_name } - publishDir { "${params.outdir}/${dataset_name}/Labeling" }, mode: params.publish_mode + publishDir "${params.outdir}/${dataset_name}/Labeling", mode: params.publish_mode input: tuple val(dataset_name), path(raw_expression_dataset) + path project_lib output: - tuple val(dataset_name), path("results/${dataset_name}/Labeling/${params.profiling}_labeling_*"), emit: labeled_pairs + tuple val(dataset_name), path("results/${dataset_name}/Labeling/*.tsv"), emit: labeled_pairs script: """ - python ${projectDir}/bin/label.py \ + export PYTHONPATH="${project_lib}/..:${project_lib}" + label.py \ --input ${raw_expression_dataset} \ --name ${dataset_name} \ --normalization ${params.normalization} \ diff --git a/modules/local/train_and_evaluate_model.nf b/modules/local/train_and_evaluate_model.nf index 7011eee..9995149 100644 --- a/modules/local/train_and_evaluate_model.nf +++ b/modules/local/train_and_evaluate_model.nf @@ -1,19 +1,20 @@ process TRAIN_AND_EVALUATE_MODEL { tag "${dataset_name}" - publishDir {"${params.outdir}/${dataset_name}/Model"}, mode: params.publish_mode + publishDir "${params.outdir}/${dataset_name}/Model", mode: params.publish_mode input: tuple val(dataset_name), path(split_directory) + path project_lib output: tuple val(dataset_name), path("results/${dataset_name}/Model"), emit: model_directory script: - optimize_argument = params.optimize ? '--optimize' : '' - + def optimize_argument = params.optimize ? '--optimize' : '' """ - python ${projectDir}/bin/model_processing.py \ + export PYTHONPATH="${project_lib}/..:${project_lib}" + model_processing.py \ --data ${split_directory} \ --name ${dataset_name} \ --metric ${params.profiling} \ diff --git a/workflows/main.nf b/workflows/main.nf index f98b93b..6b02c60 100644 --- a/workflows/main.nf +++ b/workflows/main.nf @@ -1,43 +1,55 @@ -nextflow.enable.dsl = 2 - include { LABEL_GENE_PAIRS } from '../modules/local/label_gene_pairs.nf' include { ENCODE_AND_SPLIT_PAIRS } from '../modules/local/encode_and_split_pairs.nf' include { TRAIN_AND_EVALUATE_MODEL } from '../modules/local/train_and_evaluate_model.nf' workflow PROMOTER_SEQUENCE_PIPELINE { + // --- Validate required parameters --- if (params.input == null) { error "Missing required parameter: --input" } - if (params.name == null) { error "Missing required parameter: --name" } - if (params.human_fasta == null) { error "Missing required parameter: --human_fasta" } - if (params.mouse_fasta == null) { error "Missing required parameter: --mouse_fasta" } - raw_expression_dataset_channel = Channel + // --- Stage the lib/ directory so Python scripts can import from it --- + ch_lib = Channel.value(file("${projectDir}/lib", checkIfExists: true)) + + // --- Input channels --- + // Queue channel: one tuple per dataset + ch_expression = Channel .fromPath(params.input, checkIfExists: true) - .map { raw_expression_dataset -> tuple(params.name, raw_expression_dataset) } + .map { f -> tuple(params.name, f) } - human_promoter_fasta_channel = Channel.fromPath(params.human_fasta, checkIfExists: true) - mouse_promoter_fasta_channel = Channel.fromPath(params.mouse_fasta, checkIfExists: true) - original_gene_list_channel = Channel.fromPath(params.input, checkIfExists: true) + // Value channels for single files (reusable across all items in the queue) + ch_human_fasta = Channel.value(file(params.human_fasta, checkIfExists: true)) + ch_mouse_fasta = Channel.value(file(params.mouse_fasta, checkIfExists: true)) + ch_gene_list = Channel.value(file(params.input, checkIfExists: true)) - LABEL_GENE_PAIRS(raw_expression_dataset_channel) + // --- Step 1: Label gene pairs --- + LABEL_GENE_PAIRS( + ch_expression, + ch_lib + ) + // --- Step 2: Encode sequences and split into train/val/test --- ENCODE_AND_SPLIT_PAIRS( LABEL_GENE_PAIRS.out.labeled_pairs, - human_promoter_fasta_channel, - mouse_promoter_fasta_channel, - original_gene_list_channel + ch_human_fasta, + ch_mouse_fasta, + ch_gene_list, + ch_lib ) - TRAIN_AND_EVALUATE_MODEL(ENCODE_AND_SPLIT_PAIRS.out.split_directory) + // --- Step 3: Train and evaluate the model --- + TRAIN_AND_EVALUATE_MODEL( + ENCODE_AND_SPLIT_PAIRS.out.split_directory, + ch_lib + ) }