From 2c9c487161bb37a4c06598b1f31f7c036986d467 Mon Sep 17 00:00:00 2001 From: tbrekalo Date: Thu, 12 Oct 2023 18:38:14 +0000 Subject: [PATCH 1/8] spoa-upgrade: update SPOA version, minor updates to CI; updated README.md --- .github/workflows/racon.yml | 56 +++++++++++++++++-------------------- CMakeLists.txt | 8 +++--- README.md | 8 +++--- ci/install_deps.sh | 13 +++++++++ meson.build | 2 +- vendor/spoa | 2 +- 6 files changed, 49 insertions(+), 40 deletions(-) create mode 100644 ci/install_deps.sh diff --git a/.github/workflows/racon.yml b/.github/workflows/racon.yml index c0832bb..70f43ff 100644 --- a/.github/workflows/racon.yml +++ b/.github/workflows/racon.yml @@ -6,48 +6,44 @@ on: branches: - master -env: - BUILD_TYPE: Release - jobs: test: + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - compiler: - - g++ - - g++-4.8 - - clang++ - - clang++-4.0 - - runs-on: ubuntu-18.04 + image: + - gcc:7 + - silkeh/clang:7 + build_type: + - Debug, Release + container: + image: ${{ matrix.image }} + options: --user root steps: - - uses: actions/checkout@v2 - - - if: ${{ matrix.compiler == 'g++-4.8' }} - name: Setup GCC - uses: egor-tensin/setup-gcc@v1 - with: - version: "4.8" - platform: x64 + - uses: actions/checkout@v4 + - name: Install build dependencies + run: | + chmod +x ci/install_deps.sh + ./ci/install_deps.sh - - if: ${{ matrix.compiler == 'clang++-4.0' }} - name: Setup Clang - uses: egor-tensin/setup-clang@v1 - with: - version: "4.0" - platform: x64 + - name: Create build directory + run: cmake -E make_directory build - - name: Configure CMake - run: cmake -B ${{ github.workspace }}/build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} - env: - CXX: ${{ matrix.compiler }} + - name: Generate build files + working-directory: build + run: > + cmake .. + -G Ninja + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - name: Build - run: cmake --build ${{ github.workspace }}/build --config ${{ env.BUILD_TYPE }} + working-directory: build + run: cmake --build . - name: Test - working-directory: ${{ github.workspace }}/build + working-directory: build run: | bin/racon --version bin/racon_test diff --git a/CMakeLists.txt b/CMakeLists.txt index 9332c7b..575354a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,11 +1,11 @@ -cmake_minimum_required(VERSION 3.11) +cmake_minimum_required(VERSION 3.12) project(racon VERSION 1.5.0 LANGUAGES CXX DESCRIPTION "Racon is a consensus module for de novo genome assembly.") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic") -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -57,12 +57,12 @@ if (NOT edlib_FOUND) endif () endif () -find_package(spoa 4.0.8 QUIET) +find_package(spoa 4.1.2 QUIET) if (NOT spoa_FOUND) FetchContent_Declare( spoa GIT_REPOSITORY https://github.com/rvaser/spoa - GIT_TAG 4.0.8) + GIT_TAG 4.1.2) FetchContent_GetProperties(spoa) if (NOT spoa_POPULATED) diff --git a/README.md b/README.md index 3458765..69a0626 100644 --- a/README.md +++ b/README.md @@ -18,13 +18,13 @@ Racon can also be used as a read error-correction tool. In this scenario, the MH A **wrapper script** is also available to enable easier usage to the end-user for large datasets. It has the same interface as racon but adds two additional features from the outside. Sequences can be **subsampled** to decrease the total execution time (accuracy might be lower) while target sequences can be **split** into smaller chunks and run sequentially to decrease memory consumption. Both features can be run at the same time as well. ## Dependencies -1. gcc 4.8+ or clang 3.4+ -2. cmake 3.2+ +1. gcc 7+ or clang 7+ +2. cmake 3.12+ 3. zlib ### CUDA Support -1. gcc 5.0+ -2. cmake 3.10+ +1. gcc 7+ +2. cmake 3.12+ 3. CUDA 9.0+ ## Installation diff --git a/ci/install_deps.sh b/ci/install_deps.sh new file mode 100644 index 0000000..d33988c --- /dev/null +++ b/ci/install_deps.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -eou pipefail + +apt update && apt install -y git zlib1g-dev curl ninja-build + +VERSION="3.13.4" +MIRROR_URL="https://github.com/Kitware/CMake/releases/download/v$VERSION" +DOWNLOAD="cmake-$VERSION-linux-x86_64.sh" +DOWNLOAD_FILE="/tmp/cmake.sh" + +curl -Ls "${MIRROR_URL}/${DOWNLOAD}" --output "${DOWNLOAD_FILE}" +bash "${DOWNLOAD_FILE}" --skip-license --prefix=/usr/local --exclude-subdir +which cmake diff --git a/meson.build b/meson.build index 50439d2..4def48a 100644 --- a/meson.build +++ b/meson.build @@ -5,7 +5,7 @@ project( default_options : [ 'buildtype=release', 'warning_level=3', - 'cpp_std=c++11'], + 'cpp_std=c++17'], license : 'MIT', meson_version : '>= 0.50.0') diff --git a/vendor/spoa b/vendor/spoa index 12f2ad2..1ab9ab0 160000 --- a/vendor/spoa +++ b/vendor/spoa @@ -1 +1 @@ -Subproject commit 12f2ad2a76030ed25aaa616c3a171d149ca9fd13 +Subproject commit 1ab9ab076171e5e4f5fcd4d2a2369f26c7f8f48f From f2f60c146c4bfe3c2e932b6af0403e51975de61f Mon Sep 17 00:00:00 2001 From: tbrekalo Date: Mon, 20 Nov 2023 07:26:01 +0000 Subject: [PATCH 2/8] spoa-upgrade: bumping minimum cmake version --- CMakeLists.txt | 2 +- ci/install_deps.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 575354a..b45bd36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.16) project(racon VERSION 1.5.0 LANGUAGES CXX diff --git a/ci/install_deps.sh b/ci/install_deps.sh index d33988c..bceb177 100644 --- a/ci/install_deps.sh +++ b/ci/install_deps.sh @@ -3,11 +3,11 @@ set -eou pipefail apt update && apt install -y git zlib1g-dev curl ninja-build -VERSION="3.13.4" +VERSION="3.16.5" MIRROR_URL="https://github.com/Kitware/CMake/releases/download/v$VERSION" DOWNLOAD="cmake-$VERSION-linux-x86_64.sh" DOWNLOAD_FILE="/tmp/cmake.sh" curl -Ls "${MIRROR_URL}/${DOWNLOAD}" --output "${DOWNLOAD_FILE}" bash "${DOWNLOAD_FILE}" --skip-license --prefix=/usr/local --exclude-subdir -which cmake +cmake --version From 01d061f51394b1d59999e970d1cf799280194795 Mon Sep 17 00:00:00 2001 From: tbrekalo Date: Mon, 20 Nov 2023 07:27:29 +0000 Subject: [PATCH 3/8] spoa-upgrade: simplify ci yaml --- .github/workflows/racon.yml | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/racon.yml b/.github/workflows/racon.yml index 70f43ff..29e97b5 100644 --- a/.github/workflows/racon.yml +++ b/.github/workflows/racon.yml @@ -10,40 +10,36 @@ jobs: test: runs-on: ubuntu-latest strategy: - fail-fast: false + fail-fast: true matrix: image: - gcc:7 - silkeh/clang:7 build_type: - - Debug, Release + - Debug + - Release container: image: ${{ matrix.image }} options: --user root steps: - uses: actions/checkout@v4 + - name: Install build dependencies run: | chmod +x ci/install_deps.sh ./ci/install_deps.sh - - name: Create build directory - run: cmake -E make_directory build - name: Generate build files - working-directory: build run: > - cmake .. - -G Ninja + cmake -B ./build -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - name: Build - working-directory: build - run: cmake --build . + run: cmake --build ./build - name: Test - working-directory: build run: | - bin/racon --version - bin/racon_test + ./build/bin/racon --version + ./build/bin/racon_test From 22e7f6ab8f48ba19698793da30c2f82094df0d13 Mon Sep 17 00:00:00 2001 From: tbrekalo Date: Mon, 20 Nov 2023 07:46:19 +0000 Subject: [PATCH 4/8] spoa-upgrade: supress implicit instantiation of undefined template warning --- src/polisher.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/polisher.hpp b/src/polisher.hpp index 4349840..2d4d7c5 100644 --- a/src/polisher.hpp +++ b/src/polisher.hpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace bioparser { template From 74465ce4e31e6a84cf04a79d932c7dc74335fd31 Mon Sep 17 00:00:00 2001 From: tbrekalo Date: Mon, 27 Nov 2023 18:10:33 +0000 Subject: [PATCH 5/8] spoa-upgrade: temporary spoa tag --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b45bd36..20dcbca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,7 @@ if (NOT spoa_FOUND) FetchContent_Declare( spoa GIT_REPOSITORY https://github.com/rvaser/spoa - GIT_TAG 4.1.2) + GIT_TAG fd3a5c58c1acbdf649ec956acb7dca5f24638f72) FetchContent_GetProperties(spoa) if (NOT spoa_POPULATED) From a3e8e41fd40dad47e25c45c18e1e8486e41e80fa Mon Sep 17 00:00:00 2001 From: tbrekalo Date: Mon, 27 Nov 2023 18:18:12 +0000 Subject: [PATCH 6/8] spoa-upgrade: added min_coverage argument --- README.md | 3 +++ src/main.cpp | 10 +++++++++- src/polisher.cpp | 17 +++++++++-------- src/polisher.hpp | 10 ++++++---- src/window.cpp | 4 ++-- src/window.hpp | 2 +- 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 69a0626..97d0ac9 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,9 @@ Usage of `racon` is as following: maximum allowed error rate used for filtering overlaps --no-trimming disables consensus trimming at window ends + --min-coverage + default: -1 + minimal consensus coverage -m, --match default: 3 score for matching bases diff --git a/src/main.cpp b/src/main.cpp index 74837da..b27d5f3 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -22,6 +22,7 @@ static struct option options[] = { {"quality-threshold", required_argument, 0, 'q'}, {"error-threshold", required_argument, 0, 'e'}, {"no-trimming", no_argument, 0, 'T'}, + {"min-coverage", required_argument, nullptr, 'M'}, {"match", required_argument, 0, 'm'}, {"mismatch", required_argument, 0, 'x'}, {"gap", required_argument, 0, 'g'}, @@ -48,6 +49,7 @@ int main(int argc, char** argv) { double error_threshold = 0.3; bool trim = true; + int32_t min_coverage = -1; int8_t match = 3; int8_t mismatch = -5; int8_t gap = -4; @@ -87,6 +89,9 @@ int main(int argc, char** argv) { case 'T': trim = false; break; + case 'M': + min_coverage = atoi(optarg); + break; case 'm': match = atoi(optarg); break; @@ -147,7 +152,7 @@ int main(int argc, char** argv) { auto polisher = racon::createPolisher(input_paths[0], input_paths[1], input_paths[2], type == 0 ? racon::PolisherType::kC : racon::PolisherType::kF, window_length, quality_threshold, - error_threshold, trim, match, mismatch, gap, num_threads, + error_threshold, trim, min_coverage, match, mismatch, gap, num_threads, cudapoa_batches, cuda_banded_alignment, cudaaligner_batches, cudaaligner_band_width); @@ -195,6 +200,9 @@ void help() { " maximum allowed error rate used for filtering overlaps\n" " --no-trimming\n" " disables consensus trimming at window ends\n" + " --min-coverage \n" + " default: -1\n" + " minimal consensus coverage\n" " -m, --match \n" " default: 3\n" " score for matching bases\n" diff --git a/src/polisher.cpp b/src/polisher.cpp index a5826c5..5167ac1 100644 --- a/src/polisher.cpp +++ b/src/polisher.cpp @@ -58,8 +58,9 @@ std::unique_ptr createPolisher(const std::string& sequences_path, const std::string& overlaps_path, const std::string& target_path, PolisherType type, uint32_t window_length, double quality_threshold, double error_threshold, bool trim, int8_t match, int8_t mismatch, int8_t gap, - uint32_t num_threads, uint32_t cudapoa_batches, bool cuda_banded_alignment, - uint32_t cudaaligner_batches, uint32_t cudaaligner_band_width) { + uint32_t num_threads, int32_t min_coverage, uint32_t cudapoa_batches, + bool cuda_banded_alignment, uint32_t cudaaligner_batches, + uint32_t cudaaligner_band_width) { if (type != PolisherType::kC && type != PolisherType::kF) { fprintf(stderr, "[racon::createPolisher] error: invalid polisher type!\n"); @@ -158,7 +159,7 @@ std::unique_ptr createPolisher(const std::string& sequences_path, return std::unique_ptr(new Polisher(std::move(sparser), std::move(oparser), std::move(tparser), type, window_length, quality_threshold, error_threshold, trim, match, mismatch, gap, - num_threads)); + num_threads, min_coverage)); } } @@ -167,13 +168,13 @@ Polisher::Polisher(std::unique_ptr> sparser, std::unique_ptr> tparser, PolisherType type, uint32_t window_length, double quality_threshold, double error_threshold, bool trim, int8_t match, int8_t mismatch, int8_t gap, - uint32_t num_threads) + uint32_t num_threads, int32_t min_coverage) : sparser_(std::move(sparser)), oparser_(std::move(oparser)), tparser_(std::move(tparser)), type_(type), quality_threshold_( quality_threshold), error_threshold_(error_threshold), trim_(trim), - alignment_engines_(), sequences_(), dummy_quality_(window_length, '!'), - window_length_(window_length), windows_(), - thread_pool_(std::make_shared(num_threads)), + min_coverage_(min_coverage), alignment_engines_(), sequences_(), + dummy_quality_(window_length, '!'), window_length_(window_length), + windows_(), thread_pool_(std::make_shared(num_threads)), logger_(new Logger()) { for (uint32_t i = 0; i < num_threads; ++i) { @@ -498,7 +499,7 @@ void Polisher::polish(std::vector>& dst, [&](uint64_t j) -> bool { auto it = thread_pool_->thread_map().find(std::this_thread::get_id()); // NOLINT return windows_[j]->generate_consensus( - alignment_engines_[it->second], trim_); + alignment_engines_[it->second], trim_, min_coverage_); }, i)); } diff --git a/src/polisher.hpp b/src/polisher.hpp index 2d4d7c5..8fa98b1 100644 --- a/src/polisher.hpp +++ b/src/polisher.hpp @@ -44,7 +44,7 @@ std::unique_ptr createPolisher(const std::string& sequences_path, const std::string& overlaps_path, const std::string& target_path, PolisherType type, uint32_t window_length, double quality_threshold, double error_threshold, bool trim, int8_t match, int8_t mismatch, int8_t gap, - uint32_t num_threads, uint32_t cuda_batches = 0, + uint32_t num_threads, int32_t min_coverage = -1, uint32_t cuda_batches = 0, bool cuda_banded_alignment = false, uint32_t cudaaligner_batches = 0, uint32_t cudaaligner_band_width = 0); @@ -61,8 +61,9 @@ class Polisher { const std::string& overlaps_path, const std::string& target_path, PolisherType type, uint32_t window_length, double quality_threshold, double error_threshold, bool trim, int8_t match, int8_t mismatch, int8_t gap, - uint32_t num_threads, uint32_t cuda_batches, bool cuda_banded_alignment, - uint32_t cudaaligner_batches, uint32_t cudaaligner_band_width); + uint32_t num_threads, int32_t min_coverage, uint32_t cuda_batches, + bool cuda_banded_alignment, uint32_t cudaaligner_batches, + uint32_t cudaaligner_band_width); protected: Polisher(std::unique_ptr> sparser, @@ -70,7 +71,7 @@ class Polisher { std::unique_ptr> tparser, PolisherType type, uint32_t window_length, double quality_threshold, double error_threshold, bool trim, int8_t match, int8_t mismatch, int8_t gap, - uint32_t num_threads); + uint32_t num_threads, int32_t min_coverage); Polisher(const Polisher&) = delete; const Polisher& operator=(const Polisher&) = delete; virtual void find_overlap_breaking_points(std::vector>& overlaps); @@ -83,6 +84,7 @@ class Polisher { double quality_threshold_; double error_threshold_; bool trim_; + int32_t min_coverage_; std::vector> alignment_engines_; std::vector> sequences_; diff --git a/src/window.cpp b/src/window.cpp index 8d6efe8..881708a 100644 --- a/src/window.cpp +++ b/src/window.cpp @@ -63,7 +63,7 @@ void Window::add_layer(const char* sequence, uint32_t sequence_length, } bool Window::generate_consensus(std::shared_ptr alignment_engine, - bool trim) { + bool trim, int32_t min_coverage) { if (sequences_.size() < 3) { consensus_ = std::string(sequences_.front().first, sequences_.front().second); @@ -120,7 +120,7 @@ bool Window::generate_consensus(std::shared_ptr alignment } std::vector coverages; - consensus_ = graph.GenerateConsensus(&coverages); + consensus_ = graph.GenerateConsensus(&coverages, min_coverage); if (type_ == WindowType::kTGS && trim) { uint32_t average_coverage = (sequences_.size() - 1) / 2; diff --git a/src/window.hpp b/src/window.hpp index 3ecb3db..da4988e 100644 --- a/src/window.hpp +++ b/src/window.hpp @@ -45,7 +45,7 @@ class Window { } bool generate_consensus(std::shared_ptr alignment_engine, - bool trim); + bool trim, int32_t min_coverage); void add_layer(const char* sequence, uint32_t sequence_length, const char* quality, uint32_t quality_length, uint32_t begin, From 1317be4944c9c3f0cbcfea2d52fc06dcaa57d5af Mon Sep 17 00:00:00 2001 From: tbrekalo Date: Thu, 30 Nov 2023 19:06:35 +0000 Subject: [PATCH 7/8] spoa-upgrade: update spoa tag in CMake --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 20dcbca..a8ec4a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,7 +57,7 @@ if (NOT edlib_FOUND) endif () endif () -find_package(spoa 4.1.2 QUIET) +find_package(spoa 4.1.4 QUIET) if (NOT spoa_FOUND) FetchContent_Declare( spoa From 2884beb64dfb7439ba6096503f23abf97703ca84 Mon Sep 17 00:00:00 2001 From: tbrekalo Date: Fri, 29 Dec 2023 13:24:16 +0000 Subject: [PATCH 8/8] spoa-upgrade: bugfix --- src/main.cpp | 2 +- src/polisher.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index b27d5f3..5c9d2ee 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -152,7 +152,7 @@ int main(int argc, char** argv) { auto polisher = racon::createPolisher(input_paths[0], input_paths[1], input_paths[2], type == 0 ? racon::PolisherType::kC : racon::PolisherType::kF, window_length, quality_threshold, - error_threshold, trim, min_coverage, match, mismatch, gap, num_threads, + error_threshold, trim, match, mismatch, gap, num_threads, min_coverage, cudapoa_batches, cuda_banded_alignment, cudaaligner_batches, cudaaligner_band_width); diff --git a/src/polisher.cpp b/src/polisher.cpp index 5167ac1..add5372 100644 --- a/src/polisher.cpp +++ b/src/polisher.cpp @@ -57,10 +57,10 @@ void shrinkToFit(std::vector>& src, uint64_t begin) { std::unique_ptr createPolisher(const std::string& sequences_path, const std::string& overlaps_path, const std::string& target_path, PolisherType type, uint32_t window_length, double quality_threshold, - double error_threshold, bool trim, int8_t match, int8_t mismatch, int8_t gap, - uint32_t num_threads, int32_t min_coverage, uint32_t cudapoa_batches, - bool cuda_banded_alignment, uint32_t cudaaligner_batches, - uint32_t cudaaligner_band_width) { + double error_threshold, bool trim, int8_t match, int8_t mismatch, + int8_t gap, uint32_t num_threads, int32_t min_coverage, + uint32_t cudapoa_batches, bool cuda_banded_alignment, + uint32_t cudaaligner_batches, uint32_t cudaaligner_band_width) { if (type != PolisherType::kC && type != PolisherType::kF) { fprintf(stderr, "[racon::createPolisher] error: invalid polisher type!\n");