diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e33e71..358b4d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -144,30 +144,30 @@ jobs: RUSTDOCFLAGS: "-D warnings" run: cargo doc --all-features --no-deps - minimal-versions: - name: Minimal Versions Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@nightly - - uses: swatinem/rust-cache@v2 - with: - shared-key: "ubuntu-nightly" - add-job-id-key: "false" - cache-on-failure: "true" - cache-all-crates: "true" - cache-workspace-crates: "true" - save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' }} - - name: Install cargo-hack - uses: taiki-e/install-action@v2 - with: - tool: cargo-hack - - name: Install cargo-minimal-versions - uses: taiki-e/install-action@v2 - with: - tool: cargo-minimal-versions - - name: Check with minimal versions - run: cargo hack minimal-versions check --feature-powerset + # minimal-versions: + # name: Minimal Versions Check + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v6 + # - uses: dtolnay/rust-toolchain@nightly + # - uses: swatinem/rust-cache@v2 + # with: + # shared-key: "ubuntu-nightly" + # add-job-id-key: "false" + # cache-on-failure: "true" + # cache-all-crates: "true" + # cache-workspace-crates: "true" + # save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' }} + # - name: Install cargo-hack + # uses: taiki-e/install-action@v2 + # with: + # tool: cargo-hack + # - name: Install cargo-minimal-versions + # uses: taiki-e/install-action@v2 + # with: + # tool: cargo-minimal-versions + # - name: Check with minimal versions + # run: cargo hack minimal-versions check --feature-powerset semver-checks: name: Semver Checks diff --git a/Cargo.lock b/Cargo.lock index 03639ce..36793a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -59,9 +59,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "515a1f282e33d55983c499d7e9e87082e81cbc32974825bf9032f928392d5844" dependencies = [ "compression-codecs", "compression-core", @@ -105,9 +105,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -115,9 +115,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -173,9 +173,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -197,9 +197,9 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures", @@ -239,9 +239,9 @@ dependencies = [ [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +checksum = "2fe67f2944eef52fc7b106b8c9450d243a88701a0c065f7f57235e76abaed7df" dependencies = [ "brotli", "compression-core", @@ -263,15 +263,15 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" [[package]] name = "cookie" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" dependencies = [ "percent-encoding", "time", @@ -324,33 +324,33 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crossfire" -version = "3.1.19" +version = "3.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "111ce8f7abfbac38b4bc4f32a3a2dda1a8034b873c90c7899f302cc9dbbc05ec" +checksum = "adb5e5c51e448ac2309883593482db91a5549c528d08a36aaa87bba13e5a40ae" dependencies = [ "crossbeam-utils", "futures-core", @@ -367,13 +367,13 @@ checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -409,7 +409,7 @@ dependencies = [ [[package]] name = "fast-down" -version = "5.0.2" +version = "6.0.0" dependencies = [ "async-stream", "bytes", @@ -437,6 +437,7 @@ dependencies = [ "chrono", "crossfire", "fast-down", + "file_alloc", "futures", "http-body-util", "humantime-serde", @@ -458,7 +459,7 @@ dependencies = [ [[package]] name = "fast-pull" -version = "5.0.2" +version = "6.0.0" dependencies = [ "bytes", "crossfire", @@ -473,7 +474,7 @@ dependencies = [ [[package]] name = "fast-steal" -version = "6.5.4" +version = "7.0.0" dependencies = [ "parking_lot", "portable-atomic", @@ -486,20 +487,32 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "file_alloc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152d511127cc6e0cf61912eb7f7f1a7e5c910c8c43d43f1f0ba7eb26d674f511" +dependencies = [ + "rustix", + "tokio", + "windows-sys 0.61.2", +] + [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -525,9 +538,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -540,9 +553,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -550,15 +563,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -567,38 +580,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -663,9 +676,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -688,9 +701,9 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -708,9 +721,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -749,9 +762,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -835,9 +848,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -849,9 +862,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -862,9 +875,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -876,16 +889,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -896,15 +910,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -938,9 +952,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown", @@ -968,9 +982,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "itoa" @@ -1039,9 +1053,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -1062,9 +1076,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "litrs" @@ -1083,9 +1097,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru-slab" @@ -1126,9 +1140,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -1136,9 +1150,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi", @@ -1222,9 +1236,9 @@ dependencies = [ [[package]] name = "path_helper" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1cf0534c5a78bd12bd2c8d9ad3231f22ef07fc2205b5093e63e40cab4c4a8fc" +checksum = "5c1b0eb8451e0718ff71661887df7206b717b33a17ebebe3bb74771ff5e8c1e1" dependencies = [ "mime_guess", "sanitize-filename", @@ -1245,9 +1259,9 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "pointers" @@ -1257,15 +1271,15 @@ checksum = "5dcdc93847ad24990939cce6e1804361e903efcb5f99daa5abd87943a9d6d7ba" [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -1338,9 +1352,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", @@ -1472,9 +1486,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1569,9 +1583,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.42" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "once_cell", @@ -1632,9 +1646,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -1743,7 +1757,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -1832,9 +1846,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" [[package]] name = "socket2" @@ -1880,9 +1894,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -1945,29 +1959,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] name = "time" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -1995,9 +2009,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -2005,9 +2019,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -2037,20 +2051,20 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -2072,9 +2086,9 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +checksum = "12c0ba9680044b4ce98d391a62094047eada0d64860b80166c39f4a6b5640785" dependencies = [ "indexmap", "serde_core", @@ -2096,9 +2110,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -2269,9 +2283,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -2282,9 +2296,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -2292,9 +2306,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2302,31 +2316,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -2526,9 +2540,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "yoke" @@ -2555,18 +2569,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -2602,9 +2616,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -2613,9 +2627,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -2624,15 +2638,21 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + [[package]] name = "zmij" version = "1.0.23" @@ -2650,18 +2670,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 5954ef9..0ec1e3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,16 +10,16 @@ members = [ [workspace.package] authors = ["share121 ", "Cyan Changes "] edition = "2024" -homepage = "https://github.com/fast-down/core" +homepage = "https://fd.s121.top" repository = "https://github.com/fast-down/core" license = "MIT" [workspace.dependencies] bytes = "1.7.1" crossfire = "3.0" -fast-down = { path = "crates/fast-down", version = "5.0" } -fast-pull = { path = "crates/fast-pull", version = "5.0" } -fast-steal = { path = "crates/fast-steal", version = "6.5" } +fast-down = { path = "crates/fast-down", version = "6.0" } +fast-pull = { path = "crates/fast-pull", version = "6.0" } +fast-steal = { path = "crates/fast-steal", version = "7.0" } futures = "0.3" parking_lot = "0.12" thiserror = "2.0" diff --git a/crates/fast-down-api/Cargo.toml b/crates/fast-down-api/Cargo.toml index f3d676b..9eefc71 100644 --- a/crates/fast-down-api/Cargo.toml +++ b/crates/fast-down-api/Cargo.toml @@ -5,7 +5,6 @@ authors.workspace = true edition.workspace = true description = "A convenient and easy-to-use FFI wrapper for fast-down" documentation = "https://docs.rs/fast-down-api" -readme = "README.md" homepage.workspace = true repository.workspace = true license.workspace = true @@ -40,11 +39,12 @@ reqwest = { version = "0.13.4", default-features = false, features = [ thiserror.workspace = true crossfire.workspace = true anyhow = "1.0.103" -path_helper = { version = "0.1.9", features = [ +path_helper = { version = "0.1.10", features = [ "auto_ext", "sanitize", "tokio", ] } +file_alloc = "0.2" chrono = "0.4.45" urlencoding = "2.1.3" soft-canonicalize = "0.5.6" diff --git a/crates/fast-down-api/README.md b/crates/fast-down-api/README.md index 4b9cbff..674f5e0 100644 --- a/crates/fast-down-api/README.md +++ b/crates/fast-down-api/README.md @@ -12,10 +12,10 @@ that turns the pull/push engine into a few lines of async code: spawn a download drain progress events, resume after interruption, and cancel cooperatively. - **Concurrent, resumable downloads** powered by the `fast-down` engine (work-stealing, range requests). -- **Two entry points**: `download` (auto-resume when possible) and `resume` (hard error if it can't continue). -- **Event stream**: a single channel carries prefetch, per-worker progress, rename, and error events. +- **Two layers of entry points**: the fire-and-forget `download` / `resume` wrappers, and the lower-level `plan` / `plan_resume` pair that prefetches the remote and inspects the disk _without writing a single byte_ — so you can preview the outcome and decide before committing. +- **Event stream**: a single channel carries prefetch, disk allocation, per-worker progress, resume, rename, and lifecycle events. Every run ends with exactly one `Event::Terminated(TerminationReason)`. - **Cooperative cancellation**: cancelling mid-flight preserves the `.part` / `.fd` files so you can resume later. -- **Configurable**: threads, chunk size, write method (`Mmap` / `Std`), proxies, headers, retries, and more via `PartialConfig`. +- **Configurable**: threads, chunk size, write method (`Mmap` / `Std`), proxies, headers, retries, disk pre-allocation, and more via `PartialConfig`. ## Quick start @@ -79,6 +79,12 @@ async fn main() -> anyhow::Result<()> { break; } Event::ResumeError(e) => eprintln!("resume error: {e}"), + Event::Allocating(size) => println!("pre-allocating {size} bytes on disk"), + Event::AllocError(e) => eprintln!("pre-allocation failed (continuing): {e}"), + Event::Terminated(reason) => { + println!("terminated: {reason:?}"); + break; + } _ => {} } } @@ -115,17 +121,51 @@ resume( token.cancel(); // stops fetching, keeps .part / .fd so you can resume later ``` +### Two-phase planning (inspect before you commit) + +`plan` and `plan_resume` do everything `download` / `resume` do _except_ touch +the disk: they prefetch the remote metadata, resolve the output path, and probe +the `.fd` / `.part` pair left by a previous run. The returned [`DownloadPlan`] +tells you what starting it would do — [`DownloadPlan::resume_outcome`] reports +`Resumable`, `Fresh`, or `Mismatch` — and nothing is created until you call one +of its `start` methods. Dropping the plan abandons the download with no side +effects. + +```rust,ignore +let plan = plan(url, config.clone(), tx.clone(), token.clone()).await?; + +match plan.resume_outcome() { + ResumeOutcome::Resumable { .. } => println!("will continue from a previous run"), + ResumeOutcome::Fresh => println!("will download the whole file"), + ResumeOutcome::Mismatch(e) => println!("stale state: {e} (use start_forced_resume)"), +} + +// Commit when you're ready. Each start method emits exactly one `Event::Terminated`. +plan.start().await; // resume if possible, else fresh (or refuse for plan_resume) +// plan.start_fresh().await; // ignore any saved progress and re-download +// plan.start_forced_resume().await; // continue from a mismatched state when only identity changed +``` + +`plan_resume` takes a `.part` path instead of a URL and hard-refuses a +`Mismatch` (sending `Event::ResumeError` + `TerminationReason::Failed`) rather +than restarting — because the caller asked to continue one specific file, not to +fetch it again. Pass a `url` to re-fetch the metadata, or `None` to reuse the URL +recorded in the `.fd`. + ## API overview -| Item | Purpose | -| ------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`download`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.download.html) | Start a download; auto-resume when a valid `.fd` + `.part` exist, else fresh. Observe completion by draining the `Rx` from `create_channel` until it disconnects. | -| [`resume`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.resume.html) | Resume a specific `.part` file; hard-error (`Event::ResumeError`) if it can't. Completion is observed the same way, by draining `Rx`. | -| [`create_channel`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_channel.html) | Create the `(Tx, Rx)` event channel. | -| [`create_cancellation_token`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_cancellation_token.html) | Create a `CancellationToken` for cooperative cancellation. | -| [`Event`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.Event.html) | The event enum delivered over the channel. | -| [`PartialConfig`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.PartialConfig.html) | Layered, optional configuration for a download. | -| [`StateError`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.StateError.html) | Errors surfaced via `Event::ResumeError`. | +| Item | Purpose | +| ------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`download`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.download.html) | Start a download; auto-resume when a valid `.fd` + `.part` exist, else fresh. Observe completion by draining the `Rx` from `create_channel` until it disconnects. | +| [`resume`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.resume.html) | Resume a specific `.part` file; hard-error (`Event::ResumeError`) if it can't. Completion is observed the same way, by draining `Rx`. | +| [`plan`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.plan.html) | Prefetch + probe the disk and return a [`DownloadPlan`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadPlan.html) **without writing anything**; start it only when ready. | +| [`plan_resume`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.plan_resume.html) | Like `plan` but targets a specific `.part`; refuses a `Mismatch` instead of re-downloading. | +| [`DownloadPlan`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.DownloadPlan.html) | A prepared, not-yet-started download. Inspect with `resume_outcome`, then call `start` / `start_fresh` / `start_forced_resume`. | +| [`create_channel`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_channel.html) | Create the `(Tx, Rx)` event channel. | +| [`create_cancellation_token`](https://docs.rs/fast-down-api/latest/fast_down_api/fn.create_cancellation_token.html) | Create a `CancellationToken` for cooperative cancellation. | +| [`Event`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.Event.html) | The event enum delivered over the channel. | +| [`PartialConfig`](https://docs.rs/fast-down-api/latest/fast_down_api/struct.PartialConfig.html) | Layered, optional configuration for a download. | +| [`StateError`](https://docs.rs/fast-down-api/latest/fast_down_api/enum.StateError.html) | Errors surfaced via `Event::ResumeError`. | ## How resume works @@ -140,6 +180,11 @@ size). On the next run: Cancellation leaves both files in place, so a later `resume` (or `download`) can pick up exactly where it stopped. +Every run — whether it completes, is cancelled, stops incomplete, or fails — +ends with exactly one `Event::Terminated(TerminationReason)` as the last event +on the channel, so draining `Rx` until `Terminated` is the reliable way to know +a run has finished. + ## License MIT — see [LICENSE](https://github.com/fast-down/core/blob/main/LICENSE). diff --git a/crates/fast-down-api/examples/download.rs b/crates/fast-down-api/examples/download.rs new file mode 100644 index 0000000..3aaee4e --- /dev/null +++ b/crates/fast-down-api/examples/download.rs @@ -0,0 +1,43 @@ +//! Minimal example: one-shot `download()`. +//! +//! Run: cargo run -p fast-down-api --example download + +#![allow(clippy::pedantic)] + +use fast_down_api::{ + Event, PartialConfig, TerminationReason, create_cancellation_token, create_channel, download, +}; +use url::Url; + +const URL: &str = "http://speedtest.tele2.net/10MB.zip"; + +#[tokio::main] +async fn main() { + let (tx, rx) = create_channel(); + let token = create_cancellation_token(); + download( + Url::parse(URL).unwrap(), + PartialConfig { + save_dir: Some(std::env::temp_dir().join("fd_download")), + filename: Some("file.bin".into()), + parse_filename: Some(false), + ..Default::default() + }, + tx, + token, + ); + + let mut reason = TerminationReason::Failed; + while let Ok(ev) = rx.recv().await { + match ev { + Event::Progress(p) => println!("{:5.1}% {} B/s", p.percent, p.bps), + Event::Renamed(path) => println!("renamed -> {}", path.display()), + Event::Terminated(r) => { + reason = r; + break; + } + _ => {} + } + } + println!("ended: {reason:?}"); +} diff --git a/crates/fast-down-api/examples/plan.rs b/crates/fast-down-api/examples/plan.rs new file mode 100644 index 0000000..411dbc3 --- /dev/null +++ b/crates/fast-down-api/examples/plan.rs @@ -0,0 +1,55 @@ +//! Minimal example: two-phase `plan()` + `start()`. +//! +//! Run: cargo run -p fast-down-api --example plan + +#![allow(clippy::pedantic)] + +use fast_down_api::{ + Event, PartialConfig, TerminationReason, create_cancellation_token, create_channel, plan, +}; +use url::Url; + +const URL: &str = "http://speedtest.tele2.net/10MB.zip"; + +#[tokio::main] +async fn main() { + let (tx, rx) = create_channel(); + let token = create_cancellation_token(); + let p = plan( + Url::parse(URL).unwrap(), + PartialConfig { + save_dir: Some(std::env::temp_dir().join("fd_plan")), + filename: Some("file.bin".into()), + parse_filename: Some(false), + threads: Some(8), + ..Default::default() + }, + tx, + token, + ) + .await + .unwrap(); + + // `plan()` prefetched metadata and computed paths but wrote nothing to disk. + println!( + "remote: {} bytes, final_path = {}, can_fastdown = {}", + p.info().size, + p.final_path().display(), + p.info().fast_download + ); + + p.start().await; // blocks until the run ends, emitting Terminated itself + let mut reason = TerminationReason::Failed; + while let Ok(ev) = rx.recv().await { + match ev { + Event::Progress(pr) => println!("{:5.1}% {} B/s", pr.percent, pr.bps), + Event::Renamed(path) => println!("renamed -> {}", path.display()), + Event::Terminated(r) => { + reason = r; + break; + } + _ => {} + } + } + println!("ended: {reason:?}"); +} diff --git a/crates/fast-down-api/examples/plan_resume.rs b/crates/fast-down-api/examples/plan_resume.rs new file mode 100644 index 0000000..4eddbf7 --- /dev/null +++ b/crates/fast-down-api/examples/plan_resume.rs @@ -0,0 +1,94 @@ +//! Minimal example: resume an interrupted download with `plan_resume()`. +//! +//! Run: cargo run -p fast-down-api --example plan_resume + +#![allow(clippy::pedantic)] + +use fast_down_api::{ + Event, PartialConfig, TerminationReason, create_cancellation_token, create_channel, + peek_resume, plan, plan_resume, +}; +use std::time::Duration; +use url::Url; + +const URL: &str = "http://speedtest.tele2.net/10MB.zip"; + +#[tokio::main] +async fn main() { + let out = std::env::temp_dir().join("fd_plan_resume"); + let _ = std::fs::remove_dir_all(&out); + std::fs::create_dir_all(&out).unwrap(); + let pc = PartialConfig { + save_dir: Some(out.clone()), + filename: Some("file.bin".into()), + parse_filename: Some(false), + threads: Some(8), + ..Default::default() + }; + + // Interrupt a download to leave a partial `.part`/`.fd` behind: drive a + // plan and cancel it once 30% has landed. + let (tx, rx) = create_channel(); + let token = create_cancellation_token(); + let p = plan(Url::parse(URL).unwrap(), pc.clone(), tx, token.clone()) + .await + .unwrap(); + let mut cancelled = false; + tokio::select! { + () = p.start() => {} + () = async { + while let Ok(ev) = rx.recv().await { + if let Event::Progress(pr) = ev + && pr.percent > 30.0 + { + token.cancel(); + cancelled = true; + break; + } + } + } => {} + } + println!("interrupted at 30%: {cancelled}"); + + // The final `.fd` flush races the cancellation; wait (bounded) for it. + let tmp = out.join("file.bin.part"); + let fd = out.join("file.bin.fd"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + while !fd.exists() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(50)).await; + } + + // `peek_resume`: pure inspection of the on-disk state, no download. + if let Ok(state) = peek_resume(&tmp).await { + let done: u64 = state.get_progress().iter().map(|r| r.end - r.start).sum(); + println!( + "peek_resume: {} byte-ranges, {done} bytes fetched", + state.get_progress().len() + ); + } + + // `plan_resume`: plan that continues from the partial `.part`, then start. + let (tx, rx) = create_channel(); + let token = create_cancellation_token(); + let p2 = plan_resume(&tmp, Some(Url::parse(URL).unwrap()), pc, tx, token) + .await + .unwrap(); + p2.start().await; + let mut reason = TerminationReason::Failed; + while let Ok(ev) = rx.recv().await { + match ev { + Event::Resumed { progress, .. } => { + let got: u64 = progress.iter().map(|r| r.end - r.start).sum(); + println!("resuming from {got} bytes"); + } + Event::Progress(pr) => println!("{:5.1}% {} B/s", pr.percent, pr.bps), + Event::Renamed(path) => println!("renamed -> {}", path.display()), + Event::Terminated(r) => { + reason = r; + break; + } + _ => {} + } + } + println!("ended: {reason:?}"); +} diff --git a/crates/fast-down-api/src/config.rs b/crates/fast-down-api/src/config.rs index 0fbc9c0..77b6b93 100644 --- a/crates/fast-down-api/src/config.rs +++ b/crates/fast-down-api/src/config.rs @@ -29,9 +29,6 @@ pub struct Config { /// 文件名 pub filename: String, - /// 用于在 prefetch 阶段生成占位文件名 - pub gid: String, - /// Number of threads. Recommended: `32` / `16` / `8`. More threads does not always mean faster. #[config(default = 32)] pub threads: usize, @@ -42,11 +39,11 @@ pub struct Config { /// Custom request headers. pub headers: HashMap, - /// Minimum chunk size in bytes. Recommended: `8 * 1024 * 1024` + /// Minimum chunk size in bytes. Recommended: `1024 * 1024` /// /// - Chunks that are too small may cause heavy contention. /// - When chunking is no longer possible, speculative mode is used. - #[config(default = 8 * 1024 * 1024)] + #[config(default = 1024 * 1024)] pub min_chunk_size: u64, /// Whether to ensure data is fully flushed to disk. Recommended: `false` @@ -54,6 +51,21 @@ pub struct Config { /// Set to `true` only if you need to power off immediately after download. pub sync_all: bool, + /// Reserve the whole file size on disk before downloading. Recommended: `false` + /// + /// Claiming the space up front keeps the file in fewer fragments and turns + /// a full disk into an error before any bytes are fetched instead of + /// halfway through. The cost depends on the platform: `fallocate` on Unix + /// and `SetFileValidData` on Windows reserve the space instantly, but + /// without them the fallback writes zeros across the whole file, which + /// takes as long as a full-size write pass. + /// + /// `SetFileValidData` needs the `SeManageVolumePrivilege`, which is + /// enabled automatically when available; without it Windows falls back to + /// the zero-fill path. Failure to reserve is reported as + /// [`crate::Event::AllocError`] and does not stop the download. + pub pre_alloc: bool, + /// Write buffer size in bytes. Recommended: `16 * 1024 * 1024` /// /// - Only effective for [`WriteMethod::Std`]. Reduces the number of `write` syscalls @@ -325,6 +337,11 @@ mod range_list { } ranges.push(start..end_inclusive.saturating_add(1)); } + // Normalize to ascending `start` order on load so downstream + // consumers (resume gap computation, `part_shortfall`, ...) can + // rely on the sorted invariant even when the `.fd` was hand-edited + // or written by an older build that stored chunks out of order. + ranges.sort_by_key(|r| r.start); Ok(Some(ranges)) } } @@ -505,4 +522,14 @@ mod range_list_tests { "wrapping end must be rejected with a clear error, got: {msg}" ); } + + /// Deserialize must normalize chunks to ascending `start` order so downstream + /// consumers can rely on the sorted invariant even for hand-edited `.fd` files. + #[test] + #[allow(clippy::single_range_in_vec_init)] + fn downloaded_chunk_deserialize_normalizes_order() { + let toml = "downloaded_chunk = \"5-9,1-3,100-100\"\n"; + let pc: PartialConfig = toml::from_str(toml).unwrap(); + assert_eq!(pc.downloaded_chunk, Some(vec![1..4, 5..10, 100..101])); + } } diff --git a/crates/fast-down-api/src/core/download/mod.rs b/crates/fast-down-api/src/core/download/mod.rs index a16277d..85328b5 100644 --- a/crates/fast-down-api/src/core/download/mod.rs +++ b/crates/fast-down-api/src/core/download/mod.rs @@ -1,11 +1,7 @@ -use crate::core::download::overwrite::OverwriteOption; use crate::utils::ForceSendExt; use crate::{DownloadState, Event, StateError}; -use crate::{PartialConfig, Tx, prefetch, tx_err, utils::gen_path}; +use crate::{PartialConfig, TerminationReason, Tx}; use fast_down::UrlInfo; -use inherit_config::ConfigLayer; -use overwrite::overwrite; -use path_helper::IterStemExt; use std::path::Path; use tokio::fs::{self, OpenOptions}; use tokio_util::sync::CancellationToken; @@ -13,8 +9,11 @@ use url::Url; mod overwrite; mod pipeline; +mod plan; mod progress_reporter; +pub use plan::*; + fn open_existing() -> OpenOptions { let mut o = OpenOptions::new(); o.read(true).write(true).truncate(false).create(false); @@ -33,12 +32,14 @@ fn open_create_new() -> OpenOptions { /// Attempt to load and validate a resume state from disk. /// -/// This helper consolidates the resume logic shared between `run_download` (overwrite and non-overwrite branches) -/// and `run_resume`. It checks if both `.fd` and `.part` exist, validates the state against the current server info, -/// and merges the new config into the loaded state. +/// This checks that both the `.fd` and `.part` exist, validates the state +/// against the current server info, and merges the new config into the loaded +/// state. /// -/// Returns `Ok(Some(state))` if resume is possible, `Ok(None)` if no resume state exists (caller should start fresh), -/// or `Err(StateError)` if the state exists but is invalid. +/// Returns `Ok(Some(state))` if resume is possible, `Ok(None)` if there is +/// nothing usable to resume from (the pair is incomplete, or the `.part` is +/// shorter than the recorded progress), or `Err(StateError)` if the state exists +/// but does not describe the current remote file. #[allow(clippy::result_large_err)] async fn try_load_resume_state( url: &Url, @@ -61,26 +62,16 @@ async fn try_load_resume_state( // Validate the state against current server info state.validate(info)?; - // Check that the .part file size is consistent with the recorded progress. - // Only applies to regular files — directories or other special files are not a - // valid .part and will fail later when build_pipeline tries to open them. - if let Ok(metadata) = fs::metadata(tmp_path).await - && metadata.is_file() - { - let actual_size = metadata.len(); - let recorded_progress = state.get_progress(); - let max_recorded_end = recorded_progress.iter().map(|r| r.end).max().unwrap_or(0); + // Merge the new config into the loaded state + state.merge_config(partial_config); - if actual_size < max_recorded_end { - // The .part file is smaller than what we think is already downloaded. - // This could lead to data corruption if we continue with resume. - // Treat this as if no valid state exists and start fresh. - return Ok(None); - } + // Check after merging so caller-supplied progress is validated too. A + // `.part` shorter than any claimed range would otherwise be extended with + // zeros while the download engine skipped those bytes. + if state.part_shortfall(tmp_path).await.is_some() { + return Ok(None); } - // Merge the new config into the loaded state - state.merge_config(partial_config); state.refresh_identity(url, info); Ok(Some(state)) @@ -89,30 +80,27 @@ async fn try_load_resume_state( /// Spawn a detached background download task that resumes automatically when /// possible. /// -/// The task first `prefetch`es metadata, then either resumes from a valid -/// `.fd`/`.part` state or starts a fresh download (falling back silently when -/// resume is impossible). Progress and lifecycle events are delivered through -/// `tx`. +/// This is the one-shot form of [`plan`] followed by [`DownloadPlan::start`]: +/// the task prefetches metadata, then either resumes from a valid `.fd`/`.part` +/// state or starts a fresh download (falling back silently when resume is +/// impossible). Use [`plan`] directly when the decision should be shown to a +/// user first. Progress and lifecycle events are delivered through `tx`. /// -/// Completion is observed through the [`Rx`](crate::Rx) you created alongside -/// `tx`: the spawned task holds the only `Tx` clones, so the receiver -/// disconnects once the task has fully finished — including the final -/// `overwrite`. Drain `rx` until it disconnects to await completion; keep the +/// The run always ends with exactly one [`Event::Terminated`], which is the last +/// event on the channel — including when planning itself fails or is cancelled. +/// A caller can wait for it instead of draining `rx` until it disconnects; the +/// channel still disconnects afterwards, because the spawned task holds the only +/// `Tx` clones. Keep the /// [`CancellationToken`](crate::create_cancellation_token) you passed in if you /// need to cancel. pub fn download(url: Url, partial_config: PartialConfig, tx: Tx, token: CancellationToken) { tokio::spawn( async move { let token2 = token.clone(); - let opt = token - .run_until_cancelled( - async move { run_download(url, partial_config, tx, token2).await }, - ) - .await - .flatten(); - if let Some(opt) = opt { - overwrite(opt).await; - } + let planned = + Box::pin(token.run_until_cancelled(plan(url, partial_config, tx.clone(), token2))) + .await; + Box::pin(drive(planned, &tx)).await; } .force_send(), ); @@ -121,24 +109,26 @@ pub fn download(url: Url, partial_config: PartialConfig, tx: Tx, token: Cancella /// Spawn a detached task that resumes a previously interrupted download from its /// `.part` file. /// -/// `url` is optional. When `Some`, the resume resolves and validates against -/// that URL exactly as before. When `None`, the task reuses the **initial URL** -/// persisted in the `.fd` state file — the one the original `download` recorded -/// (the durable initial URL, not the transient redirect/`final_url`). So a -/// caller can resume purely from the `.part` path; redirects are re-resolved -/// through a fresh prefetch on every resume. +/// This is the one-shot form of [`plan_resume`] followed by +/// [`DownloadPlan::start`]. `url` is optional. When `Some`, the resume resolves +/// and validates against that URL exactly as before. When `None`, the task +/// reuses the **initial URL** persisted in the `.fd` state file — the one the +/// original `download` recorded (the durable initial URL, not the transient +/// redirect/`final_url`). So a caller can resume purely from the `.part` path; +/// redirects are re-resolved through a fresh prefetch on every resume. /// -/// If the download cannot be continued — the `.fd` state file is missing, the -/// server does not support range requests, or the remote file changed — the -/// task emits [`Event::ResumeError`](crate::Event::ResumeError) and returns -/// **without** falling back to a full re-download. If `tmp_path` itself does not -/// exist, the call falls back to a fresh download **only when a `url` is -/// available**; with `url = None` there is nothing to fetch, so it emits +/// If the download cannot be continued — `tmp_path` is not a `.part` file, the +/// `.fd` state file is missing, the server does not support range requests, or +/// the remote file changed — the task emits +/// [`Event::ResumeError`](crate::Event::ResumeError) and stops **without** +/// falling back to a full re-download. If `tmp_path` itself does not exist, the +/// call falls back to a fresh download **only when a `url` is available**; with +/// `url = None` there is nothing to fetch, so it emits /// `ResumeError(StateError::NoUrl)` instead. Likewise, when `url = None` but the /// `.fd` carries no resolvable URL, the call reports `StateError::NoUrl`. /// -/// Completion is observed the same way as [`download`](crate::download): drain the `Rx` paired -/// with `tx` until it disconnects. +/// Completion is observed the same way as [`download`]: wait for the single +/// [`Event::Terminated`], or drain the `Rx` until it disconnects. pub fn resume( tmp_path: impl AsRef, url: Option, @@ -146,188 +136,73 @@ pub fn resume( tx: Tx, token: CancellationToken, ) { - let tmp_path = tmp_path.as_ref(); - if tmp_path.extension() != Some(std::ffi::OsStr::new("part")) { - let _ = tx.send(Event::ResumeError(StateError::Open(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "tmp_path must end with .part extension", - )))); - return; - } - let tmp_path = tmp_path.to_path_buf(); - + let tmp_path = tmp_path.as_ref().to_path_buf(); tokio::spawn( async move { let token2 = token.clone(); - let opt = Box::pin(token.run_until_cancelled(async move { - run_resume(&tmp_path, url, partial_config, tx, token2).await - })) - .await - .flatten(); - if let Some(opt) = opt { - overwrite(opt).await; - } + let planned = Box::pin(token.run_until_cancelled(plan_resume( + tmp_path, + url, + partial_config, + tx.clone(), + token2, + ))) + .await; + Box::pin(drive(planned, &tx)).await; } .force_send(), ); } -async fn run_download( - url: Url, +/// Spawn a detached task that downloads from a `.fd` state file used as a +/// download manifest. +/// +/// This is the one-shot form of [`plan_from_fd`]: the task loads the `.fd`, +/// resolves and validates against the remote file, then either resumes from a +/// present `.part` or — when the `.part` is missing — reuses the `.fd`'s url / +/// config and downloads the whole file from scratch. See [`plan_from_fd`] for +/// the full contract. +/// +/// Completion is observed the same way as [`download`]: wait for the single +/// [`Event::Terminated`], or drain the `Rx` until it disconnects. +pub fn download_from_fd( + fd_path: impl AsRef, + url: Option, partial_config: PartialConfig, tx: Tx, token: CancellationToken, -) -> Option { - let config = partial_config.clone().build(); - let (info, resp) = prefetch(&url, &config, &tx).await?; - let can_resume = config.resume && info.fast_download; - - let origin_path = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError, None); - - if config.overwrite { - let cfg_path = origin_path.with_added_extension("fd"); - let tmp_path = origin_path.with_added_extension("part"); - - let state = if can_resume - && let Ok(Some(s)) = - try_load_resume_state(&url, &cfg_path, &tmp_path, &info, &partial_config).await - { - let _ = tx.send(Event::Resumed { - config_path: cfg_path, - progress: s.get_progress(), - size: info.size, - }); - s - } else { - tx_err!( - open_create().open(tmp_path).await, - tx, - BuildPusherError, - None - ); - DownloadState::new(&url, &info, &partial_config, &cfg_path) - }; - return Some(OverwriteOption { - state, - final_path: origin_path, - info, - resp, - tx, - token, - }); - } - - for base_path in origin_path.iter_stem() { - let tmp_path = base_path.with_added_extension("part"); - let cfg_path = base_path.with_added_extension("fd"); - - let state = if can_resume - && let Ok(Some(s)) = - try_load_resume_state(&url, &cfg_path, &tmp_path, &info, &partial_config).await - { - let _ = tx.send(Event::Resumed { - config_path: cfg_path, - progress: s.get_progress(), - size: info.size, - }); - s - } else { - match open_create_new().open(&tmp_path).await { - Ok(_) => DownloadState::new(&url, &info, &partial_config, &cfg_path), - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(e) => { - let _ = tx.send(Event::BuildPusherError(e)); - return None; - } - } - }; - return Some(OverwriteOption { - state, - final_path: origin_path, - info, - resp, - tx, - token, - }); - } - unreachable!() +) { + let fd_path = fd_path.as_ref().to_path_buf(); + tokio::spawn( + async move { + let token2 = token.clone(); + let planned = Box::pin(token.run_until_cancelled(plan_from_fd( + fd_path, + url, + partial_config, + tx.clone(), + token2, + ))) + .await; + Box::pin(drive(planned, &tx)).await; + } + .force_send(), + ); } -async fn run_resume( - tmp_path: &Path, - url: Option, - mut partial_config: PartialConfig, - tx: Tx, - token: CancellationToken, -) -> Option { - partial_config.overwrite = Some(false); - let tmp_exists = fs::try_exists(tmp_path).await.unwrap_or(false); - if !tmp_exists { - // A missing tmp_path falls back to a fresh download, but that still needs a - // URL to fetch. Without one there is nothing to resume against. - let Some(url) = url else { - let _ = tx.send(Event::ResumeError(StateError::NoUrl( - tmp_path.to_path_buf(), - ))); - return None; - }; - partial_config.resume = Some(false); - return run_download(url, partial_config, tx, token).await; - } - - let cfg_path = tmp_path.with_extension("fd"); - let state = tx_err!(DownloadState::load(&cfg_path).await, tx, ResumeError, None); - - // Resolve the URL to prefetch/validate against: the caller's URL when given, - // otherwise the durable initial URL recorded in the `.fd`. If neither exists - // (url = None and an old `.fd` that stored no URL) we cannot resume. - let Some(url) = url.or_else(|| { - state - .lock_inner() - .url - .clone() - .filter(|s| matches!(s.scheme(), "http" | "https")) - }) else { - let _ = tx.send(Event::ResumeError(StateError::NoUrl( - tmp_path.to_path_buf(), - ))); - return None; - }; - - partial_config.resume = Some(true); - let config = partial_config.clone().build(); - let (info, resp) = prefetch(&url, &config, &tx).await?; - if !info.fast_download { - let _ = tx.send(Event::ResumeError(StateError::NotResumable(info, resp))); - return None; - } - - match try_load_resume_state(&url, &cfg_path, tmp_path, &info, &partial_config).await { - Ok(Some(state)) => { - let _ = tx.send(Event::Resumed { - config_path: cfg_path, - progress: state.get_progress(), - size: info.size, - }); - - let final_path = tx_err!(gen_path(&url, &info, &config).await, tx, GenPathError, None); - Some(OverwriteOption { - state, - final_path, - info, - resp, - tx, - token, - }) - } - Ok(None) => { - // No valid state found, fall back to fresh download - partial_config.resume = Some(false); - run_download(url, partial_config, tx, token).await +/// Start a freshly-made plan, or report why there is none. +/// +/// `None` means the cancellation token fired while planning. Either way exactly +/// one [`Event::Terminated`] reaches the channel. +async fn drive(planned: Option>, tx: &Tx) { + match planned { + None => { + let _ = tx.send(Event::Terminated(TerminationReason::Cancelled)); } - Err(e) => { - let _ = tx.send(Event::ResumeError(e)); - None + Some(Err(e)) => { + e.emit(tx); + let _ = tx.send(Event::Terminated(TerminationReason::Failed)); } + Some(Ok(prepared)) => Box::pin(prepared.start()).await, } } diff --git a/crates/fast-down-api/src/core/download/overwrite.rs b/crates/fast-down-api/src/core/download/overwrite.rs index d8997e8..ef71c6b 100644 --- a/crates/fast-down-api/src/core/download/overwrite.rs +++ b/crates/fast-down-api/src/core/download/overwrite.rs @@ -11,7 +11,8 @@ //! `overwrite` is disabled). use super::progress_reporter::ProgressReporter; use crate::{ - DownloadState, Event, PartialConfig, Tx, core::download::pipeline::build_pipeline, tx_err, + DownloadState, Event, PartialConfig, TerminationReason, Tx, + core::download::pipeline::build_pipeline, tx_err, }; use fast_down::{UrlInfo, invert, multi::download_multi, single::download_single}; use inherit_config::ConfigLayer; @@ -59,8 +60,12 @@ pub struct OverwriteOption { /// /// If the token is cancelled or the download did not complete, the `.part` and /// `.fd` files are left in place so a later resume can continue. +/// +/// The returned [`TerminationReason`] describes how the run ended. This +/// function does not emit [`crate::Event::Terminated`]; the caller owns that +/// so exactly one terminal event reaches the channel per run. #[allow(clippy::too_many_lines)] -pub async fn overwrite(option: OverwriteOption) { +pub async fn overwrite(option: OverwriteOption) -> TerminationReason { let OverwriteOption { state, final_path, @@ -69,7 +74,12 @@ pub async fn overwrite(option: OverwriteOption) { tx, token, } = option; - tx_err!(state.store().await, tx, StateSaveError); + tx_err!( + state.store().await, + tx, + StateSaveError, + TerminationReason::Failed + ); let _ = state.take_dirty(); let inner_state = state.lock_inner().clone(); @@ -78,10 +88,9 @@ pub async fn overwrite(option: OverwriteOption) { let tmp_path = state.tmp_path(); let config = &inner_state.config; - let pipeline = - build_pipeline(&info.final_url, config, &info, resp, &tmp_path, &tx, &token).await; + let pipeline = build_pipeline(config, &info, resp, &tmp_path, &tx, &token).await; let Some((puller, pusher)) = pipeline else { - return; + return TerminationReason::Failed; }; let _ = tx.send(Event::Start { @@ -188,21 +197,31 @@ pub async fn overwrite(option: OverwriteOption) { if let Err(e) = state.store().await { let _ = tx.send(Event::StateSaveError(e)); } - return; + return if token.is_cancelled() { + TerminationReason::Cancelled + } else { + TerminationReason::Incomplete + }; } let final_path = if config.overwrite { final_path } else { - tx_err!(gen_unique_path(final_path).await, tx, GenPathError) + tx_err!( + gen_unique_path(final_path).await, + tx, + GenPathError, + TerminationReason::Failed + ) }; if let Err(e) = fs::rename(tmp_path, &final_path).await { if !config.overwrite { let _ = fs::remove_file(&final_path).await; } let _ = tx.send(Event::RenameFailed(e)); - return; + return TerminationReason::Failed; } let _ = fs::remove_file(&state.config_path).await; let _ = tx.send(Event::Renamed(final_path)); + TerminationReason::Completed } diff --git a/crates/fast-down-api/src/core/download/pipeline.rs b/crates/fast-down-api/src/core/download/pipeline.rs index 2d8d69d..78a35c4 100644 --- a/crates/fast-down-api/src/core/download/pipeline.rs +++ b/crates/fast-down-api/src/core/download/pipeline.rs @@ -9,11 +9,11 @@ use fast_down::{ BoxPusher, CacheFilePusher, MmapFilePusher, UrlInfo, fast_puller::{FastDownPuller, FastDownPullerOptions}, }; +use file_alloc::FileAlloc; use parking_lot::Mutex; use reqwest::Response; use std::{path::Path, sync::Arc}; use tokio_util::sync::CancellationToken; -use url::Url; /// Construct the (puller, pusher) pipeline for a `.part` file. /// @@ -31,8 +31,12 @@ use url::Url; /// without an extra round-trip. /// * `path` is the `.part` file; `tx` receives error events; `token` makes /// construction cancellable. +/// +/// With [`Config::pre_alloc`] enabled and a known size, the whole file is +/// reserved on disk right after the `.part` file is opened +/// ([`Event::Allocating`]). A reservation that fails is reported as +/// [`Event::AllocError`] and does not abort the pipeline. pub async fn build_pipeline( - url: &Url, config: &Config, info: &UrlInfo, resp: Response, @@ -44,7 +48,7 @@ pub async fn build_pipeline( let built = token .run_until_cancelled(async move { let puller = FastDownPuller::new(FastDownPullerOptions { - url: url.clone(), + url: info.final_url.clone(), headers: build_header(&config.headers).into(), proxy: config.proxy.as_deref(), accept_invalid_certs: config.accept_invalid_certs, @@ -57,10 +61,21 @@ pub async fn build_pipeline( }) .map_err(Event::BuildClientError)?; - let file = open_existing() + file_alloc::init_fast_alloc(); + let mut file = open_existing() .open(path) .await .map_err(Event::BuildPusherError)?; + if info.size > 0 { + let _ = tx.send(Event::Allocating(info.size)); + if config.pre_alloc { + if let Err(e) = file.allocate(info.size).await { + let _ = tx.send(Event::AllocError(e)); + } + } else if let Err(e) = file.try_allocate(info.size).await { + let _ = tx.send(Event::AllocError(e)); + } + } let pusher = if cfg!(target_pointer_width = "64") && info.fast_download && config.write_method == WriteMethod::Mmap diff --git a/crates/fast-down-api/src/core/download/plan/common.rs b/crates/fast-down-api/src/core/download/plan/common.rs new file mode 100644 index 0000000..0319973 --- /dev/null +++ b/crates/fast-down-api/src/core/download/plan/common.rs @@ -0,0 +1,134 @@ +use super::super::{ + open_create, open_create_new, + overwrite::{OverwriteOption, overwrite}, +}; +use crate::{ + Config, DownloadState, Event, PartialConfig, StateError, TerminationReason, Tx, tx_err, +}; +use fast_down::UrlInfo; +use path_helper::IterStemExt; +use reqwest::Response; +use std::io; +use std::path::PathBuf; +use tokio::fs; +use tokio_util::sync::CancellationToken; +use url::Url; + +#[derive(Clone, Copy)] +pub(super) enum Fallback { + Fresh, + ManifestFresh, + Fail, +} + +/// Data shared by every planned action. +pub(super) struct PlanCommon { + pub(super) url: Url, + pub(super) partial_config: PartialConfig, + pub(super) config: Config, + pub(super) info: UrlInfo, + pub(super) resp: Response, + pub(super) final_path: PathBuf, + pub(super) cfg_path: PathBuf, + pub(super) tmp_path: PathBuf, + pub(super) tx: Tx, + pub(super) token: CancellationToken, +} + +impl PlanCommon { + pub(super) async fn ensure_parent(&self) -> bool { + if let Some(parent) = self.final_path.parent() + && let Err(e) = fs::create_dir_all(parent).await + { + let _ = self.tx.send(Event::BuildPusherError(e)); + return false; + } + true + } + + fn emit_resumed(&self, state: &DownloadState) { + let _ = self.tx.send(Event::Resumed { + config_path: self.cfg_path.clone(), + progress: state.get_progress(), + size: self.info.size, + }); + } + + async fn run_overwrite(self, state: DownloadState) -> TerminationReason { + overwrite(OverwriteOption { + state, + final_path: self.final_path, + info: self.info, + resp: self.resp, + tx: self.tx, + token: self.token, + }) + .await + } + + pub(super) async fn run_existing( + self, + state: DownloadState, + emit_resumed: bool, + ) -> TerminationReason { + if !self.ensure_parent().await { + return TerminationReason::Failed; + } + if emit_resumed { + self.emit_resumed(&state); + } + self.run_overwrite(state).await + } + + /// Atomically claim a fresh `.part`/`.fd` pair and start from byte zero. + pub(super) async fn claim_and_run(mut self) -> TerminationReason { + self.partial_config.downloaded_chunk = None; + if !self.ensure_parent().await { + return TerminationReason::Failed; + } + + if self.config.overwrite { + tx_err!( + open_create().open(&self.tmp_path).await, + self.tx, + BuildPusherError, + TerminationReason::Failed + ); + let state = + DownloadState::new(&self.url, &self.info, &self.partial_config, &self.cfg_path); + return self.run_overwrite(state).await; + } + + for base_path in self.final_path.iter_stem() { + let tmp_path = base_path.with_added_extension("part"); + let cfg_path = base_path.with_added_extension("fd"); + match open_create_new().open(&tmp_path).await { + Ok(_) => match open_create_new().open(&cfg_path).await { + Ok(_) => { + let state = DownloadState::new( + &self.url, + &self.info, + &self.partial_config, + &cfg_path, + ); + return self.run_overwrite(state).await; + } + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + let _ = fs::remove_file(&tmp_path).await; + } + Err(e) => { + let _ = fs::remove_file(&tmp_path).await; + let _ = self.tx.send(Event::StateSaveError(StateError::Save(e))); + return TerminationReason::Failed; + } + }, + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} + Err(e) => { + let _ = self.tx.send(Event::BuildPusherError(e)); + return TerminationReason::Failed; + } + } + } + unreachable!() + } +} diff --git a/crates/fast-down-api/src/core/download/plan/fresh.rs b/crates/fast-down-api/src/core/download/plan/fresh.rs new file mode 100644 index 0000000..b720ad0 --- /dev/null +++ b/crates/fast-down-api/src/core/download/plan/fresh.rs @@ -0,0 +1,216 @@ +use super::{ + DownloadPlan, Fallback, MismatchPlan, MismatchPolicy, PlanCommon, PlanError, PlanKind, + ResumeOutcome, ResumePlan, plan_common, probe, probe_resume, +}; +use crate::{PartialConfig, StateError, TerminationReason, Tx}; +use path_helper::IterStemExt; +use std::path::PathBuf; +use tokio::fs; +use tokio_util::sync::CancellationToken; +use url::Url; + +pub(super) struct FreshPlan { + common: PlanCommon, + outcome: ResumeOutcome, +} + +impl FreshPlan { + pub(super) const fn new(common: PlanCommon) -> Self { + Self { + common, + outcome: ResumeOutcome::Fresh, + } + } + + pub(super) const fn common(&self) -> &PlanCommon { + &self.common + } + + pub(super) const fn outcome(&self) -> &ResumeOutcome { + &self.outcome + } + + pub(super) fn into_common(self) -> PlanCommon { + self.common + } + + pub(super) async fn run(self) -> TerminationReason { + self.common.claim_and_run().await + } +} + +/// Prepare a download without touching the filesystem. +/// +/// Prefetches the remote metadata, resolves the output path, and inspects the +/// `.fd`/`.part` pair that a previous run may have left behind. The returned +/// [`DownloadPlan`] reports what starting it would do; nothing is created until +/// it is started. +/// +/// When `overwrite` is disabled the probe walks the `name`, `name (1)`, +/// `name (2)` … sequence looking for a resumable pair, and settles on the first +/// unclaimed name. A pair whose state exists but fails validation is reported as +/// [`ResumeOutcome::Mismatch`] instead of being skipped silently, so the caller +/// can offer to force the resume. +/// +/// # Errors +/// +/// Returns [`PlanError`] when the client cannot be built, the prefetch exhausts +/// its retries, or the output path cannot be computed. +#[allow(clippy::result_large_err)] +pub async fn plan( + url: Url, + mut partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, +) -> Result { + partial_config.downloaded_chunk = None; + let probe = probe(&url, &partial_config, &tx).await?; + let can_resume = probe.config.resume && probe.info.fast_download; + + if probe.config.overwrite { + plan_overwrite(url, partial_config, probe, tx, token, can_resume).await + } else { + plan_without_overwrite(url, partial_config, probe, tx, token, can_resume).await + } +} + +#[allow(clippy::result_large_err)] +async fn plan_overwrite( + url: Url, + partial_config: PartialConfig, + probe: super::Probe, + tx: Tx, + token: CancellationToken, + can_resume: bool, +) -> Result { + let cfg_path = probe.final_path.with_added_extension("fd"); + let tmp_path = probe.final_path.with_added_extension("part"); + let resume_probe = if can_resume { + probe_resume(&url, &cfg_path, &tmp_path, &probe.info, &partial_config).await + } else { + (None, ResumeOutcome::Fresh) + }; + let inner = match resume_probe { + (Some(state), _) => { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + Some(&state), + ); + PlanKind::Resume(ResumePlan::new(common, &state, Fallback::Fresh)) + } + (None, ResumeOutcome::Mismatch(error)) => { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + None, + ); + PlanKind::Mismatch(MismatchPlan::new(common, error, MismatchPolicy::Fresh)) + } + (None, _) => { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + None, + ); + PlanKind::Fresh(FreshPlan::new(common)) + } + }; + Ok(DownloadPlan { inner }) +} + +#[allow(clippy::result_large_err)] +async fn plan_without_overwrite( + url: Url, + partial_config: PartialConfig, + probe: super::Probe, + tx: Tx, + token: CancellationToken, + can_resume: bool, +) -> Result { + let mut mismatch: Option<(StateError, PathBuf, PathBuf)> = None; + for base_path in probe.final_path.iter_stem() { + let tmp_path = base_path.with_added_extension("part"); + let cfg_path = base_path.with_added_extension("fd"); + + let tmp_exists = fs::try_exists(&tmp_path).await.unwrap_or(false); + let cfg_exists = fs::try_exists(&cfg_path).await.unwrap_or(false); + if !tmp_exists && !cfg_exists { + // The first unclaimed name. If an earlier stem held a state that + // failed validation, report that instead: the caller may prefer to + // force it rather than download the file again under a new name. + let inner = if let Some((error, cfg_path, tmp_path)) = mismatch { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + None, + ); + PlanKind::Mismatch(MismatchPlan::new(common, error, MismatchPolicy::Fresh)) + } else { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + None, + ); + PlanKind::Fresh(FreshPlan::new(common)) + }; + return Ok(DownloadPlan { inner }); + } + + if can_resume { + match super::super::try_load_resume_state( + &url, + &cfg_path, + &tmp_path, + &probe.info, + &partial_config, + ) + .await + { + Ok(Some(state)) => { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + Some(&state), + ); + return Ok(DownloadPlan { + inner: PlanKind::Resume(ResumePlan::new(common, &state, Fallback::Fresh)), + }); + } + Err(e) if mismatch.is_none() => mismatch = Some((e, cfg_path, tmp_path)), + Ok(None) | Err(_) => {} + } + } + } + unreachable!() +} diff --git a/crates/fast-down-api/src/core/download/plan/manifest.rs b/crates/fast-down-api/src/core/download/plan/manifest.rs new file mode 100644 index 0000000..f0a725d --- /dev/null +++ b/crates/fast-down-api/src/core/download/plan/manifest.rs @@ -0,0 +1,161 @@ +use super::super::open_create; +use super::{ + DownloadPlan, Fallback, PlanCommon, PlanError, PlanKind, ResumeOutcome, ResumePlan, + inherit_persisted_config, plan_common, probe, state_url, +}; +use crate::{DownloadState, Event, PartialConfig, StateError, TerminationReason, Tx, tx_err}; +use std::ffi::OsStr; +use std::io; +use std::path::Path; +use std::time::Duration; +use tokio_util::sync::CancellationToken; +use url::Url; + +pub(super) struct ManifestFreshPlan { + common: PlanCommon, + outcome: ResumeOutcome, +} + +impl ManifestFreshPlan { + pub(super) const fn new(common: PlanCommon) -> Self { + Self { + common, + outcome: ResumeOutcome::Fresh, + } + } + + pub(super) const fn common(&self) -> &PlanCommon { + &self.common + } + + pub(super) const fn outcome(&self) -> &ResumeOutcome { + &self.outcome + } + + pub(super) fn into_common(self) -> PlanCommon { + self.common + } + + pub(super) async fn run(self) -> TerminationReason { + let state = match DownloadState::load(&self.common.cfg_path).await { + Ok(state) => state, + Err(e) => { + let _ = self.common.tx.send(Event::ResumeError(e)); + return TerminationReason::Failed; + } + }; + let state = fresh_from_loaded( + state, + &self.common.partial_config, + &self.common.url, + &self.common.info, + ); + if !self.common.ensure_parent().await { + return TerminationReason::Failed; + } + tx_err!( + open_create().open(&self.common.tmp_path).await, + self.common.tx, + BuildPusherError, + TerminationReason::Failed + ); + self.common.run_existing(state, false).await + } +} + +fn fresh_from_loaded( + state: DownloadState, + partial_config: &crate::PartialConfig, + url: &url::Url, + info: &fast_down::UrlInfo, +) -> DownloadState { + state.update(|inner| { + if let Some(c) = &mut inner.config { + c.downloaded_chunk = None; + } + inner.elapsed = Some(Duration::ZERO); + }); + let mut fresh_config = partial_config.clone(); + fresh_config.downloaded_chunk = None; + state.merge_config(&fresh_config); + state.refresh_identity(url, info); + state +} + +/// Prepare a download driven by a `.fd` state file used as a download manifest. +/// +/// Unlike [`super::plan_resume`], which is given the `.part` and looks for its +/// companion `.fd`, this entry takes the `.fd` itself. The sibling `.part` +/// decides whether the run resumes recorded progress or reuses the manifest +/// configuration for a fresh download. +/// +/// `url` is optional: when omitted, the durable initial URL recorded in the +/// `.fd` is used. Resume requires a range-capable server; without one the +/// manifest is still reused for a fresh, single-stream download. +/// +/// # Errors +/// +/// Returns [`PlanError::Resume`] when `fd_path` is not a `.fd` file, the `.fd` +/// cannot be read or decoded, no URL can be resolved, or the prefetch fails. +#[allow(clippy::result_large_err)] +pub async fn plan_from_fd( + fd_path: impl AsRef, + url: Option, + mut partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, +) -> Result { + let fd_path = fd_path.as_ref(); + if fd_path.extension() != Some(OsStr::new("fd")) { + return Err(PlanError::Resume(StateError::Open(io::Error::new( + io::ErrorKind::InvalidInput, + "fd_path must end with .fd extension", + )))); + } + partial_config.overwrite = Some(false); + + let loaded = DownloadState::load(fd_path) + .await + .map_err(PlanError::Resume)?; + let Some(url) = state_url(url, &loaded) else { + return Err(PlanError::Resume(StateError::NoUrl(fd_path.to_path_buf()))); + }; + + inherit_persisted_config(&mut partial_config, &loaded); + partial_config.overwrite = Some(false); + partial_config.resume = Some(true); + let probe = probe(&url, &partial_config, &tx).await?; + + let tmp_path = fd_path.with_extension("part"); + let state = if probe.info.fast_download { + match super::super::try_load_resume_state( + &url, + fd_path, + &tmp_path, + &probe.info, + &partial_config, + ) + .await + { + Ok(Some(state)) => Some(state), + _ => None, + } + } else { + None + }; + let common = plan_common( + url, + partial_config, + probe, + fd_path.to_path_buf(), + tmp_path, + tx, + token, + state.as_ref(), + ); + let inner = match state { + Some(state) => PlanKind::Resume(ResumePlan::new(common, &state, Fallback::ManifestFresh)), + None => PlanKind::ManifestFresh(ManifestFreshPlan::new(common)), + }; + Ok(DownloadPlan { inner }) +} diff --git a/crates/fast-down-api/src/core/download/plan/mismatch.rs b/crates/fast-down-api/src/core/download/plan/mismatch.rs new file mode 100644 index 0000000..f98e647 --- /dev/null +++ b/crates/fast-down-api/src/core/download/plan/mismatch.rs @@ -0,0 +1,113 @@ +use super::{PlanCommon, ResumeOutcome}; +use crate::{Event, StateError, TerminationReason}; +use std::io; +use tokio::fs; + +#[derive(Clone, Copy)] +pub(super) enum MismatchPolicy { + Fresh, + Fail, +} + +pub(super) struct MismatchPlan { + common: PlanCommon, + outcome: ResumeOutcome, + policy: MismatchPolicy, +} + +impl MismatchPlan { + pub(super) const fn new(common: PlanCommon, error: StateError, policy: MismatchPolicy) -> Self { + Self { + common, + outcome: ResumeOutcome::Mismatch(error), + policy, + } + } + + pub(super) const fn common(&self) -> &PlanCommon { + &self.common + } + + pub(super) const fn outcome(&self) -> &ResumeOutcome { + &self.outcome + } + + pub(super) fn into_common(self) -> PlanCommon { + self.common + } + + pub(super) async fn run(self) -> TerminationReason { + match self.policy { + MismatchPolicy::Fresh => self.common.claim_and_run().await, + MismatchPolicy::Fail => { + let ResumeOutcome::Mismatch(error) = self.outcome else { + unreachable!("MismatchPlan must contain a mismatch outcome") + }; + let _ = self.common.tx.send(Event::ResumeError(error)); + TerminationReason::Failed + } + } + } + + pub(super) async fn run_forced(self) -> TerminationReason { + let ResumeOutcome::Mismatch(planned_error) = self.outcome else { + unreachable!("MismatchPlan must contain a mismatch outcome") + }; + if !matches!( + &planned_error, + StateError::FileChanged { + local_file_size, + remote_file_size, + .. + } if local_file_size == remote_file_size + ) { + let _ = self.common.tx.send(Event::ResumeError(planned_error)); + return TerminationReason::Failed; + } + + let state = match crate::DownloadState::load(&self.common.cfg_path).await { + Ok(state) => state, + Err(e) => { + let _ = self.common.tx.send(Event::ResumeError(e)); + return TerminationReason::Failed; + } + }; + match state.validate(&self.common.info) { + Ok(()) => {} + Err(StateError::FileChanged { + local_file_size, + remote_file_size, + .. + }) if local_file_size == remote_file_size => {} + Err(e) => { + let _ = self.common.tx.send(Event::ResumeError(e)); + return TerminationReason::Failed; + } + } + if !fs::try_exists(&self.common.tmp_path).await.unwrap_or(false) { + let _ = self + .common + .tx + .send(Event::ResumeError(StateError::Open(io::Error::new( + io::ErrorKind::NotFound, + "the .part file disappeared after the plan was created", + )))); + return TerminationReason::Failed; + } + state.merge_config(&self.common.partial_config); + if let Some((actual_size, recorded_size)) = + state.part_shortfall(&self.common.tmp_path).await + { + let _ = self + .common + .tx + .send(Event::ResumeError(StateError::Truncated { + actual_size, + recorded_size, + })); + return TerminationReason::Failed; + } + state.refresh_identity(&self.common.url, &self.common.info); + self.common.run_existing(state, true).await + } +} diff --git a/crates/fast-down-api/src/core/download/plan/mod.rs b/crates/fast-down-api/src/core/download/plan/mod.rs new file mode 100644 index 0000000..615813d --- /dev/null +++ b/crates/fast-down-api/src/core/download/plan/mod.rs @@ -0,0 +1,412 @@ +//! Two-phase download entry point: probe the remote and the disk first, commit +//! to writing later. +//! +//! [`plan`] and [`plan_resume`] perform every step that has no lasting effect — +//! prefetching the remote metadata, resolving the output path, and inspecting +//! any `.fd`/`.part` pair already on disk — and hand back a [`DownloadPlan`] +//! describing what *would* happen. Nothing is created, truncated or renamed +//! until one of the `start*` methods is called, so a caller can show the plan to +//! a user, ask a question, and then either commit or drop the plan without +//! leaving anything behind. +//! +//! [`crate::download`] and [`crate::resume`] are the one-shot wrappers over this +//! pair: they plan and immediately start, reporting a [`PlanError`] as the +//! matching `*Error` event. +mod common; +mod fresh; +mod manifest; +mod mismatch; +mod resume; + +use self::{ + common::{Fallback, PlanCommon}, + fresh::FreshPlan, + manifest::ManifestFreshPlan, + mismatch::{MismatchPlan, MismatchPolicy}, + resume::ResumePlan, +}; +pub use self::{fresh::plan, manifest::plan_from_fd, resume::peek_resume, resume::plan_resume}; +use super::try_load_resume_state; +use crate::{ + Config, DownloadState, Event, PartialConfig, StateError, TerminationReason, Tx, prefetch, + utils::gen_path, +}; +use fast_down::{ProgressEntry, Total, UrlInfo, reqwest::ReqwestResponseError}; +use inherit_config::ConfigLayer; +use reqwest::Response; +use std::path::{Path, PathBuf}; +use tokio_util::sync::CancellationToken; +use url::Url; + +/// Why a download plan could not be produced. +/// +/// Every variant is fatal for the plan: no [`DownloadPlan`] exists, and nothing +/// has been written to disk. [`PlanError::emit`] maps each one onto the +/// equivalent [`Event`] for callers that consume the event stream rather than +/// the return value. +#[derive(Debug, thiserror::Error)] +#[allow(clippy::large_enum_variant)] +pub enum PlanError { + /// The HTTP client could not be built (TLS / backend initialization, etc.). + #[error("failed to build the HTTP client: {0}")] + BuildClient(reqwest::Error), + /// Prefetch exhausted its retry budget; this is the failure of the last + /// attempt. Earlier attempts were reported as [`Event::PrefetchError`]. + #[error("failed to fetch the remote metadata: {0}")] + Prefetch(ReqwestResponseError), + /// The output path could not be computed (unwritable directory, a file name + /// that is invalid on this platform, etc.). + #[error("failed to compute the output path: {0}")] + GenPath(std::io::Error), + /// An explicit [`plan_resume`] could not continue the interrupted download. + #[error(transparent)] + Resume(StateError), +} + +impl PlanError { + /// Forward this error to the event channel as the matching `*Error` event. + /// + /// Used by the one-shot [`crate::download`] / [`crate::resume`] wrappers, + /// whose callers only observe the event stream. + pub fn emit(self, tx: &Tx) { + let _ = match self { + Self::BuildClient(e) => tx.send(Event::BuildClientError(e)), + Self::Prefetch(e) => tx.send(Event::PrefetchError(e)), + Self::GenPath(e) => tx.send(Event::GenPathError(e)), + Self::Resume(e) => tx.send(Event::ResumeError(e)), + }; + } +} + +/// What the disk probe found for the download a [`DownloadPlan`] describes. +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum ResumeOutcome { + /// Nothing to continue from: either no state was found, or resuming is + /// disabled / unsupported by the server. Starting the plan downloads the + /// whole file. + Fresh, + /// A `.fd`/`.part` pair was found and validates against the current remote + /// file. Starting the plan continues from `progress`. + Resumable { + /// Path of the `.fd` state file backing the resume. + config_path: PathBuf, + /// Byte ranges already written to the `.part` file. + progress: Vec, + /// Total bytes already on disk (the sum of the `progress` lengths). + downloaded: u64, + }, + /// A `.fd`/`.part` pair was found but cannot be continued — most often + /// because the remote file changed since it was written. + /// + /// What [`DownloadPlan::start`] does depends on which entry point produced + /// the plan. From [`plan`] it falls back to a fresh download (overwriting + /// the stale `.part`, or claiming a new name when `overwrite` is disabled). + /// From [`plan_resume`] the caller asked for a resume specifically, so it + /// refuses instead: [`Event::ResumeError`] and + /// [`TerminationReason::Failed`], leaving the `.part` and `.fd` untouched. + /// + /// Either way [`DownloadPlan::start_forced_resume`] continues from the + /// stale state, but only when the mismatch is limited to the identity + /// headers. + Mismatch(StateError), +} + +enum PlanKind { + Fresh(FreshPlan), + Resume(ResumePlan), + ManifestFresh(ManifestFreshPlan), + Mismatch(MismatchPlan), +} + +impl PlanKind { + const fn common(&self) -> &PlanCommon { + match self { + Self::Fresh(plan) => plan.common(), + Self::Resume(plan) => plan.common(), + Self::ManifestFresh(plan) => plan.common(), + Self::Mismatch(plan) => plan.common(), + } + } + + const fn outcome(&self) -> &ResumeOutcome { + match self { + Self::Fresh(plan) => plan.outcome(), + Self::Resume(plan) => plan.outcome(), + Self::ManifestFresh(plan) => plan.outcome(), + Self::Mismatch(plan) => plan.outcome(), + } + } + + fn into_common(self) -> PlanCommon { + match self { + Self::Fresh(plan) => plan.into_common(), + Self::Resume(plan) => plan.into_common(), + Self::ManifestFresh(plan) => plan.into_common(), + Self::Mismatch(plan) => plan.into_common(), + } + } + + async fn run(self) -> TerminationReason { + match self { + Self::Fresh(plan) => Box::pin(plan.run()).await, + Self::Resume(plan) => Box::pin(plan.run()).await, + Self::ManifestFresh(plan) => Box::pin(plan.run()).await, + Self::Mismatch(plan) => Box::pin(plan.run()).await, + } + } + + async fn run_forced(self) -> TerminationReason { + match self { + Self::Mismatch(plan) => Box::pin(plan.run_forced()).await, + other => Box::pin(other.run()).await, + } + } + + /// Discard the variant and download the whole file from byte zero, + /// ignoring any resumable state on disk. + async fn run_fresh(self) -> TerminationReason { + self.into_common().claim_and_run().await + } +} + +/// A prepared, not-yet-started download. +/// +/// Produced by [`plan`] / [`plan_resume`] after the remote has been prefetched +/// and the disk inspected. Holding one keeps the prefetch [`Response`] open so +/// the first range request can reuse it; dropping one abandons the download +/// without having touched the filesystem. +#[must_use = "a DownloadPlan does nothing until one of the start methods is called; drop it to abandon the download"] +pub struct DownloadPlan { + inner: PlanKind, +} + +impl DownloadPlan { + /// Metadata resolved for the remote file: size, identity headers, and + /// whether the server supports ranged (and therefore parallel) requests. + #[must_use] + pub const fn info(&self) -> &UrlInfo { + &self.inner.common().info + } + + /// The fully-resolved configuration this plan will run with. + #[must_use] + pub const fn config(&self) -> &Config { + &self.inner.common().config + } + + /// Where the finished file is intended to land. + /// + /// When `overwrite` is disabled the actual destination is uniquified at the + /// very end, so the file may land on a `name (1).ext` variant instead; + /// [`Event::Renamed`] carries the path it really landed on. + #[must_use] + pub fn final_path(&self) -> &Path { + &self.inner.common().final_path + } + + /// The `.part` file this plan inspected. + /// + /// For [`ResumeOutcome::Resumable`] and [`ResumeOutcome::Mismatch`] this is + /// the existing file that was examined. For a normal [`ResumeOutcome::Fresh`] + /// it is the name that was free at probe time — because planning claims + /// nothing, a concurrent download can take it first, in which case starting + /// the plan moves on to the next free name. A manifest-driven Fresh plan + /// instead reports the `.part` paired with that manifest. + #[must_use] + pub fn tmp_path(&self) -> &Path { + &self.inner.common().tmp_path + } + + /// The `.fd` state file paired with [`DownloadPlan::tmp_path`]. + #[must_use] + pub fn config_path(&self) -> &Path { + &self.inner.common().cfg_path + } + + /// What the disk probe found, and therefore what starting the plan will do. + #[must_use] + pub const fn resume_outcome(&self) -> &ResumeOutcome { + self.inner.outcome() + } + + /// Run the plan as probed. + /// + /// [`ResumeOutcome::Resumable`] continues from the saved progress and + /// [`ResumeOutcome::Fresh`] downloads the whole file. A + /// [`ResumeOutcome::Mismatch`] falls back to a fresh download, except for a + /// plan made by [`plan_resume`], where it is reported as + /// [`Event::ResumeError`] and ends the run — that caller asked to continue + /// one specific file, not to fetch it again. + /// + /// Emits exactly one [`Event::Terminated`] as the last event on the channel. + pub async fn start(self) { + let tx = self.inner.common().tx.clone(); + let reason = self.inner.run().await; + let _ = tx.send(Event::Terminated(reason)); + } + + /// Ignore any resumable state and download the whole file again. + /// + /// With `overwrite` enabled the existing `.part` is reused as scratch space + /// and every byte is rewritten; otherwise a new, unclaimed `.part` name is + /// taken so the existing one is left untouched. + /// + /// Emits exactly one [`Event::Terminated`] as the last event on the channel. + pub async fn start_fresh(self) { + let tx = self.inner.common().tx.clone(); + let reason = self.inner.run_fresh().await; + let _ = tx.send(Event::Terminated(reason)); + } + + /// Continue from a [`ResumeOutcome::Mismatch`] state despite the failed + /// validation. + /// + /// This is only safe when the byte layout is unchanged, so the forced + /// resume is refused unless the mismatch is limited to the identity headers + /// (etag / last-modified) and both sides agree on the file size. A size + /// change means the recorded ranges describe different bytes, and reusing + /// them would splice two versions of the file together. + /// + /// The `.part` is also measured against the recorded progress: a file + /// shorter than the highest recorded offset is rejected with + /// [`StateError::Truncated`], because the sink would extend it with zeros + /// over the missing span and never fetch those bytes. + /// + /// A refusal is reported as [`Event::ResumeError`] followed by + /// [`TerminationReason::Failed`]. Called on a plan that is not a mismatch, + /// this behaves exactly like [`DownloadPlan::start`]. + /// + /// Emits exactly one [`Event::Terminated`] as the last event on the channel. + pub async fn start_forced_resume(self) { + let tx = self.inner.common().tx.clone(); + let reason = self.inner.run_forced().await; + let _ = tx.send(Event::Terminated(reason)); + } +} + +/// Turn a validated state into the public [`ResumeOutcome::Resumable`] view. +fn resumable_outcome(state: &DownloadState, config_path: &Path) -> ResumeOutcome { + let progress = state.get_progress(); + let downloaded = progress.total(); + ResumeOutcome::Resumable { + config_path: config_path.to_path_buf(), + progress, + downloaded, + } +} + +/// The common, side-effect-free front half of every plan: resolve the config, +/// prefetch the remote metadata, and compute the output path. +struct Probe { + config: Config, + info: UrlInfo, + resp: Response, + final_path: PathBuf, +} + +#[allow(clippy::result_large_err)] +async fn probe(url: &Url, partial_config: &PartialConfig, tx: &Tx) -> Result { + let config = partial_config.clone().build(); + let (info, resp) = prefetch(url, &config, tx).await?; + let final_path = gen_path(url, &info, &config) + .await + .map_err(PlanError::GenPath)?; + Ok(Probe { + config, + info, + resp, + final_path, + }) +} + +/// Probe the `.fd`/`.part` pair at `cfg_path`/`tmp_path` and map the outcome onto +/// the public [`ResumeOutcome`] view. +async fn probe_resume( + url: &Url, + cfg_path: &Path, + tmp_path: &Path, + info: &UrlInfo, + partial_config: &PartialConfig, +) -> (Option, ResumeOutcome) { + match try_load_resume_state(url, cfg_path, tmp_path, info, partial_config).await { + Ok(Some(state)) => { + let outcome = resumable_outcome(&state, cfg_path); + (Some(state), outcome) + } + Ok(None) => (None, ResumeOutcome::Fresh), + Err(e) => (None, ResumeOutcome::Mismatch(e)), + } +} + +/// The caller's URL when given, otherwise the durable initial URL recorded in +/// the `.fd` state. +fn state_url(url: Option, loaded: &DownloadState) -> Option { + url.or_else(|| { + let guard = loaded.lock_inner(); + if let Some(url) = &guard.url + && matches!(url.scheme(), "http" | "https") + { + Some(url.clone()) + } else { + None + } + }) +} + +/// Layer caller overrides over the persisted configuration before prefetching +/// or computing paths. Persisted byte progress is intentionally excluded here: +/// it is restored only after the current `.fd`/`.part` pair validates. +fn inherit_persisted_config(config: &mut PartialConfig, loaded: &DownloadState) { + let requested_progress = config.downloaded_chunk.take(); + let saved = loaded.lock_inner().config.clone(); + if let Some(saved) = saved { + config.inherit_from(&saved); + } + config.downloaded_chunk = requested_progress; +} + +fn resolved_plan_config(probe_config: Config, state: Option<&DownloadState>) -> Config { + let Some(state) = state else { + return probe_config; + }; + state + .lock_inner() + .config + .clone() + .map_or(probe_config, ConfigLayer::build) +} + +#[allow(clippy::too_many_arguments)] +fn plan_common( + url: Url, + partial_config: PartialConfig, + probe: Probe, + cfg_path: PathBuf, + tmp_path: PathBuf, + tx: Tx, + token: CancellationToken, + state: Option<&DownloadState>, +) -> PlanCommon { + let Probe { + config: probe_config, + info, + resp, + final_path, + } = probe; + PlanCommon { + url, + partial_config, + config: resolved_plan_config(probe_config, state), + info, + resp, + final_path, + cfg_path, + tmp_path, + tx, + token, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/fast-down-api/src/core/download/plan/resume.rs b/crates/fast-down-api/src/core/download/plan/resume.rs new file mode 100644 index 0000000..3a9d5b9 --- /dev/null +++ b/crates/fast-down-api/src/core/download/plan/resume.rs @@ -0,0 +1,203 @@ +use super::{ + DownloadPlan, Fallback, FreshPlan, ManifestFreshPlan, MismatchPlan, MismatchPolicy, PlanCommon, + PlanError, PlanKind, ResumeOutcome, inherit_persisted_config, plan_common, probe, probe_resume, + resumable_outcome, state_url, +}; +use crate::{DownloadState, Event, PartialConfig, StateError, TerminationReason, Tx}; +use std::path::Path; +use tokio::fs; +use tokio_util::sync::CancellationToken; +use url::Url; + +pub(super) struct ResumePlan { + common: PlanCommon, + outcome: ResumeOutcome, + fallback: Fallback, +} + +impl ResumePlan { + pub(super) fn new( + common: PlanCommon, + state: &crate::DownloadState, + fallback: Fallback, + ) -> Self { + let outcome = resumable_outcome(state, &common.cfg_path); + Self { + common, + outcome, + fallback, + } + } + + pub(super) const fn common(&self) -> &PlanCommon { + &self.common + } + + pub(super) const fn outcome(&self) -> &ResumeOutcome { + &self.outcome + } + + pub(super) fn into_common(self) -> PlanCommon { + self.common + } + + pub(super) async fn run(self) -> TerminationReason { + match super::super::try_load_resume_state( + &self.common.url, + &self.common.cfg_path, + &self.common.tmp_path, + &self.common.info, + &self.common.partial_config, + ) + .await + { + Ok(Some(state)) => self.common.run_existing(state, true).await, + Ok(None) => Box::pin(self.run_fallback(None)).await, + Err(error) => Box::pin(self.run_fallback(Some(error))).await, + } + } + + async fn run_fallback(self, error: Option) -> TerminationReason { + match self.fallback { + Fallback::Fresh => self.common.claim_and_run().await, + Fallback::ManifestFresh => ManifestFreshPlan::new(self.common).run().await, + Fallback::Fail => match error { + Some(error) => { + let _ = self.common.tx.send(Event::ResumeError(error)); + TerminationReason::Failed + } + // `Ok(None)` means the `.part` disappeared or became too short; + // explicit resume preserves the established fresh fallback. + None => self.common.claim_and_run().await, + }, + } + } +} + +/// Prepare a resume of an interrupted download from its `.part` file. +/// +/// `url` is optional: when omitted the durable initial URL recorded in the `.fd` +/// state is used, so a caller can resume from the `.part` path alone. Redirects +/// are always re-resolved through a fresh prefetch. +/// +/// Unlike [`super::plan`], a state that exists but no longer describes the +/// remote file is not silently replaced by a full download. It comes back as +/// [`ResumeOutcome::Mismatch`], and starting such a plan reports +/// [`Event::ResumeError`] and stops — the caller asked to continue one specific +/// file, so it decides whether to force the resume, download afresh, or give up. +/// +/// A `tmp_path` that does not exist at all is the one case that does defer to +/// [`super::plan`]: there is no partial file to continue, so with a `url` +/// available this is just a download. +/// +/// # Errors +/// +/// Returns [`PlanError::Resume`] when `tmp_path` is not a `.part` file, no URL +/// can be resolved, the `.fd` state cannot be read, or the server does not +/// support ranged requests. The [`super::plan`] errors apply as well. +#[allow(clippy::result_large_err)] +pub async fn plan_resume( + tmp_path: impl AsRef, + url: Option, + mut partial_config: PartialConfig, + tx: Tx, + token: CancellationToken, +) -> Result { + let tmp_path = tmp_path.as_ref(); + if tmp_path.extension() != Some(std::ffi::OsStr::new("part")) { + return Err(PlanError::Resume(StateError::Open(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "tmp_path must end with .part extension", + )))); + } + partial_config.overwrite = Some(false); + + if !fs::try_exists(tmp_path).await.unwrap_or(false) { + let Some(url) = url else { + return Err(PlanError::Resume(StateError::NoUrl(tmp_path.to_path_buf()))); + }; + partial_config.resume = Some(false); + return super::fresh::plan(url, partial_config, tx, token).await; + } + + let cfg_path = tmp_path.with_extension("fd"); + let loaded = DownloadState::load(&cfg_path) + .await + .map_err(PlanError::Resume)?; + let Some(url) = state_url(url, &loaded) else { + return Err(PlanError::Resume(StateError::NoUrl(tmp_path.to_path_buf()))); + }; + inherit_persisted_config(&mut partial_config, &loaded); + partial_config.overwrite = Some(false); + partial_config.resume = Some(true); + drop(loaded); + + let probe = probe(&url, &partial_config, &tx).await?; + if !probe.info.fast_download { + return Err(PlanError::Resume(StateError::NotResumable( + probe.info, probe.resp, + ))); + } + + let (state, outcome) = + probe_resume(&url, &cfg_path, tmp_path, &probe.info, &partial_config).await; + let tmp_path = tmp_path.to_path_buf(); + let inner = match (state, outcome) { + (Some(state), _) => { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + Some(&state), + ); + PlanKind::Resume(ResumePlan::new(common, &state, Fallback::Fail)) + } + (None, ResumeOutcome::Mismatch(error)) => { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + None, + ); + PlanKind::Mismatch(MismatchPlan::new(common, error, MismatchPolicy::Fail)) + } + (None, _) => { + let common = plan_common( + url, + partial_config, + probe, + cfg_path, + tmp_path, + tx, + token, + None, + ); + PlanKind::Fresh(FreshPlan::new(common)) + } + }; + + Ok(DownloadPlan { inner }) +} + +/// Read a previously-saved download state from disk without contacting the +/// network. The returned state is not validated against the current remote. +/// +/// # Errors +/// Returns [`StateError::NotAPartFile`] when `tmp_path` is not a `.part` file, +/// or [`StateError::Open`] when its companion `.fd` cannot be read or decoded. +#[allow(clippy::result_large_err)] +pub async fn peek_resume(tmp_path: impl AsRef) -> Result { + let tmp_path = tmp_path.as_ref(); + if tmp_path.extension() != Some(std::ffi::OsStr::new("part")) { + return Err(StateError::NotAPartFile(tmp_path.to_path_buf())); + } + DownloadState::load(&tmp_path.with_extension("fd")).await +} diff --git a/crates/fast-down-api/src/core/download/plan/tests.rs b/crates/fast-down-api/src/core/download/plan/tests.rs new file mode 100644 index 0000000..4ec888b --- /dev/null +++ b/crates/fast-down-api/src/core/download/plan/tests.rs @@ -0,0 +1,831 @@ +#![allow(clippy::unwrap_used)] +use super::*; +use fast_down::{FileId, ProgressEntry, UrlInfo}; +use std::time::Duration; +use url::Url; + +// Test-only helpers for the planning entry points. The `plan` / `plan_resume` +// / `plan_from_fd` paths all call `prefetch`, so the server-backed tests spin +// up a minimal local HTTP server that answers a normal GET with 200 + +// content-length and a Range probe with 206 + content-range (no identity +// headers, so `info.file_id` is `{None, None}` and a hand-written `.fd` +// validates against it as long as the size matches). +use bytes::Bytes; +use http_body_util::Full; +use hyper::body::Incoming; +use hyper::header::{ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, RANGE}; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use std::convert::Infallible; + +fn make_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("plan_test_{name}_{}", std::process::id())); + let _ = std::fs::create_dir_all(&dir); + dir +} + +fn info_no_id(size: u64) -> UrlInfo { + UrlInfo { + size, + raw_name: "a.bin".to_string(), + supports_range: true, + fast_download: true, + final_url: Url::parse("https://example.com/a.bin").unwrap(), + file_id: FileId::new(None, None), + content_type: Some("application/octet-stream".to_string()), + } +} + +#[allow(clippy::too_many_lines)] +async fn spawn_server(body: &[u8]) -> Url { + let body = body.to_vec(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + break; + }; + let body = body.clone(); + let io = TokioIo::new(stream); + tokio::spawn(async move { + let service = service_fn(move |req: Request| { + let body = body.clone(); + async move { + let total = body.len(); + let is_range = req.headers().contains_key(RANGE); + let (status, content_length, body_bytes, content_range) = if is_range { + ( + StatusCode::PARTIAL_CONTENT, + 1usize, + Bytes::from(vec![0xABu8]), + Some(format!("bytes 0-0/{total}")), + ) + } else { + (StatusCode::OK, total, Bytes::from(body), None) + }; + let mut builder = Response::builder() + .status(status) + .header(CONTENT_LENGTH, content_length.to_string()) + .header(ACCEPT_RANGES, "bytes"); + if let Some(cr) = content_range { + builder = builder.header(CONTENT_RANGE, cr); + } + Ok::<_, Infallible>(builder.body(Full::new(body_bytes)).unwrap()) + } + }); + let _ = http1::Builder::new().serve_connection(io, service).await; + }); + } + }); + Url::parse(&format!("http://{addr}/file.bin")).unwrap() +} + +async fn prefetch_info(url: &Url, tx: &Tx) -> UrlInfo { + let cfg = Config::default(); + let (info, _resp) = prefetch(url, &cfg, tx).await.unwrap(); + info +} + +// Write a `.fd` whose recorded size is `size` and whose identity fields are +// left at their serde defaults (matching a server that sends no +// etag/last-modified, so `info.file_id` is `{None, None}`). TOML has no +// `null` literal, so the identity fields are simply omitted. +async fn write_fd_raw(cfg_path: &Path, url: &Url, size: u64) { + let toml = format!("url = \"{url}\"\nsize = {size}\n"); + tokio::fs::write(cfg_path, toml).await.unwrap(); +} + +// Build a `.fd` that validates against `info` (size + null identity) and, +// when `progress` is given, records that progress so a resume reports it. +async fn write_fd_matching( + url: &Url, + info: &UrlInfo, + cfg_path: &Path, + progress: Vec, +) { + let state = DownloadState::new(url, info, &PartialConfig::default(), cfg_path); + for r in progress { + state.merge_progress(r); + } + state.store().await.unwrap(); +} + +fn base_pc(save_dir: &Path, overwrite: bool) -> PartialConfig { + PartialConfig { + save_dir: Some(save_dir.to_path_buf()), + filename: Some("a.bin".to_string()), + overwrite: Some(overwrite), + resume: Some(true), + ..Default::default() + } +} + +fn make_info() -> UrlInfo { + UrlInfo { + size: 100, + raw_name: "a.bin".to_string(), + supports_range: true, + fast_download: true, + final_url: Url::parse("https://example.com/a.bin").unwrap(), + file_id: FileId::new(Some("etag-1"), None), + content_type: Some("application/octet-stream".to_string()), + } +} + +#[tokio::test] +async fn peek_resume_reads_saved_state_without_network() { + let dir = std::env::temp_dir().join(format!( + "fd_peek_{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let _ = std::fs::create_dir_all(&dir); + let part = dir.join("a.part"); + let fd = dir.join("a.fd"); + std::fs::write(&part, vec![0u8; 100]).unwrap(); + + let url = Url::parse("https://example.com/a.bin").unwrap(); + let info = make_info(); + let state = DownloadState::new(&url, &info, &PartialConfig::default(), &fd); + state.merge_progress(0u64..50); + state.store().await.unwrap(); + + let loaded = peek_resume(&part).await.unwrap(); + assert_eq!(loaded.get_progress(), vec![0u64..50]); + assert_eq!(loaded.file_id(), FileId::new(Some("etag-1"), None)); + assert_eq!( + loaded.lock_inner().url.as_ref().unwrap().as_str(), + "https://example.com/a.bin" + ); + assert_eq!(loaded.get_elapsed(), Duration::ZERO); + + let _ = std::fs::remove_dir_all(&dir); +} + +#[tokio::test] +async fn peek_resume_rejects_non_part_path() { + let dir = std::env::temp_dir().join(format!( + "fd_peek_bad_{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let _ = std::fs::create_dir_all(&dir); + let not_part = dir.join("a.txt"); + std::fs::write(¬_part, b"x").unwrap(); + assert!(matches!( + peek_resume(¬_part).await, + Err(StateError::NotAPartFile(_)) + )); + let _ = std::fs::remove_dir_all(&dir); +} + +// ---- Pure (no network) tests ---- + +#[tokio::test] +async fn resumable_outcome_sums_progress_lengths() { + let dir = make_dir("res_outcome"); + let fd = dir.join("a.fd"); + let url = Url::parse("https://example.com/a.bin").unwrap(); + let info = info_no_id(1024); + let state = DownloadState::new(&url, &info, &PartialConfig::default(), &fd); + // Disjoint ranges so normalization keeps them separate. + state.merge_progress(0u64..10); + state.merge_progress(20u64..30); + state.merge_progress(50u64..70); + match resumable_outcome(&state, &fd) { + ResumeOutcome::Resumable { + downloaded, + progress, + .. + } => { + assert_eq!(downloaded, 40); + assert_eq!(progress, vec![0u64..10, 20u64..30, 50u64..70]); + } + other => panic!("expected Resumable, got {other:?}"), + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn state_url_prefers_caller_url_and_filters_non_http() { + let url = Url::parse("https://example.com/a.bin").unwrap(); + let info = info_no_id(1024); + let fd = PathBuf::from("dummy.fd"); + let loaded = DownloadState::new(&url, &info, &PartialConfig::default(), &fd); + // Caller URL always wins. + assert_eq!(state_url(Some(url.clone()), &loaded), Some(url.clone())); + // No caller URL and a loaded http(s) URL -> resolved from the `.fd`. + assert_eq!(state_url(None, &loaded), Some(url)); + // Loaded non-http(s) URL is filtered out -> None. + loaded.update(|inner| inner.url = Some(Url::parse("ftp://host/x").unwrap())); + assert_eq!(state_url(None, &loaded), None); + // The default `about:blank` URL is also filtered out -> None. + loaded.update(|inner| inner.url = Some(Url::parse("about:blank").unwrap())); + assert_eq!(state_url(None, &loaded), None); +} + +#[tokio::test] +async fn probe_resume_maps_outcome_without_network() { + let dir = make_dir("probe_resume"); + let url = Url::parse("https://example.com/a.bin").unwrap(); + let info = info_no_id(1024); + + // Ok(Some): a valid `.fd` + `.part` pair validates and loads. + let cfg = dir.join("a.fd"); + let tmp = dir.join("a.part"); + write_fd_raw(&cfg, &url, 1024).await; + std::fs::write(&tmp, vec![0u8; 100]).unwrap(); + let (state, outcome) = probe_resume(&url, &cfg, &tmp, &info, &PartialConfig::default()).await; + assert!(state.is_some()); + assert!(matches!(outcome, ResumeOutcome::Resumable { .. })); + + // Ok(None): `.fd` present but `.part` missing -> nothing usable. + let cfg2 = dir.join("b.fd"); + let tmp2 = dir.join("b.part"); + write_fd_raw(&cfg2, &url, 1024).await; + let (state, outcome) = probe_resume(&url, &cfg2, &tmp2, &info, &PartialConfig::default()).await; + assert!(state.is_none()); + assert!(matches!(outcome, ResumeOutcome::Fresh)); + + // Err(FileChanged): size recorded in the `.fd` no longer matches. + let cfg3 = dir.join("c.fd"); + let tmp3 = dir.join("c.part"); + write_fd_raw(&cfg3, &url, 2048).await; + std::fs::write(&tmp3, vec![0u8; 100]).unwrap(); + let (state, outcome) = probe_resume(&url, &cfg3, &tmp3, &info, &PartialConfig::default()).await; + assert!(state.is_none()); + assert!(matches!( + outcome, + ResumeOutcome::Mismatch(StateError::FileChanged { .. }) + )); + + let _ = std::fs::remove_dir_all(&dir); +} + +// A `.fd` that exists and is readable but is not valid TOML must surface as +// `Mismatch(StateError::Decode)`, not silently fall back to a fresh download. +// This is the load-stage error that `ResumeOutcome::Mismatch` can carry in +// addition to `FileChanged`; every existing mismatch test only exercises the +// `FileChanged` branch, so this path was previously uncovered. +#[tokio::test] +async fn probe_resume_maps_corrupt_fd_to_mismatch_decode() { + let dir = make_dir("probe_resume_corrupt"); + let url = Url::parse("https://example.com/a.bin").unwrap(); + let info = info_no_id(1024); + let cfg = dir.join("d.fd"); + let tmp = dir.join("d.part"); + std::fs::write(&cfg, b"not valid toml @@").unwrap(); + std::fs::write(&tmp, vec![0u8; 100]).unwrap(); + let (state, outcome) = probe_resume(&url, &cfg, &tmp, &info, &PartialConfig::default()).await; + assert!(state.is_none()); + assert!(matches!( + outcome, + ResumeOutcome::Mismatch(StateError::Decode(..)) + )); + let _ = std::fs::remove_dir_all(&dir); +} + +// ---- Server-backed tests for the planning entry points ---- + +#[tokio::test] +async fn plan_overwrite_true_no_pair_is_fresh() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_ow_fresh"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let plan = plan(url, base_pc(&save_dir, true), tx, token) + .await + .unwrap(); + assert!(matches!(plan.resume_outcome(), ResumeOutcome::Fresh)); + assert!(plan.config().overwrite); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_overwrite_true_resumable_pair_is_resumable() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_ow_resume"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let tmp = save_dir.join("a.bin.part"); + let cfg = save_dir.join("a.bin.fd"); + write_fd_matching(&url, &info, &cfg, vec![]).await; + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + let plan = plan(url, base_pc(&save_dir, true), tx, token) + .await + .unwrap(); + assert!(matches!( + plan.resume_outcome(), + ResumeOutcome::Resumable { .. } + )); + assert!(plan.tmp_path().ends_with("a.bin.part")); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_overwrite_true_filechanged_is_mismatch() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_ow_mismatch"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let tmp = save_dir.join("a.bin.part"); + let cfg = save_dir.join("a.bin.fd"); + write_fd_raw(&cfg, &url, 2048).await; // size mismatch vs 1024 + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + let plan = plan(url, base_pc(&save_dir, true), tx, token) + .await + .unwrap(); + assert!(matches!( + plan.resume_outcome(), + ResumeOutcome::Mismatch(StateError::FileChanged { .. }) + )); + let _ = std::fs::remove_dir_all(&save_dir); +} + +// A corrupt `.fd` produces `Mismatch(StateError::Decode)`. Forcing the resume +// re-reads the same broken file and can only report the decode error and fail; +// `start_forced_resume` does not help for non-`FileChanged` mismatches, so the +// README hint to "use start_forced_resume" is a no-op for this case. +#[tokio::test] +async fn plan_overwrite_true_corrupt_fd_forced_resume_fails() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_ow_corrupt_fd"); + let (tx, rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let tmp = save_dir.join("a.bin.part"); + let cfg = save_dir.join("a.bin.fd"); + std::fs::write(&cfg, b"not valid toml @@").unwrap(); + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + let plan = plan(url, base_pc(&save_dir, true), tx, token) + .await + .unwrap(); + assert!(matches!( + plan.resume_outcome(), + ResumeOutcome::Mismatch(StateError::Decode(..)) + )); + + Box::pin(plan.start_forced_resume()).await; + let mut reported_decode = false; + let mut terminated = false; + while let Ok(event) = rx.recv().await { + reported_decode |= matches!(&event, Event::ResumeError(StateError::Decode(..))); + if matches!(&event, Event::Terminated(_)) { + terminated = true; + break; + } + } + assert!( + reported_decode, + "forced resume must surface the Decode error" + ); + assert!(terminated, "forced resume must terminate, not hang"); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_no_overwrite_resumable_stem0() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_nm_resume"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let tmp = save_dir.join("a.bin.part"); + let cfg = save_dir.join("a.bin.fd"); + write_fd_matching(&url, &info, &cfg, vec![]).await; + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + let plan = plan(url, base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(matches!( + plan.resume_outcome(), + ResumeOutcome::Resumable { .. } + )); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_no_overwrite_free_stem_is_fresh() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_nm_fresh"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let plan = plan(url, base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(matches!(plan.resume_outcome(), ResumeOutcome::Fresh)); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_no_overwrite_preserves_an_fd_only_manifest() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_fd_only_reserved"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let fd = save_dir.join("a.bin.fd"); + write_fd_matching(&url, &info, &fd, vec![]).await; + + let plan = plan(url, base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(plan.tmp_path().ends_with("a (1).bin.part")); + assert!(fd.exists()); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_no_overwrite_resume_disabled_ignores_pair() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_nm_noresume"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let tmp = save_dir.join("a.bin.part"); + let cfg = save_dir.join("a.bin.fd"); + write_fd_matching(&url, &info, &cfg, vec![]).await; + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + // `resume = false` must suppress the resume attempt even with a valid pair. + let pc = PartialConfig { + save_dir: Some(save_dir.clone()), + filename: Some("a.bin".to_string()), + overwrite: Some(false), + resume: Some(false), + ..Default::default() + }; + let plan = plan(url, pc, tx, token).await.unwrap(); + assert!(matches!(plan.resume_outcome(), ResumeOutcome::Fresh)); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_resume_rejects_non_part_path() { + let save_dir = make_dir("plan_res_badpath"); + let bad = save_dir.join("a.txt"); + std::fs::write(&bad, b"x").unwrap(); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let res = plan_resume( + &bad, + Some(Url::parse("https://example.com/a.bin").unwrap()), + PartialConfig::default(), + tx, + token, + ) + .await; + assert!(matches!( + res, + Err(PlanError::Resume(StateError::Open(e))) if e.kind() == std::io::ErrorKind::InvalidInput + )); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_resume_missing_tmp_with_url_defers_to_fresh() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_res_missing"); + let part = save_dir.join("a.bin.part"); // does NOT exist + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let plan = plan_resume(&part, Some(url), base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(matches!(plan.resume_outcome(), ResumeOutcome::Fresh)); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_resume_missing_tmp_without_url_is_nourl() { + let save_dir = make_dir("plan_res_missing_nourl"); + let part = save_dir.join("a.bin.part"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let res = plan_resume(&part, None, PartialConfig::default(), tx, token).await; + assert!(matches!(res, Err(PlanError::Resume(StateError::NoUrl(_))))); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_resume_valid_pair_is_resumable() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_res_valid"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let tmp = save_dir.join("a.bin.part"); + let cfg = save_dir.join("a.bin.fd"); + write_fd_matching(&url, &info, &cfg, vec![]).await; + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + let plan = plan_resume(&tmp, Some(url), base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(matches!( + plan.resume_outcome(), + ResumeOutcome::Resumable { .. } + )); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_resume_filechanged_pair_is_mismatch() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("plan_res_fc"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let tmp = save_dir.join("a.bin.part"); + let cfg = save_dir.join("a.bin.fd"); + write_fd_raw(&cfg, &url, 2048).await; + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + let plan = plan_resume(&tmp, Some(url), base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(matches!( + plan.resume_outcome(), + ResumeOutcome::Mismatch(StateError::FileChanged { .. }) + )); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_from_fd_rejects_non_fd_path() { + let save_dir = make_dir("fromfd_bad"); + let bad = save_dir.join("a.part"); + std::fs::write(&bad, b"x").unwrap(); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let res = plan_from_fd(&bad, None, PartialConfig::default(), tx, token).await; + assert!(matches!( + res, + Err(PlanError::Resume(StateError::Open(e))) if e.kind() == std::io::ErrorKind::InvalidInput + )); + let _ = std::fs::remove_dir_all(&save_dir); +} + +// A manifest with no usable `.part` still contributes its URL/config, but +// the bytes start at zero and must therefore be reported as Fresh. +#[tokio::test] +async fn plan_from_fd_fd_only_is_fresh() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("fromfd_only"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let fd = save_dir.join("a.bin.fd"); + // No `.part` present at all. + write_fd_matching(&url, &info, &fd, vec![]).await; + let plan = plan_from_fd(&fd, Some(url), base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(matches!(plan.resume_outcome(), ResumeOutcome::Fresh)); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +#[allow(clippy::single_range_in_vec_init)] +async fn plan_from_fd_with_part_resumes_recorded_progress() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("fromfd_part"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let fd = save_dir.join("a.bin.fd"); + let tmp = save_dir.join("a.bin.part"); + write_fd_matching(&url, &info, &fd, vec![0u64..512]).await; + std::fs::write(&tmp, vec![0u8; 600]).unwrap(); // >= recorded frontier 512 + let plan = plan_from_fd(&fd, Some(url), base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + match plan.resume_outcome() { + ResumeOutcome::Resumable { downloaded, .. } => assert_eq!(*downloaded, 512), + other => panic!("expected Resumable, got {other:?}"), + } + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_from_fd_filechanged_part_falls_back_to_fresh_manifest() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("fromfd_fc"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let tmp = save_dir.join("a.bin.part"); + let fd = save_dir.join("a.bin.fd"); + write_fd_raw(&fd, &url, 2048).await; // size mismatch vs 1024 + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + let plan = plan_from_fd(&fd, Some(url), base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(matches!(plan.resume_outcome(), ResumeOutcome::Fresh)); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_from_fd_resolves_url_from_fd() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("fromfd_resolve"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let fd = save_dir.join("a.bin.fd"); + write_fd_matching(&url, &info, &fd, vec![]).await; // `.fd` carries the URL + let plan = plan_from_fd(&fd, None, base_pc(&save_dir, false), tx, token) + .await + .unwrap(); + assert!(matches!(plan.resume_outcome(), ResumeOutcome::Fresh)); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_resume_inherits_persisted_config_before_probe() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("resume_inherit_config"); + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let final_path = save_dir.join("persisted.bin"); + let tmp = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + let persisted = PartialConfig { + save_dir: Some(save_dir.clone()), + filename: Some("persisted.bin".to_string()), + threads: Some(3), + ..Default::default() + }; + DownloadState::new(&url, &info, &persisted, &fd) + .store() + .await + .unwrap(); + std::fs::write(&tmp, vec![0u8; 1024]).unwrap(); + + let plan = plan_resume(&tmp, Some(url), PartialConfig::default(), tx, token) + .await + .unwrap(); + assert!(plan.final_path().ends_with("persisted.bin")); + assert_eq!(plan.config().save_dir, save_dir); + assert_eq!(plan.config().threads, 3); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +#[allow(clippy::single_range_in_vec_init)] +async fn fresh_plan_discards_unvalidated_caller_progress() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("fresh_discards_progress"); + let (tx, rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let plan = plan( + url, + PartialConfig { + save_dir: Some(save_dir.clone()), + filename: Some("a.bin".to_string()), + overwrite: Some(true), + downloaded_chunk: Some(vec![0u64..512]), + ..Default::default() + }, + tx, + token.clone(), + ) + .await + .unwrap(); + token.cancel(); + Box::pin(plan.start()).await; + while let Ok(event) = rx.recv().await { + if matches!(&event, Event::Terminated(_)) { + break; + } + } + let state = DownloadState::load(&save_dir.join("a.bin.fd")) + .await + .unwrap(); + assert_eq!(state.get_progress(), Vec::new()); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +#[allow(clippy::single_range_in_vec_init)] +async fn start_rechecks_a_resumable_part_that_disappeared() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("resume_part_disappeared"); + let (tx, rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let tmp = save_dir.join("a.bin.part"); + let fd = save_dir.join("a.bin.fd"); + write_fd_matching(&url, &info, &fd, vec![0u64..512]).await; + std::fs::write(&tmp, vec![0u8; 600]).unwrap(); + let plan = plan_resume( + &tmp, + Some(url), + base_pc(&save_dir, false), + tx, + token.clone(), + ) + .await + .unwrap(); + std::fs::remove_file(&tmp).unwrap(); + token.cancel(); + Box::pin(plan.start()).await; + + let mut resumed = false; + while let Ok(event) = rx.recv().await { + resumed |= matches!(&event, Event::Resumed { .. }); + if matches!(&event, Event::Terminated(_)) { + break; + } + } + assert!(!resumed, "a missing .part must not be announced as resumed"); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn forced_resume_revalidates_the_reloaded_fd_size() { + let body = vec![0xABu8; 1024]; + let url = spawn_server(&body).await; + let save_dir = make_dir("forced_reloads_size"); + let (tx, rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let info = prefetch_info(&url, &tx).await; + let tmp = save_dir.join("a.bin.part"); + let fd = save_dir.join("a.bin.fd"); + let mut stale_info = info.clone(); + stale_info.file_id = FileId::new(Some("old-etag"), None); + DownloadState::new(&url, &stale_info, &PartialConfig::default(), &fd) + .store() + .await + .unwrap(); + std::fs::write(&tmp, vec![0u8; 2048]).unwrap(); + let plan = plan_resume( + &tmp, + Some(url.clone()), + base_pc(&save_dir, false), + tx, + token, + ) + .await + .unwrap(); + assert!(matches!( + plan.resume_outcome(), + ResumeOutcome::Mismatch(StateError::FileChanged { + local_file_size: 1024, + remote_file_size: 1024, + .. + }) + )); + + write_fd_raw(&fd, &url, 2048).await; + Box::pin(plan.start_forced_resume()).await; + let mut rejected_new_size = false; + while let Ok(event) = rx.recv().await { + rejected_new_size |= matches!( + &event, + Event::ResumeError(StateError::FileChanged { + local_file_size: 2048, + remote_file_size: 1024, + .. + }) + ); + if matches!(&event, Event::Terminated(_)) { + break; + } + } + assert!(rejected_new_size); + let _ = std::fs::remove_dir_all(&save_dir); +} + +#[tokio::test] +async fn plan_from_fd_no_url_and_unresolvable_fd_is_nourl() { + let save_dir = make_dir("fromfd_nourl"); + let fd = save_dir.join("a.bin.fd"); + write_fd_raw(&fd, &Url::parse("about:blank").unwrap(), 1024).await; + let (tx, _rx) = crate::create_channel(); + let token = crate::create_cancellation_token(); + let res = plan_from_fd(&fd, None, PartialConfig::default(), tx, token).await; + assert!(matches!(res, Err(PlanError::Resume(StateError::NoUrl(_))))); + let _ = std::fs::remove_dir_all(&save_dir); +} diff --git a/crates/fast-down-api/src/core/prefetch.rs b/crates/fast-down-api/src/core/prefetch.rs index 152da16..30d5a39 100644 --- a/crates/fast-down-api/src/core/prefetch.rs +++ b/crates/fast-down-api/src/core/prefetch.rs @@ -1,9 +1,29 @@ -use crate::{Config, Event, Tx, tx_err, utils::build_header}; +use crate::{Config, Event, PlanError, Tx, utils::build_header}; use fast_down::{UrlInfo, fast_puller::build_client, http::Prefetch}; use reqwest::Response; use url::Url; -pub async fn prefetch(url: &Url, config: &Config, tx: &Tx) -> Option<(UrlInfo, Response)> { +/// Resolve the remote file's metadata, retrying up to +/// [`Config::retry_times`](crate::Config::retry_times). +/// +/// On success emits [`Event::Prefetch`] and hands back the [`UrlInfo`] together +/// with the open [`Response`], which the download pipeline reuses to seed the +/// first range request instead of paying for another round-trip. +/// +/// # Errors +/// +/// Retries that still have an attempt left are reported as +/// [`Event::PrefetchError`]; the attempt that exhausts the budget is returned +/// as [`PlanError::Prefetch`] instead, because +/// [`ReqwestResponseError`](fast_down::reqwest::ReqwestResponseError) cannot be +/// cloned into both places. A client that cannot be built at all is returned as +/// [`PlanError::BuildClient`] without emitting an event. +#[allow(clippy::result_large_err)] +pub async fn prefetch( + url: &Url, + config: &Config, + tx: &Tx, +) -> Result<(UrlInfo, Response), PlanError> { let client = build_client( build_header(&config.headers), config.proxy.as_deref(), @@ -12,21 +32,21 @@ pub async fn prefetch(url: &Url, config: &Config, tx: &Tx) -> Option<(UrlInfo, R config.cookie_store, config.local_address.first().copied(), config.max_redirects, - ); - let client = tx_err!(client, tx, BuildClientError, None); + ) + .map_err(PlanError::BuildClient)?; let mut retry_count = 0; loop { match client.prefetch(url.clone()).await { Ok(t) => { let _ = tx.send(Event::Prefetch(t.0.clone())); - break Some(t); + break Ok(t); } Err((e, t)) => { - let _ = tx.send(Event::PrefetchError(e)); retry_count += 1; if retry_count >= config.retry_times { - return None; + return Err(PlanError::Prefetch(e)); } + let _ = tx.send(Event::PrefetchError(e)); tokio::time::sleep(t.unwrap_or(config.retry_gap)).await; } } @@ -55,9 +75,9 @@ mod tests { #[tokio::test] async fn prefetch_gives_up_after_retries_on_unreachable() { - // Exercises the Err branch of prefetch (prefetch.rs lines 24-30): a - // connection that is refused must emit `Event::PrefetchError`, retry up - // to `retry_times`, then return `None`. + // Exercises the Err branch of prefetch: a connection that is refused + // must emit `Event::PrefetchError` for every attempt that still has a + // retry left, then return the final failure instead of emitting it. let url = Url::parse("http://127.0.0.1:1/never").unwrap(); let config = Config { retry_times: 2, @@ -67,8 +87,8 @@ mod tests { let (tx, rx) = create_channel(); let result = prefetch(&url, &config, &tx).await; assert!( - result.is_none(), - "prefetch must give up after exhausting retries" + matches!(result, Err(PlanError::Prefetch(_))), + "prefetch must return the last error after exhausting retries" ); drop(tx); let mut errors = 0; @@ -77,7 +97,10 @@ mod tests { errors += 1; } } - assert!(errors >= 1, "expected at least one Event::PrefetchError"); + assert_eq!( + errors, 1, + "2 attempts means 1 retryable failure is reported as an event and 1 is returned" + ); } #[tokio::test] @@ -147,7 +170,7 @@ mod tests { let result = prefetch(&url, &config, &tx).await; assert!( - result.is_some(), + result.is_ok(), "prefetch must succeed against a well-behaved server" ); let (info, _resp) = result.unwrap(); diff --git a/crates/fast-down-api/src/core/state.rs b/crates/fast-down-api/src/core/state.rs index 34818ad..f2e17b9 100644 --- a/crates/fast-down-api/src/core/state.rs +++ b/crates/fast-down-api/src/core/state.rs @@ -57,6 +57,18 @@ pub enum StateError { remote_file_id: FileId, remote_file_size: u64, }, + /// The `.part` file is shorter than the progress recorded in the `.fd`, so + /// the recorded ranges no longer describe bytes that exist on disk. + /// + /// Continuing would let the sink extend the file with zeros across the + /// missing span and never fetch it, silently corrupting the output. + #[error( + "the .part file was truncated: it holds {actual_size} bytes but the .fd state records progress up to {recorded_size}" + )] + Truncated { + actual_size: u64, + recorded_size: u64, + }, /// The server does not support resumable (range) downloads. #[error( "server does not support resumable download\n url_info: {:?}\n url: {}\n status: {}\n headers: {:?}", @@ -74,6 +86,12 @@ pub enum StateError { "cannot resume without a URL: none was supplied and the .fd state has none (tmp_path = {0})" )] NoUrl(PathBuf), + /// `peek_resume` 收到的 `tmp_path` 不是 `.part` 文件。 + /// + /// `peek_resume` 需要的是 `.part` 临时文件路径，其同名 `.fd` 才是保存的状态； + /// 传入其他扩展名的路径属于参数错误，而非文件读取失败。 + #[error("peek_resume requires a .part file, got: {0}")] + NotAPartFile(PathBuf), } /// Full (resolved) download state that is serialized into the `.fd` file. @@ -140,6 +158,22 @@ pub struct DownloadState { pub config_path: PathBuf, } +/// 已保存、可续传下载的只读快照，由 [`DownloadState::snapshot`] 产出。 +/// +/// 与引擎内部的 `DownloadState`（持有 `Mutex` 与脏标记、可持久化）不同，这是一个纯数据视图， +/// 供 UI 等只读消费者在不持有内部 `MutexGuard` 的情况下读取进度元数据。 +#[derive(Debug, Clone)] +pub struct ResumeInfo { + /// 持久化的初始 URL；旧 `.fd` 可能缺失，故为 `Option`。 + pub url: Option, + /// 保存时记录的总大小；缺失时回退为 `0`。 + pub size: u64, + /// 已写入 `.part` 的字节区间（续传进度的唯一真相）。 + pub progress: Vec, + /// 跨所有续传累计的活跃下载时长。 + pub elapsed: Duration, +} + impl DownloadState { /// Returns a reference to the shared inner state, locked for reading. /// Prefer this over the previous `Deref`-based field access. @@ -256,8 +290,13 @@ impl DownloadState { /// # Errors #[allow(clippy::result_large_err)] pub fn validate(&self, info: &UrlInfo) -> Result<(), StateError> { - let local_file_id = self.file_id(); - let local_file_size = self.inner.lock().size.unwrap_or(0); + let inner = self.inner.lock(); + let local_file_id = FileId { + etag: inner.etag.clone().flatten(), + last_modified: inner.last_modified.clone().flatten(), + }; + let local_file_size = inner.size.unwrap_or(0); + drop(inner); let is_same = local_file_size == info.size && local_file_id == info.file_id; if is_same { Ok(()) @@ -292,6 +331,25 @@ impl DownloadState { self.inner.lock().elapsed.unwrap_or(Duration::ZERO) } + /// 产出一份只读快照，供 UI 等只读消费者读取进度元数据，而无需持有内部 `MutexGuard`。 + /// + /// 与 [`DownloadState::lock_inner`] 不同，这里返回的是拥有所有权的 [`ResumeInfo`]， + /// 调用方可以随意读取 `url` / `size` / `progress` / `elapsed`，不会触及引擎内部的并发原语。 + #[must_use] + pub fn snapshot(&self) -> ResumeInfo { + let guard = self.inner.lock(); + ResumeInfo { + url: guard.url.clone(), + size: guard.size.unwrap_or(0), + progress: guard + .config + .as_ref() + .and_then(|c| c.downloaded_chunk.clone()) + .unwrap_or_default(), + elapsed: guard.elapsed.unwrap_or(Duration::ZERO), + } + } + /// Set the total accumulated active download time (absolute, not additive). /// /// Called by the download driver as time accrues, so the value persisted on @@ -382,6 +440,28 @@ impl DownloadState { pub fn tmp_path(&self) -> PathBuf { self.config_path.with_extension("part") } + + /// Measure `tmp_path` against the progress recorded in this state. + /// + /// Returns `Some((actual_size, recorded_size))` when the file is shorter + /// than the highest recorded offset — the state claims bytes that are not + /// on disk, so resuming from it would leave that span filled with zeros + /// that are never fetched. + /// + /// Returns `None` when the file is long enough, is not a regular file, or + /// cannot be inspected; those cases surface later when the sink is built. + pub async fn part_shortfall(&self, tmp_path: &Path) -> Option<(u64, u64)> { + let metadata = fs::metadata(tmp_path).await.ok()?; + if !metadata.is_file() { + return None; + } + let actual_size = metadata.len(); + // `downloaded_chunk` is normalized to ascending `start` order when the + // `.fd` is loaded, so the last entry's `end` is the highest recorded + // offset — the frontier a resume would continue from. + let recorded_size = self.get_progress().last().map_or(0, |r| r.end); + (actual_size < recorded_size).then_some((actual_size, recorded_size)) + } } #[cfg(test)] @@ -406,6 +486,30 @@ mod tests { DownloadState::new(&url, &url_info, &PartialConfig::default(), path) } + #[tokio::test] + async fn part_shortfall_reports_short_file_against_recorded_frontier() { + let dir = std::env::temp_dir().join(format!("fd_ps_short_{}", std::process::id())); + let _ = std::fs::create_dir_all(&dir); + let part = dir.join("a.part"); + let state = make_state(Path::new("dummy.fd")); + state.merge_progress(0u64..600); + std::fs::write(&part, vec![0u8; 500]).unwrap(); + assert_eq!(state.part_shortfall(&part).await, Some((500, 600))); + let _ = std::fs::remove_dir_all(&dir); + } + + #[tokio::test] + async fn part_shortfall_is_none_when_file_covers_frontier() { + let dir = std::env::temp_dir().join(format!("fd_ps_ok_{}", std::process::id())); + let _ = std::fs::create_dir_all(&dir); + let part = dir.join("a.part"); + let state = make_state(Path::new("dummy.fd")); + state.merge_progress(0u64..600); + std::fs::write(&part, vec![0u8; 700]).unwrap(); + assert_eq!(state.part_shortfall(&part).await, None); + let _ = std::fs::remove_dir_all(&dir); + } + #[tokio::test] async fn elapsed_stored_as_human_readable_string_and_round_trips() { let path = std::env::temp_dir().join(format!( diff --git a/crates/fast-down-api/src/event.rs b/crates/fast-down-api/src/event.rs index 2dd4a4b..6119bf2 100644 --- a/crates/fast-down-api/src/event.rs +++ b/crates/fast-down-api/src/event.rs @@ -12,6 +12,10 @@ use std::{path::PathBuf, time::Duration}; /// and completion ([`Event::Renamed`]). Error variants (`*Error`) report failures /// without aborting the stream, so a consumer can decide whether to retry, /// cancel, or surface them in a UI. +/// +/// Every run ends with exactly one [`Event::Terminated`], which is always the +/// last event on the channel. A consumer that only needs the outcome can wait +/// for it instead of draining until the channel disconnects. #[allow(clippy::large_enum_variant)] #[derive(Debug)] pub enum Event { @@ -38,6 +42,20 @@ pub enum Event { BuildClientError(reqwest::Error), /// Creating the output sink — opening the `.part` file — failed. BuildPusherError(std::io::Error), + /// Disk space for the whole file is about to be reserved, carrying the + /// target size in bytes. + /// + /// Only emitted when [`crate::Config::pre_alloc`] is enabled and the remote + /// size is known. Where the platform has no fast-reservation path this is + /// followed by a full-size zero-fill pass, which can take a while — this + /// event exists so a UI can say so instead of appearing frozen. + Allocating(u64), + /// Reserving disk space failed. + /// + /// Non-fatal: the download continues and the file grows on demand. The + /// trade-off is more fragmentation and the chance of running out of space + /// mid-download rather than up front. + AllocError(std::io::Error), /// The final rename of the `.part` file to its destination failed. /// /// The success counterpart is [`Event::Renamed`]. The bytes are already on @@ -115,6 +133,32 @@ pub enum Event { FlushError(anyhow::Error), /// Worker `id` completed its assigned range and exited. Finished(WorkerId), + + /// The run has ended. Sent exactly once, as the last event on the channel. + /// + /// Preceding `*Error` events carry the details of whatever went wrong; this + /// one only reports the outcome. + Terminated(TerminationReason), +} + +/// How a download run ended, carried by [`Event::Terminated`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TerminationReason { + /// Every byte was written and the `.part` file was renamed into place. + /// [`Event::Renamed`] carries the path it landed on. + Completed, + /// The [`CancellationToken`](crate::create_cancellation_token) was + /// triggered. The `.part` and `.fd` files are left on disk so a later + /// resume can pick up where this run stopped. + Cancelled, + /// The run stopped on its own without completing the file — every worker + /// gave up, for example because the connection kept failing. Like + /// [`TerminationReason::Cancelled`], the `.part` and `.fd` files are left + /// on disk. + Incomplete, + /// A fatal error ended the run: metadata could not be fetched, the output + /// file could not be opened, the rename failed, and so on. + Failed, } /// Computed aggregate view of the current download progress, carried by diff --git a/crates/fast-down-api/src/lib.rs b/crates/fast-down-api/src/lib.rs index e3b0f09..aa1a83c 100644 --- a/crates/fast-down-api/src/lib.rs +++ b/crates/fast-down-api/src/lib.rs @@ -11,7 +11,7 @@ pub use event::*; pub use fast_down; -use tokio_util::sync::CancellationToken; +pub use tokio_util::sync::CancellationToken; /// Sender half of the event channel, used to push [`Event`]s from the download task. pub type Tx = crossfire::MTx>; @@ -26,13 +26,12 @@ pub fn create_channel() -> (Tx, Rx) { crossfire::mpmc::unbounded_async() } -/// Create a new cancellation token for use with download tasks. +/// Create a new [`CancellationToken`] for cooperative cancellation of a download. /// -/// Pass the token to [`download`] or [`resume`] -/// to cancel the download at any time. Cancellation is cooperative: the running -/// task stops fetching, leaves the `.part`/`.fd` files in place, and returns -/// without renaming — so a later [`resume`] call can continue -/// from where it stopped. +/// Pass the returned token to [`download`] (or a related entry +/// point) and call +/// [`CancellationToken::cancel`](tokio_util::sync::CancellationToken::cancel) to +/// abort the in-flight download. #[must_use] pub fn create_cancellation_token() -> CancellationToken { CancellationToken::new() diff --git a/crates/fast-down-api/src/utils/filename_template.rs b/crates/fast-down-api/src/utils/filename_template.rs index de53085..71f56ce 100644 --- a/crates/fast-down-api/src/utils/filename_template.rs +++ b/crates/fast-down-api/src/utils/filename_template.rs @@ -3,9 +3,9 @@ use path_helper::sanitize_filename; use std::panic; use url::Url; -pub fn parse_filename_template(template: String, url: &Url, filename: &str) -> String { - let template = - panic::catch_unwind(|| Local::now().format(&template).to_string()).unwrap_or(template); +pub fn parse_filename_template(template: &str, url: &Url, filename: &str) -> String { + let template = panic::catch_unwind(|| Local::now().format(template).to_string()) + .unwrap_or_else(|_| template.to_string()); let host = sanitize_filename(url.host_str().unwrap_or("unknown"), 255); let mut parent_path: Vec<_> = url .path_segments() @@ -13,11 +13,12 @@ pub fn parse_filename_template(template: String, url: &Url, filename: &str) -> S .flat_map(|segments| { segments.map(|seg| { let decoded = urlencoding::decode_binary(seg.as_bytes()); - sanitize_filename(String::from_utf8_lossy(&decoded), 255) + sanitize_filename(String::from_utf8_lossy(&decoded).as_ref(), 255) }) }) .collect(); parent_path.pop(); + parent_path.retain(|segment| !segment.is_empty()); let parent_path = if parent_path.is_empty() { ".".to_string() } else { @@ -45,7 +46,7 @@ mod tests { fn all_placeholders() { let url = Url::parse("https://example.com/path/to/file.txt").unwrap(); let t = "{host}/{parent_path}/{file_name}_{file_stem}{file_ext}"; - let out = parse_filename_template(t.to_string(), &url, "file.txt"); + let out = parse_filename_template(t, &url, "file.txt"); assert!(out.starts_with("example.com")); assert!(out.contains("path")); assert!(out.contains("to")); @@ -55,26 +56,20 @@ mod tests { #[test] fn no_placeholders_passthrough() { let url = Url::parse("https://example.com/x").unwrap(); - assert_eq!( - parse_filename_template("plain".to_string(), &url, "f.txt"), - "plain" - ); + assert_eq!(parse_filename_template("plain", &url, "f.txt"), "plain"); } #[test] fn host_unknown_when_no_host() { let url = Url::parse("file:///etc/hosts").unwrap(); - assert_eq!( - parse_filename_template("{host}".to_string(), &url, "hosts"), - "unknown" - ); + assert_eq!(parse_filename_template("{host}", &url, "hosts"), "unknown"); } #[test] fn parent_path_root_when_no_dir() { let url = Url::parse("https://example.com/file.txt").unwrap(); assert_eq!( - parse_filename_template("{parent_path}".to_string(), &url, "file.txt"), + parse_filename_template("{parent_path}", &url, "file.txt"), "." ); } @@ -82,14 +77,14 @@ mod tests { #[test] fn file_ext_includes_dot() { let url = Url::parse("https://example.com/a/b.tar.gz").unwrap(); - let out = parse_filename_template("{file_stem}{file_ext}".to_string(), &url, "b.tar.gz"); + let out = parse_filename_template("{file_stem}{file_ext}", &url, "b.tar.gz"); assert_eq!(out, "b.tar.gz"); } #[test] fn no_dot_file_has_empty_ext() { let url = Url::parse("https://example.com/README").unwrap(); - let out = parse_filename_template("{file_stem}|{file_ext}".to_string(), &url, "README"); + let out = parse_filename_template("{file_stem}|{file_ext}", &url, "README"); assert_eq!(out, "README|"); } @@ -98,7 +93,7 @@ mod tests { // `mailto:` URLs are cannot-be-a-base, so `path_segments()` is `None` and // the parent path collapses to "." (filename_template.rs lines 10-25). let url = Url::parse("mailto:foo@x").unwrap(); - let out = parse_filename_template("{parent_path}/{file_name}".to_string(), &url, "foo.txt"); + let out = parse_filename_template("{parent_path}/{file_name}", &url, "foo.txt"); assert_eq!(out, "./foo.txt"); } @@ -107,7 +102,7 @@ mod tests { // A leading `%Y` is a chrono format spec expanded by `Local::now().format` // before the `{...}` placeholders are substituted (filename_template.rs line 7). let url = Url::parse("https://example.com/file.txt").unwrap(); - let out = parse_filename_template("%Y/file.txt".to_string(), &url, "file.txt"); + let out = parse_filename_template("%Y/file.txt", &url, "file.txt"); let year = chrono::Local::now().format("%Y").to_string(); assert_eq!(out, format!("{year}/file.txt")); } diff --git a/crates/fast-down-api/src/utils/gen_path.rs b/crates/fast-down-api/src/utils/gen_path.rs index df13c44..641ecea 100644 --- a/crates/fast-down-api/src/utils/gen_path.rs +++ b/crates/fast-down-api/src/utils/gen_path.rs @@ -3,7 +3,6 @@ use fast_down::UrlInfo; use path_helper::{auto_ext, sanitize_filename, sanitize_path}; use soft_canonicalize::soft_canonicalize; use std::{borrow::Cow, path::PathBuf}; -use tokio::fs; use url::Url; pub async fn gen_path(url: &Url, info: &UrlInfo, config: &Config) -> std::io::Result { @@ -12,18 +11,15 @@ pub async fn gen_path(url: &Url, info: &UrlInfo, config: &Config) -> std::io::Re auto_ext(&info.raw_name, info.content_type.as_deref()) } else { Cow::Borrowed(config.filename.as_str()) - }, + } + .as_ref(), 248, ); let mut save_dir = soft_canonicalize::soft_canonicalize(&config.save_dir)?; if config.parse_filename && !config.filename.is_empty() { - let path = PathBuf::from(parse_filename_template( - config.filename.clone(), - url, - &filename, - )); + let path = PathBuf::from(parse_filename_template(&config.filename, url, &filename)); if let Some(s) = path.file_name() { - filename = sanitize_filename(s.to_string_lossy(), 248); + filename = sanitize_filename(s.to_string_lossy().as_ref(), 248); } if let Some(parent_path) = path.parent() && let Ok(new_save_dir) = soft_canonicalize(save_dir.join(sanitize_path(parent_path))) @@ -32,7 +28,6 @@ pub async fn gen_path(url: &Url, info: &UrlInfo, config: &Config) -> std::io::Re save_dir = new_save_dir; } } - fs::create_dir_all(&save_dir).await?; Ok(save_dir.join(&filename)) } @@ -106,8 +101,14 @@ mod tests { let p = gen_path(&url, &info, &cfg).await.unwrap(); // parent_path of /a/b/data.bin is "a/b", so the resolved path ends with it. assert!(p.ends_with("a/b/data.bin"), "unexpected path: {p:?}"); - // The synthesized parent directory must have been created by gen_path. - assert!(p.parent().is_some_and(std::path::Path::exists)); + // `gen_path` only computes the path; it must NOT create the directory. + // Directory creation is the download executor's job (`claim_and_run`), + // covered by the integration test `download_creates_template_subdir` + // in tests/resume.rs. + assert!( + !p.parent().unwrap().exists(), + "gen_path must not touch the filesystem" + ); } #[tokio::test] @@ -143,4 +144,26 @@ mod tests { "the file name must be the template's leaf, not the traversal target, got {p:?}" ); } + + #[cfg(windows)] + #[tokio::test] + async fn unix_style_save_dir_maps_to_rooted_path() { + // Regression baseline for the path-mangling defect: a Git-Bash style + // `/c/Users/...` save_dir is canonicalized as a rooted path under the + // current drive (e.g. `E:\c\Users\...`) rather than `C:\Users\...`. + // `gen_path` itself faithfully follows Windows path semantics; the fix + // belongs in the CLI layer (translate `/x/...` -> `X:/...` before the + // config is built). Keeping this test pins the current behavior so a + // future change to `gen_path`'s path handling is caught. + let dir = std::path::PathBuf::from("/c/Users/example/downloads"); + let url = Url::parse("https://example.com/a.bin").unwrap(); + let info = make_info("a.bin", None); + let cfg = make_config(&dir, "a.bin", false); + let p = gen_path(&url, &info, &cfg).await.unwrap(); + let s = p.to_string_lossy(); + assert!( + s.contains("c\\Users") || s.contains("c/Users"), + "expected a rooted `/c/...` mapping under the current drive, got {s}" + ); + } } diff --git a/crates/fast-down-api/tests/resume.rs b/crates/fast-down-api/tests/resume.rs index 317e871..6b8515a 100644 --- a/crates/fast-down-api/tests/resume.rs +++ b/crates/fast-down-api/tests/resume.rs @@ -23,7 +23,7 @@ use std::time::Duration; use bytes::Bytes; use fast_down_api::{ Event, PartialConfig, Rx, StateError, WriteMethod, create_cancellation_token, create_channel, - download, resume, + download, download_from_fd, resume, }; use futures::StreamExt; use futures::stream::unfold; @@ -2271,3 +2271,285 @@ async fn test_resume_rejects_non_part_extension() { "should not rename when tmp_path has wrong extension" ); } + +/// A filename template that expands into a subdirectory of `save_dir` (e.g. +/// `{parent_path}/{file_name}`) must make the download executor create that +/// subdirectory before writing, so the file lands inside it. +/// +/// This is the integration-level counterpart of the pure-`gen_path` contract: +/// `gen_path` only computes the path and must not touch the filesystem, while +/// `claim_and_run` creates the parent directory right before opening the file. +#[tokio::test] +async fn download_creates_template_subdir() { + let dir = temp_dir("template_subdir"); + let (_server, base) = start_server(original_bytes(), "orig", "LM-A", true).await; + // A multi-segment URL path so `{parent_path}` expands to a non-empty subdir. + let url = format!("{base}/a/b/data.bin"); + + let cfg = PartialConfig { + save_dir: Some(dir.clone()), + filename: Some("{parent_path}/{file_name}".to_string()), + parse_filename: Some(true), + overwrite: Some(true), + write_method: Some(WriteMethod::Mmap), + min_chunk_size: Some(1024 * 1024), + threads: Some(32), + cache_high_watermark: Some(1), + cache_low_watermark: Some(0), + write_buffer_size: Some(1), + ..Default::default() + }; + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + download(Url::parse(&url).expect("valid url"), cfg, tx, cancel); + let events = drain(rx).await; + + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "a template-subdir download must complete with Renamed" + ); + + let final_path = dir.join("a").join("b").join("data.bin"); + assert!( + final_path.exists(), + "the template subdir must be created and the file must land inside it" + ); + let got = tokio::fs::read(&final_path).await.expect("read final file"); + assert_eq!( + got, + original_bytes(), + "downloaded content must match source" + ); +} + +// ---- download_from_fd (manifest-driven) tests ---- + +/// `download_from_fd` resumes when both `.fd` and `.part` are present and +/// describe the remote file: it emits `Event::Resumed` (inheriting the +/// cancelled run's progress) and completes with `Event::Renamed`. +#[tokio::test] +async fn test_download_from_fd_resumes_when_part_present() { + let dir = temp_dir("from_fd_resume"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + assert!( + part.exists() && fd.exists(), + "precondition: .part and .fd must exist" + ); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel2 = create_cancellation_token(); + download_from_fd( + fd.clone(), + Some(Url::parse(&url).expect("valid url")), + cfg, + tx, + cancel2, + ); + let events = timeout(Duration::from_secs(30), drain(rx)) + .await + .expect("drain timed out"); + + assert!( + events.iter().any(|e| matches!(e, Event::Resumed { .. })), + "download_from_fd must resume (emit Event::Resumed) when .part exists" + ); + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "download_from_fd must complete with Renamed" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "download_from_fd must NOT emit a ResumeError on a valid state" + ); + let got = tokio::fs::read(&final_path).await.expect("read final file"); + assert_eq!(got, original_bytes(), "resumed content mismatch"); +} + +/// `download_from_fd` reuses the `.fd` manifest but restarts byte progress from +/// zero when the `.part` is missing: it must complete (Renamed) with the correct +/// content and must NOT emit a `ResumeError`. +#[tokio::test] +async fn test_download_from_fd_reuses_fd_when_part_missing() { + let dir = temp_dir("from_fd_missing_part"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + assert!(fd.exists(), "precondition: .fd must exist"); + // The `.part` is gone; only the manifest remains — exactly the case the + // feature targets. + tokio::fs::remove_file(&part).await.expect("delete .part"); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel2 = create_cancellation_token(); + download_from_fd( + fd.clone(), + Some(Url::parse(&url).expect("valid url")), + cfg, + tx, + cancel2, + ); + let events = timeout(Duration::from_secs(30), drain(rx)) + .await + .expect("drain timed out"); + + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "download_from_fd must NOT emit a ResumeError when .part is missing" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::Resumed { .. })), + "a manifest-only fresh download must not be reported as resumed" + ); + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "download_from_fd must complete with Renamed even without a .part" + ); + let got = tokio::fs::read(&final_path).await.expect("read final file"); + assert_eq!( + got, + original_bytes(), + "fresh download from the manifest must produce correct content" + ); +} + +/// `download_from_fd` reuses the `.fd` manifest for a fresh, single-stream +/// download when the server does not support range requests (so byte-resume is +/// impossible) and the `.part` is missing. This exercises the +/// `!info.fast_download` branch of `plan_from_fd` (which always reuses the +/// manifest) together with the explicit manifest-fresh start action. +#[tokio::test] +async fn test_download_from_fd_reuses_fd_when_part_missing_non_range() { + let dir = temp_dir("from_fd_missing_part_non_range"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", false).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + assert!(fd.exists(), "precondition: .fd must exist"); + tokio::fs::remove_file(&part).await.expect("delete .part"); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel2 = create_cancellation_token(); + download_from_fd( + fd.clone(), + Some(Url::parse(&url).expect("valid url")), + cfg, + tx, + cancel2, + ); + let events = timeout(Duration::from_secs(30), drain(rx)) + .await + .expect("drain timed out"); + + assert!( + !events.iter().any(|e| matches!(e, Event::ResumeError(_))), + "download_from_fd must NOT emit a ResumeError when .part is missing" + ); + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "download_from_fd must complete with Renamed even without a .part on a non-range server" + ); + let got = tokio::fs::read(&final_path).await.expect("read final file"); + assert_eq!( + got, + original_bytes(), + "fresh single-stream download from the manifest must produce correct content" + ); +} + +/// `download_from_fd` reports `StateError::Open` when the `.fd` itself is +/// missing — there is no manifest to drive the download from. +#[tokio::test] +async fn test_download_from_fd_missing_fd_reports_error() { + let dir = temp_dir("from_fd_missing"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let final_path = dir.join("out.bin"); + let fd = final_path.with_added_extension("fd"); + assert!(!fd.exists(), "precondition: .fd must not exist"); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel = create_cancellation_token(); + download_from_fd( + fd.clone(), + Some(Url::parse(&url).expect("valid url")), + cfg, + tx, + cancel, + ); + let events = timeout(Duration::from_secs(30), drain(rx)) + .await + .expect("drain timed out"); + + let err = events + .iter() + .find_map(|e| match e { + Event::ResumeError(r) => Some(r), + _ => None, + }) + .expect("expected Event::ResumeError for a missing .fd"); + assert!( + matches!(err, StateError::Open(_)), + "expected Event::ResumeError(StateError::Open) for a missing .fd, got {err:?}" + ); + assert!( + !events.iter().any(|e| matches!(e, Event::Renamed(_))), + "download_from_fd must NOT rename when there is no .fd" + ); +} + +/// `download_from_fd` with `url = None` reuses the durable initial URL recorded +/// in the `.fd`, so an external program can drive a resume from the `.fd` path +/// alone — the core "hand over a `.fd`" use case. +#[tokio::test] +async fn test_download_from_fd_uses_fd_url_when_none_given() { + let dir = temp_dir("from_fd_url_none"); + let (_server, url) = start_server(original_bytes(), "orig", "LM-A", true).await; + + let cancel = create_cancellation_token(); + partial_download_via_cancel(&url, &dir, cancel).await; + + let final_path = dir.join("out.bin"); + let part = final_path.with_added_extension("part"); + let fd = final_path.with_added_extension("fd"); + assert!(part.exists() && fd.exists()); + + let cfg = make_config(&dir); + let (tx, rx) = create_channel(); + let cancel2 = create_cancellation_token(); + // No url: the `.fd` must supply the durable initial URL. + download_from_fd(fd.clone(), None, cfg, tx, cancel2); + let events = timeout(Duration::from_secs(30), drain(rx)) + .await + .expect("drain timed out"); + + assert!( + events.iter().any(|e| matches!(e, Event::Resumed { .. })), + "download_from_fd(None) must resume using the .fd's durable url" + ); + assert!( + events.iter().any(|e| matches!(e, Event::Renamed(_))), + "download_from_fd(None) must complete with Renamed" + ); + let got = tokio::fs::read(&final_path).await.expect("read final file"); + assert_eq!(got, original_bytes(), "resumed content mismatch"); +} diff --git a/crates/fast-down/Cargo.toml b/crates/fast-down/Cargo.toml index 91cfa5c..f1fcc41 100644 --- a/crates/fast-down/Cargo.toml +++ b/crates/fast-down/Cargo.toml @@ -1,11 +1,10 @@ [package] name = "fast-down" -version = "5.0.2" +version = "6.0.0" authors.workspace = true edition.workspace = true description = "Download everything fast" documentation = "https://docs.rs/fast-down" -readme = "README.md" homepage.workspace = true repository.workspace = true license.workspace = true @@ -38,7 +37,12 @@ mockito = "1.7" tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } [features] -http = ["dep:parking_lot", "dep:thiserror", "dep:urlencoding", "dep:async-stream"] +http = [ + "dep:parking_lot", + "dep:thiserror", + "dep:urlencoding", + "dep:async-stream", +] reqwest = ["dep:httpdate", "dep:reqwest", "http"] sanitize-filename = ["dep:path_helper"] serde = ["dep:serde", "url/serde"] diff --git a/crates/fast-pull/Cargo.toml b/crates/fast-pull/Cargo.toml index a126b03..5d5eadb 100644 --- a/crates/fast-pull/Cargo.toml +++ b/crates/fast-pull/Cargo.toml @@ -1,11 +1,10 @@ [package] name = "fast-pull" -version = "5.0.2" +version = "6.0.0" authors.workspace = true edition.workspace = true description = "Pull everything fast" documentation = "https://docs.rs/fast-pull" -readme = "README.md" homepage.workspace = true repository.workspace = true license.workspace = true diff --git a/crates/fast-pull/src/core/single.rs b/crates/fast-pull/src/core/single.rs index c43db11..856222c 100644 --- a/crates/fast-pull/src/core/single.rs +++ b/crates/fast-pull/src/core/single.rs @@ -828,4 +828,73 @@ mod tests { while result.event_chain().recv().await.is_ok() {} assert_eq!(&**receive.lock(), mock_data); } + + // ------------------------------------------------------------------------- + // Regression baseline for the stalled-body hang (FluxDown-style #545). + // ------------------------------------------------------------------------- + + use crate::{PullResult, PullStream}; + use std::pin::Pin; + use std::task::{Context, Poll}; + + /// A stream whose `next` never resolves — models a server that answers + /// the request (headers) but never delivers the body (a "stall"/"slowloris" + /// server). `TryStream` is satisfied automatically because `Item` is a + /// `Result`, so we only need to implement the underlying `Stream`. + struct PendingStream; + impl futures::Stream for PendingStream { + type Item = Result)>; + fn poll_next(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Pending + } + } + impl Unpin for PendingStream {} + + /// A puller that returns a never-resolving stream (see [`PendingStream`]). + #[derive(Clone)] + struct StallPuller; + impl Puller for StallPuller { + type Error = std::convert::Infallible; + fn pull( + &mut self, + _range: Option<&ProgressEntry>, + ) -> impl Future, Self::Error>> + Send + { + std::future::ready(Ok(PendingStream)) + } + } + + /// `download_single` reads the body with a bare `stream.try_next().await` + /// (single.rs) and its `DownloadOptions` has no `pull_timeout` field, so a + /// stalled body blocks forever. This asserts the session ends (a + /// `PullTimeout`/error surfaces and the event channel closes) within 3s. + /// + /// `#[ignore]`d because the fix (adding `pull_timeout` to `DownloadOptions` + /// and giving up with an error instead of hanging) does not exist yet; remove + /// `#[ignore]` once that lands. + #[tokio::test] + #[ignore = "regression baseline: download_single has no pull_timeout, a stalled body hangs forever; enable after adding pull_timeout to DownloadOptions and surfacing a PullTimeout/error"] + async fn test_single_stall_body_hangs_without_timeout() { + let puller = StallPuller; + let pusher = MemPusher::with_capacity(0); + let result = download_single( + puller, + pusher, + DownloadOptions { + retry_gap: Duration::from_secs(1), + push_queue_cap: 1024, + }, + ); + // The session must not block forever: within 3s the event channel must + // close (a timeout/error must surface and end the download). Today it + // hangs, so the outer timeout fires and the assertion fails. + let drained = tokio::time::timeout(Duration::from_secs(3), async { + while result.event_chain().recv().await.is_ok() {} + }) + .await; + assert!( + drained.is_ok(), + "download_single must not hang forever on a stalled body; a pull_timeout should surface a PullTimeout or error and end the session" + ); + } } diff --git a/crates/fast-steal/Cargo.toml b/crates/fast-steal/Cargo.toml index c2247d2..ba24a91 100644 --- a/crates/fast-steal/Cargo.toml +++ b/crates/fast-steal/Cargo.toml @@ -1,11 +1,10 @@ [package] name = "fast-steal" -version = "6.5.4" +version = "7.0.0" authors.workspace = true edition.workspace = true description = "An ultra-fast multi-threaded task scheduler with fine-grained work stealing" documentation = "https://docs.rs/fast-steal" -readme = "README.md" homepage.workspace = true repository.workspace = true license.workspace = true