From 21d0b9afdb04f21c933b53ddfe15fec0c03b6d81 Mon Sep 17 00:00:00 2001 From: njzjz-bot Date: Mon, 24 Aug 2026 03:58:38 +0800 Subject: [PATCH] docs: describe workflow artifact downloads Document the training, exploration, and labeling download interface and cover its iteration and artifact filters. Coding-Agent: Codex Codex-Version: codex-cli 0.149.1 Model: gpt-5.6-sol Reasoning-Effort: xhigh --- docs/quickcli.md | 30 ++++++++++++++++++++++++++++++ dpgen2/entrypoint/main.py | 6 +++--- tests/entrypoint/test_argparse.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/docs/quickcli.md b/docs/quickcli.md index e9003715..fa63871c 100644 --- a/docs/quickcli.md +++ b/docs/quickcli.md @@ -39,6 +39,36 @@ INFO:root:steps iter-000001--prep-run-explore--------------------- finished The artifacts can be downloaded on-the-fly with `-d` flag. Note that the existing files are automatically skipped if one sets `dflow_config["archive_mode"] = None`. +## Download workflow results + +The `download` command retrieves training, exploration, and labeling artifacts without requiring direct access to the workflow storage backend. List the supported artifact names first: + +```bash +dpgen2 download input.json WFID --list-supported +``` + +Running without filters downloads every supported artifact from every successful iteration. Use iteration and artifact filters for a smaller result set: + +```bash +dpgen2 download input.json WFID \ + --iterations 0-2 \ + --step-definitions \ + prep-run-train/output/models \ + prep-run-train/output/lcurves \ + prep-run-train/output/logs \ + prep-run-explore/output/trajs \ + prep-run-explore/output/model_devis \ + prep-run-fp/output/labeled_data \ + --prefix results +``` + +Files are organized below `results/iter-000000///`. Existing completed downloads are skipped by default; pass `--no-check-point` to request them again. The corresponding result groups are: + +- training: models, learning curves, logs, and generated scripts; +- exploration: trajectories, model deviations, logs, and extra outputs; +- labeling: input configurations, labeled data, logs, and extra outputs. + + ## Show the keys of steps Each dpgen2 step is assigned a unique key. The keys of the finished steps can be checked with `showkey` command diff --git a/dpgen2/entrypoint/main.py b/dpgen2/entrypoint/main.py index 00ee55e3..dfba2bfb 100644 --- a/dpgen2/entrypoint/main.py +++ b/dpgen2/entrypoint/main.py @@ -149,13 +149,13 @@ def main_parser() -> argparse.ArgumentParser: 1. list all supported steps and their input/output artifacts $ dpgen2 download CONFIG ID -l - 2. donwload all the input/output of all the steps. + 2. download all supported input/output artifacts of all successful steps. $ dpgen2 download CONFIG ID - 3. donwload specified input/output artifacts of certain steps. For example + 3. download specified input/output artifacts of certain steps. For example $ dpgen2 download CONFIG ID -i 0-8 8 9 -d prep-run-train/input/init_data prep-run-explore/output/trajs - The command will download the init_data of prep-run-train's input and trajs of the prep-run-explore's output from iterations 0 to 9 (by -i 0-8 8 9). + The command downloads prep-run-train init_data and prep-run-explore trajectories from iterations 0 to 9 (selected by -i 0-8 8 9). The supported step and the names of input/output can be checked by the -l flag. """ ) diff --git a/tests/entrypoint/test_argparse.py b/tests/entrypoint/test_argparse.py index 6f494afb..a64c3bfe 100644 --- a/tests/entrypoint/test_argparse.py +++ b/tests/entrypoint/test_argparse.py @@ -75,6 +75,37 @@ def test_dld(self): self.assertEqual(parsed.keys, ["foo", "bar", "tar"]) self.assertEqual(parsed.prefix, "myprefix") + def test_download_by_definition(self): + parsed = self.parser.parse_args( + [ + "download", + "input.json", + "workflow-id", + "--iterations", + "0-2", + "4", + "--step-definitions", + "prep-run-train/output/models", + "prep-run-explore/output/trajs", + "prep-run-fp/output/labeled_data", + "--prefix", + "results", + "--no-check-point", + ] + ) + + self.assertEqual(parsed.iterations, ["0-2", "4"]) + self.assertEqual( + parsed.step_definitions, + [ + "prep-run-train/output/models", + "prep-run-explore/output/trajs", + "prep-run-fp/output/labeled_data", + ], + ) + self.assertEqual(parsed.prefix, "results") + self.assertFalse(parsed.no_check_point) + def test_resubmit(self): parsed = self.parser.parse_args( [