From 6ad80dba9cefe4b4d75d9ca1045b8634429cfe7b Mon Sep 17 00:00:00 2001 From: dylan Date: Sat, 5 Sep 2026 00:29:52 +0000 Subject: [PATCH 1/5] data: add released v23 holdout datasets to public registry (18 mirrors) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 18 v23 holdouts that had no usable upstream repo and were mirrored to 34data (bmcore PR #359). Uploaded and verified: 18/18 repos hold data, 56.92 GB across 326k source files. real_images +7 medical imaging (kidney-stone-ct, malaria, pulmonary chest x-ray, dermnet, coronahack, lits-256, tcga-coad) synthetic_images +3 synthetic-human, ai-generated-ecommerce x2 synthetic_videos +7 VideoGen-RewardBench x5 (easyanimate/opensora1.2/ qingying/tongyi/vidu) + edtalk, float-talking-head synthetic_audio +1 indic-tts-urdu Archive bounds are per-run download limits, not sample counts — the per-dataset sample cap comes from calculate_weighted_dataset_sampling. In full mode the YAML values are respected (DOWNLOAD_SIZE_OVERRIDES is debug/small only), so -1/-1 would fetch each dataset in full every run to use a few hundred samples, and would also lose variety: _select_files_to_download draws a seeded random subset when the count is bounded, but returns everything when it is -1. Bounded values give per-run economy and rotate archives across rounds as the seed changes. image 1000/1 -> 1000 available vs 355 cap video 500/1 -> 500 available vs 149 cap audio 500/2 -> 1000 available vs 336 cap For these 18 that is 56.9 GB -> 20.6 GB per full run (64% less), concentrated in synthetic-human (16.6 -> 2.4), float-talking-head (13.1 -> 4.4), tcga (5.4 -> 1.8) and edtalk (5.3 -> 1.8). Video entries declare source_format: mp4 but hold zips; the loader's fallback recalculates n_files on the archive branch, so archives_per_dataset governs. Raise full-mode targets so 18 new datasets do not dilute the existing ones: image 69000->72500, video 33000->34000, audio 47000->47500. Per-dataset caps are held at 355 / 149 / 336 (audio +1), unchanged from before this wave. edtalk and float-talking-head keep media_type: semisynthetic, joining the 29 existing semisynthetic video entries. The other 33 v23 entries are not here: 22 defer to existing ungated upstream repos and will land separately with hf_revision pinned, 7 have gated upstreams and still need mirroring, and 4 wild-* stay withheld. Registry loads clean at 565 datasets, no duplicate names or paths, and all 18 paths resolve on HuggingFace with data. --- src/gasbench/dataset/config.py | 2 +- src/gasbench/dataset/configs/real_images.yaml | 72 +++++++++++++++++++ .../dataset/configs/synthetic_audio.yaml | 12 ++++ .../dataset/configs/synthetic_images.yaml | 32 +++++++++ .../dataset/configs/synthetic_videos.yaml | 72 +++++++++++++++++++ 5 files changed, 189 insertions(+), 1 deletion(-) diff --git a/src/gasbench/dataset/config.py b/src/gasbench/dataset/config.py index a814e2d..5c5c2b3 100644 --- a/src/gasbench/dataset/config.py +++ b/src/gasbench/dataset/config.py @@ -25,7 +25,7 @@ BENCHMARK_TOTAL_OVERRIDES = { "debug": {"image": 100, "video": 50, "audio": 50}, "small": {"image": 5000, "video": 5000, "audio": 5000}, - "full": {"image": 69000, "video": 33000, "audio": 47000}, + "full": {"image": 72500, "video": 34000, "audio": 47500}, } # Per-dataset download limits (only applied in debug/small modes for faster testing) diff --git a/src/gasbench/dataset/configs/real_images.yaml b/src/gasbench/dataset/configs/real_images.yaml index 3a9afac..39f68f3 100644 --- a/src/gasbench/dataset/configs/real_images.yaml +++ b/src/gasbench/dataset/configs/real_images.yaml @@ -1151,3 +1151,75 @@ datasets: generator_family: real content_category: documents notes: Released from v22 bmcore holdouts + + # ── Released v23 holdouts ───────────────────────────────────────────────── + + - name: kidney-stone-ct + path: 34data/kidney-stone-ct + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: malaria-bounding-boxes + path: 34data/malaria-bounding-boxes + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: pulmonary-chest-xray-abnormalities + path: 34data/pulmonary-chest-xray-abnormalities + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: dermnet + path: 34data/dermnet + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: coronahack-chest-xray + path: 34data/coronahack-chest-xray + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: lits-256 + path: 34data/lits-256 + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: tcga-coad-msi-mss + path: 34data/tcga-coad-msi-mss + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical diff --git a/src/gasbench/dataset/configs/synthetic_audio.yaml b/src/gasbench/dataset/configs/synthetic_audio.yaml index c285ba2..da74040 100644 --- a/src/gasbench/dataset/configs/synthetic_audio.yaml +++ b/src/gasbench/dataset/configs/synthetic_audio.yaml @@ -796,3 +796,15 @@ datasets: generator_family: mixed-generation content_category: speech notes: Synthetic subset of the official RuASD anti-spoofing dataset + + # ── Released v23 holdouts ───────────────────────────────────────────────── + + - name: indic-tts-urdu + path: 34data/indic-tts-urdu + modality: audio + media_type: synthetic + source_format: zip + media_per_archive: 500 + archives_per_dataset: 2 + generator_family: mixed-generation + content_category: speech diff --git a/src/gasbench/dataset/configs/synthetic_images.yaml b/src/gasbench/dataset/configs/synthetic_images.yaml index a031471..ef12539 100644 --- a/src/gasbench/dataset/configs/synthetic_images.yaml +++ b/src/gasbench/dataset/configs/synthetic_images.yaml @@ -942,3 +942,35 @@ datasets: notes: PICA-100K is purely synthetic (PICABench; video-derived src and edited tgt). Not split — src is not camera-real. generator_family: mixed-generation content_category: diverse + + # ── Released v23 holdouts ───────────────────────────────────────────────── + + - name: synthetic-human + path: 34data/synthetic-human + modality: image + media_type: synthetic + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: mixed-generation + content_category: faces + + - name: ai-generated-ecommerce-damaged-product + path: 34data/ai-generated-ecommerce-damaged-product + modality: image + media_type: synthetic + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: mixed-generation + content_category: diverse + + - name: ai-generated-ecommerce-fake-logistics + path: 34data/ai-generated-ecommerce-fake-logistics + modality: image + media_type: synthetic + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: mixed-generation + content_category: diverse diff --git a/src/gasbench/dataset/configs/synthetic_videos.yaml b/src/gasbench/dataset/configs/synthetic_videos.yaml index 7ed8cb6..eb997b8 100644 --- a/src/gasbench/dataset/configs/synthetic_videos.yaml +++ b/src/gasbench/dataset/configs/synthetic_videos.yaml @@ -1266,3 +1266,75 @@ datasets: generator_family: mixed-generation content_category: diverse notes: Released from v22 bmcore holdouts + + # ── Released v23 holdouts ───────────────────────────────────────────────── + + - name: videogen-rewardbench-easyanimatev4 + path: 34data/videogen-rewardbench-easyanimatev4 + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: easyanimate + content_category: diverse + + - name: videogen-rewardbench-opensora1-2 + path: 34data/videogen-rewardbench-opensora1-2 + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: opensora + content_category: diverse + + - name: videogen-rewardbench-qingying + path: 34data/videogen-rewardbench-qingying + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: qingying + content_category: diverse + + - name: videogen-rewardbench-tongyi + path: 34data/videogen-rewardbench-tongyi + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: tongyi + content_category: diverse + + - name: videogen-rewardbench-vidu + path: 34data/videogen-rewardbench-vidu + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: vidu + content_category: diverse + + - name: edtalk + path: 34data/edtalk + modality: video + media_type: semisynthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: edtalk + content_category: faces + + - name: float-talking-head + path: 34data/float-talking-head + modality: video + media_type: semisynthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: float + content_category: faces From 511dc698777b999e9209d996326612f09fd2cd7d Mon Sep 17 00:00:00 2001 From: dylan Date: Sat, 5 Sep 2026 02:35:04 +0000 Subject: [PATCH 2/5] data: register 22 v23 holdouts against upstream repos (no 34data mirror) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second half of the v23 release. These 22 holdouts already exist in public upstream datasets, so instead of copying Wasabi -> 34data they point at the upstream repo directly. Same approach already used for cml-tts-german/spanish, ai4bharat-speecharenabench-kn/mr and indic-tts-kannada/malayalam. ylacombe/cml-tts 4 dutch, italian, polish, portuguese Sumsub/Swappir 5 GPEN x2, roop x2, simswap scaledf/ScaleDF 4 FFpp Deepfakes/Face2Face, E4E, DiffusionCLIP RekaAI/RekaDaily-10k-raw 3 phone shards 00000-00002 obscure-entropy/conceptual_captions_hu_filtered 1 eltorio/ROCOv2-radiology 1 rshaojimmy/DGM4 1 manipulation/StyleCLIP.zip MCG-NJU/TimeLens2-93K 1 videos-00003-of-00018.tar mvp-lab/LLaVA-OneVision-2-Data 1 mid_training_video/30s WenhaoWang/TIP-I2V 1 i2vgenxl_videos_subset_1.tar Upstream filenames match the Wasabi path segments exactly, so these resolve to the same material that was held out. Every repo is ungated and every entry pins hf_revision, so an upstream force-push cannot silently change what the benchmark reads. include_paths is a substring match, which is a trap here: 'CelebA_HQ_roop.zip' also matches 'GPEN_CelebA_HQ_roop.zip', and 'fairface_roop.zip' also matches 'GPEN_fairface_roop.zip'. Both carry exclude_paths: ['GPEN_'] so they resolve to one file each. Every include_paths was checked against the pinned revision's actual file listing: 20 resolve to exactly 1 archive, rocov2 to its 27 train shards, cml-tts to 373/61/12/42 per language. ScaleDF is pinned to scaledf/ScaleDF (the org repo) rather than WenhaoWang/ScaleDF; both carry all four tars. Archive bounds follow the same reasoning as the mirrored half — image 1000/1, video 500/1, audio 500/2, all clearing the per-dataset caps, which land at 335 / 145 / 327 with these 22 added. Registry loads clean at 587 datasets, no validation failures, no duplicate names and no duplicate (path, include_paths) pairs introduced. --- src/gasbench/dataset/configs/real_audio.yaml | 54 ++++++++ src/gasbench/dataset/configs/real_images.yaml | 26 ++++ src/gasbench/dataset/configs/real_videos.yaml | 62 +++++++++ .../dataset/configs/synthetic_images.yaml | 125 ++++++++++++++++++ .../dataset/configs/synthetic_videos.yaml | 14 ++ 5 files changed, 281 insertions(+) diff --git a/src/gasbench/dataset/configs/real_audio.yaml b/src/gasbench/dataset/configs/real_audio.yaml index f36703d..a29ac13 100644 --- a/src/gasbench/dataset/configs/real_audio.yaml +++ b/src/gasbench/dataset/configs/real_audio.yaml @@ -755,3 +755,57 @@ datasets: generator_family: real content_category: speech notes: CML-TTS Spanish train split, released from v22 bmcore holdouts + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: cml-tts-dutch + path: ylacombe/cml-tts + hf_revision: 48aedd7a8ea08efc75cae3e81fbefa9720df0812 + modality: audio + media_type: real + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_subfolders: ['dutch'] + include_paths: ['train-'] + generator_family: real + content_category: speech + + - name: cml-tts-italian + path: ylacombe/cml-tts + hf_revision: 48aedd7a8ea08efc75cae3e81fbefa9720df0812 + modality: audio + media_type: real + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_subfolders: ['italian'] + include_paths: ['train-'] + generator_family: real + content_category: speech + + - name: cml-tts-polish + path: ylacombe/cml-tts + hf_revision: 48aedd7a8ea08efc75cae3e81fbefa9720df0812 + modality: audio + media_type: real + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_subfolders: ['polish'] + include_paths: ['train-'] + generator_family: real + content_category: speech + + - name: cml-tts-portuguese + path: ylacombe/cml-tts + hf_revision: 48aedd7a8ea08efc75cae3e81fbefa9720df0812 + modality: audio + media_type: real + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_subfolders: ['portuguese'] + include_paths: ['train-'] + generator_family: real + content_category: speech diff --git a/src/gasbench/dataset/configs/real_images.yaml b/src/gasbench/dataset/configs/real_images.yaml index 39f68f3..c9b5452 100644 --- a/src/gasbench/dataset/configs/real_images.yaml +++ b/src/gasbench/dataset/configs/real_images.yaml @@ -1223,3 +1223,29 @@ datasets: archives_per_dataset: 1 generator_family: real content_category: medical + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: conceptual-captions-0-1m + path: obscure-entropy/conceptual_captions_hu_filtered + hf_revision: e70871e77287ca4aec4a12ff2f9cec32cad5694a + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['data/conceptual_captions_0_1m.parquet'] + generator_family: real + content_category: diverse + + - name: rocov2-radiology + path: eltorio/ROCOv2-radiology + hf_revision: 80ffeef4eb8d34d27cb5c2815305f1d8aee8a83c + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['data/train-'] + generator_family: real + content_category: medical diff --git a/src/gasbench/dataset/configs/real_videos.yaml b/src/gasbench/dataset/configs/real_videos.yaml index a0fc879..f006803 100644 --- a/src/gasbench/dataset/configs/real_videos.yaml +++ b/src/gasbench/dataset/configs/real_videos.yaml @@ -1093,3 +1093,65 @@ datasets: generator_family: real content_category: action notes: Released from v22 bmcore holdouts + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: timelens2-93k-00003 + path: MCG-NJU/TimeLens2-93K + hf_revision: fce4049cdf5d720e2651b956c58e664c3386c021 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['videos/videos-00003-of-00018.tar'] + generator_family: real + content_category: diverse + + - name: llava-onevision-2-mid-training-00000 + path: mvp-lab/LLaVA-OneVision-2-Data + hf_revision: bf7038ba5d17771dc113a3072f98591ad2e29541 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['mid_training_video/30s/train_00000_of_09707.tar'] + generator_family: real + content_category: diverse + + - name: rekadaily-phone-shard-00000 + path: RekaAI/RekaDaily-10k-raw + hf_revision: a42e2da9aaeef7c8653d9de50772c97ceb954251 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['data/video_capture_first_person_videos_phone/shard-00000.tar'] + generator_family: real + content_category: action + + - name: rekadaily-phone-shard-00001 + path: RekaAI/RekaDaily-10k-raw + hf_revision: a42e2da9aaeef7c8653d9de50772c97ceb954251 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['data/video_capture_first_person_videos_phone/shard-00001.tar'] + generator_family: real + content_category: action + + - name: rekadaily-phone-shard-00002 + path: RekaAI/RekaDaily-10k-raw + hf_revision: a42e2da9aaeef7c8653d9de50772c97ceb954251 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['data/video_capture_first_person_videos_phone/shard-00002.tar'] + generator_family: real + content_category: action diff --git a/src/gasbench/dataset/configs/synthetic_images.yaml b/src/gasbench/dataset/configs/synthetic_images.yaml index ef12539..76da567 100644 --- a/src/gasbench/dataset/configs/synthetic_images.yaml +++ b/src/gasbench/dataset/configs/synthetic_images.yaml @@ -974,3 +974,128 @@ datasets: archives_per_dataset: 1 generator_family: mixed-generation content_category: diverse + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: swappir-gpen-celeba-hq + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: synthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['GPEN_CelebA_HQ.zip'] + generator_family: gpen + content_category: faces + + - name: swappir-gpen-fairface + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: synthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['GPEN_fairface.zip'] + generator_family: gpen + content_category: faces + + - name: swappir-celeba-hq-roop + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: semisynthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['CelebA_HQ_roop.zip'] + exclude_paths: ['GPEN_'] + generator_family: faceswap + content_category: faces + + - name: swappir-celeba-hq-simswap + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: semisynthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['CelebA_HQ_simswap.zip'] + exclude_paths: ['GPEN_'] + generator_family: faceswap + content_category: faces + + - name: swappir-fairface-roop + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: semisynthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['fairface_roop.zip'] + exclude_paths: ['GPEN_'] + generator_family: faceswap + content_category: faces + + - name: scaledf-ffpp-deepfakes + path: scaledf/ScaleDF + hf_revision: fdc0f92f1f133d5c8742578917aa280b32d9d8ce + modality: image + media_type: semisynthetic + source_format: tar + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['ScaleDF/train/FFpp_Deepfakes.tar'] + generator_family: faceswap + content_category: faces + + - name: scaledf-ffpp-face2face + path: scaledf/ScaleDF + hf_revision: fdc0f92f1f133d5c8742578917aa280b32d9d8ce + modality: image + media_type: semisynthetic + source_format: tar + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['ScaleDF/train/FFpp_Face2Face.tar'] + generator_family: face2face + content_category: faces + + - name: scaledf-e4e + path: scaledf/ScaleDF + hf_revision: fdc0f92f1f133d5c8742578917aa280b32d9d8ce + modality: image + media_type: semisynthetic + source_format: tar + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['ScaleDF/train/E4E_faces.tar'] + generator_family: e4e + content_category: faces + + - name: scaledf-diffusionclip + path: scaledf/ScaleDF + hf_revision: fdc0f92f1f133d5c8742578917aa280b32d9d8ce + modality: image + media_type: semisynthetic + source_format: tar + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['ScaleDF/train/DiffusionCLIP_faces.tar'] + generator_family: diffusionclip + content_category: faces + + - name: dgm4-styleclip + path: rshaojimmy/DGM4 + hf_revision: 9efd3f1ca14cb1dac1692cdce0beea6844b968a6 + modality: image + media_type: semisynthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['manipulation/StyleCLIP.zip'] + generator_family: styleclip + content_category: faces diff --git a/src/gasbench/dataset/configs/synthetic_videos.yaml b/src/gasbench/dataset/configs/synthetic_videos.yaml index eb997b8..4758c89 100644 --- a/src/gasbench/dataset/configs/synthetic_videos.yaml +++ b/src/gasbench/dataset/configs/synthetic_videos.yaml @@ -1338,3 +1338,17 @@ datasets: archives_per_dataset: 1 generator_family: float content_category: faces + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: tip-i2v-i2vgenxl + path: WenhaoWang/TIP-I2V + hf_revision: 958f51df0ba725f53e2387bc19011a001d5ce7a4 + modality: video + media_type: synthetic + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['subset_videos_tar/i2vgenxl_videos_subset_1.tar'] + generator_family: i2vgenxl + content_category: diverse From d2e7585c32eefc3a8a3e79ed54d08cf86df6dd39 Mon Sep 17 00:00:00 2001 From: dylan Date: Sat, 5 Sep 2026 05:35:16 +0000 Subject: [PATCH 3/5] data: register the 4 v23 SpeechArenaBench holdouts Adds ai4bharat-speecharenabench-gu/ml/or/ur, pointing at ai4bharat/SpeechArenaBench with the same hf_revision, hf_subfolders and data_columns shape as the existing kn/mr entries. No upload required. Completes the v23 audio release: 9 of 9 holdouts registered, 4 real / 5 synthetic. Registry loads at 591 datasets, no validation failures, no duplicate names. --- .../dataset/configs/synthetic_audio.yaml | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/gasbench/dataset/configs/synthetic_audio.yaml b/src/gasbench/dataset/configs/synthetic_audio.yaml index da74040..8d3c996 100644 --- a/src/gasbench/dataset/configs/synthetic_audio.yaml +++ b/src/gasbench/dataset/configs/synthetic_audio.yaml @@ -808,3 +808,55 @@ datasets: archives_per_dataset: 2 generator_family: mixed-generation content_category: speech + + - name: ai4bharat-speecharenabench-gu + path: ai4bharat/SpeechArenaBench + modality: audio + media_type: synthetic + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_revision: fdd26e85ff051036e3eb49d7053d2b8f15c14ec0 + hf_subfolders: [gu] + data_columns: [audio_a] + generator_family: mixed-generation + content_category: speech + + - name: ai4bharat-speecharenabench-ml + path: ai4bharat/SpeechArenaBench + modality: audio + media_type: synthetic + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_revision: fdd26e85ff051036e3eb49d7053d2b8f15c14ec0 + hf_subfolders: [ml] + data_columns: [audio_a] + generator_family: mixed-generation + content_category: speech + + - name: ai4bharat-speecharenabench-or + path: ai4bharat/SpeechArenaBench + modality: audio + media_type: synthetic + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_revision: fdd26e85ff051036e3eb49d7053d2b8f15c14ec0 + hf_subfolders: [or] + data_columns: [audio_a] + generator_family: mixed-generation + content_category: speech + + - name: ai4bharat-speecharenabench-ur + path: ai4bharat/SpeechArenaBench + modality: audio + media_type: synthetic + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_revision: fdd26e85ff051036e3eb49d7053d2b8f15c14ec0 + hf_subfolders: [ur] + data_columns: [audio_a] + generator_family: mixed-generation + content_category: speech From fd919b1962e2bea9cf3ccfebded5a13ac508645f Mon Sep 17 00:00:00 2001 From: dylan Date: Sat, 5 Sep 2026 13:04:28 -0700 Subject: [PATCH 4/5] Select img column for conceptual captions parquet --- src/gasbench/dataset/configs/real_images.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gasbench/dataset/configs/real_images.yaml b/src/gasbench/dataset/configs/real_images.yaml index c9b5452..708c5fd 100644 --- a/src/gasbench/dataset/configs/real_images.yaml +++ b/src/gasbench/dataset/configs/real_images.yaml @@ -1232,6 +1232,7 @@ datasets: modality: image media_type: real source_format: parquet + data_columns: [img] media_per_archive: 1000 archives_per_dataset: 1 include_paths: ['data/conceptual_captions_0_1m.parquet'] From a752f8facc047050dc8cf7efcd8f4502e5117da6 Mon Sep 17 00:00:00 2001 From: dylan Date: Sat, 5 Sep 2026 15:33:48 -0700 Subject: [PATCH 5/5] Prepare 0.9.4 release and correct taxonomy test --- pyproject.toml | 2 +- tests/unit/test_multiclass_taxonomy.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f594632..ec2e515 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "gasbench" -version = "0.9.3" +version = "0.9.4" description = "GASBench - ML model benchmark evaluation package" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/unit/test_multiclass_taxonomy.py b/tests/unit/test_multiclass_taxonomy.py index d1045fe..1ce0669 100644 --- a/tests/unit/test_multiclass_taxonomy.py +++ b/tests/unit/test_multiclass_taxonomy.py @@ -147,7 +147,6 @@ def test_full_frame_neural_edits_are_synthetic(self): "fakeparts-stylechange", "senorita-controllable-videos", "senorita-style-transfer", - "semisynthetic-video", ): assert names[name].media_type == "synthetic", name