diff --git a/pyproject.toml b/pyproject.toml index f594632..ec2e515 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "gasbench" -version = "0.9.3" +version = "0.9.4" description = "GASBench - ML model benchmark evaluation package" readme = "README.md" requires-python = ">=3.10" diff --git a/src/gasbench/dataset/config.py b/src/gasbench/dataset/config.py index a814e2d..5c5c2b3 100644 --- a/src/gasbench/dataset/config.py +++ b/src/gasbench/dataset/config.py @@ -25,7 +25,7 @@ BENCHMARK_TOTAL_OVERRIDES = { "debug": {"image": 100, "video": 50, "audio": 50}, "small": {"image": 5000, "video": 5000, "audio": 5000}, - "full": {"image": 69000, "video": 33000, "audio": 47000}, + "full": {"image": 72500, "video": 34000, "audio": 47500}, } # Per-dataset download limits (only applied in debug/small modes for faster testing) diff --git a/src/gasbench/dataset/configs/real_audio.yaml b/src/gasbench/dataset/configs/real_audio.yaml index f36703d..a29ac13 100644 --- a/src/gasbench/dataset/configs/real_audio.yaml +++ b/src/gasbench/dataset/configs/real_audio.yaml @@ -755,3 +755,57 @@ datasets: generator_family: real content_category: speech notes: CML-TTS Spanish train split, released from v22 bmcore holdouts + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: cml-tts-dutch + path: ylacombe/cml-tts + hf_revision: 48aedd7a8ea08efc75cae3e81fbefa9720df0812 + modality: audio + media_type: real + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_subfolders: ['dutch'] + include_paths: ['train-'] + generator_family: real + content_category: speech + + - name: cml-tts-italian + path: ylacombe/cml-tts + hf_revision: 48aedd7a8ea08efc75cae3e81fbefa9720df0812 + modality: audio + media_type: real + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_subfolders: ['italian'] + include_paths: ['train-'] + generator_family: real + content_category: speech + + - name: cml-tts-polish + path: ylacombe/cml-tts + hf_revision: 48aedd7a8ea08efc75cae3e81fbefa9720df0812 + modality: audio + media_type: real + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_subfolders: ['polish'] + include_paths: ['train-'] + generator_family: real + content_category: speech + + - name: cml-tts-portuguese + path: ylacombe/cml-tts + hf_revision: 48aedd7a8ea08efc75cae3e81fbefa9720df0812 + modality: audio + media_type: real + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_subfolders: ['portuguese'] + include_paths: ['train-'] + generator_family: real + content_category: speech diff --git a/src/gasbench/dataset/configs/real_images.yaml b/src/gasbench/dataset/configs/real_images.yaml index 3a9afac..708c5fd 100644 --- a/src/gasbench/dataset/configs/real_images.yaml +++ b/src/gasbench/dataset/configs/real_images.yaml @@ -1151,3 +1151,102 @@ datasets: generator_family: real content_category: documents notes: Released from v22 bmcore holdouts + + # ── Released v23 holdouts ───────────────────────────────────────────────── + + - name: kidney-stone-ct + path: 34data/kidney-stone-ct + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: malaria-bounding-boxes + path: 34data/malaria-bounding-boxes + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: pulmonary-chest-xray-abnormalities + path: 34data/pulmonary-chest-xray-abnormalities + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: dermnet + path: 34data/dermnet + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: coronahack-chest-xray + path: 34data/coronahack-chest-xray + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: lits-256 + path: 34data/lits-256 + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + - name: tcga-coad-msi-mss + path: 34data/tcga-coad-msi-mss + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: real + content_category: medical + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: conceptual-captions-0-1m + path: obscure-entropy/conceptual_captions_hu_filtered + hf_revision: e70871e77287ca4aec4a12ff2f9cec32cad5694a + modality: image + media_type: real + source_format: parquet + data_columns: [img] + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['data/conceptual_captions_0_1m.parquet'] + generator_family: real + content_category: diverse + + - name: rocov2-radiology + path: eltorio/ROCOv2-radiology + hf_revision: 80ffeef4eb8d34d27cb5c2815305f1d8aee8a83c + modality: image + media_type: real + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['data/train-'] + generator_family: real + content_category: medical diff --git a/src/gasbench/dataset/configs/real_videos.yaml b/src/gasbench/dataset/configs/real_videos.yaml index a0fc879..f006803 100644 --- a/src/gasbench/dataset/configs/real_videos.yaml +++ b/src/gasbench/dataset/configs/real_videos.yaml @@ -1093,3 +1093,65 @@ datasets: generator_family: real content_category: action notes: Released from v22 bmcore holdouts + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: timelens2-93k-00003 + path: MCG-NJU/TimeLens2-93K + hf_revision: fce4049cdf5d720e2651b956c58e664c3386c021 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['videos/videos-00003-of-00018.tar'] + generator_family: real + content_category: diverse + + - name: llava-onevision-2-mid-training-00000 + path: mvp-lab/LLaVA-OneVision-2-Data + hf_revision: bf7038ba5d17771dc113a3072f98591ad2e29541 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['mid_training_video/30s/train_00000_of_09707.tar'] + generator_family: real + content_category: diverse + + - name: rekadaily-phone-shard-00000 + path: RekaAI/RekaDaily-10k-raw + hf_revision: a42e2da9aaeef7c8653d9de50772c97ceb954251 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['data/video_capture_first_person_videos_phone/shard-00000.tar'] + generator_family: real + content_category: action + + - name: rekadaily-phone-shard-00001 + path: RekaAI/RekaDaily-10k-raw + hf_revision: a42e2da9aaeef7c8653d9de50772c97ceb954251 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['data/video_capture_first_person_videos_phone/shard-00001.tar'] + generator_family: real + content_category: action + + - name: rekadaily-phone-shard-00002 + path: RekaAI/RekaDaily-10k-raw + hf_revision: a42e2da9aaeef7c8653d9de50772c97ceb954251 + modality: video + media_type: real + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['data/video_capture_first_person_videos_phone/shard-00002.tar'] + generator_family: real + content_category: action diff --git a/src/gasbench/dataset/configs/synthetic_audio.yaml b/src/gasbench/dataset/configs/synthetic_audio.yaml index c285ba2..8d3c996 100644 --- a/src/gasbench/dataset/configs/synthetic_audio.yaml +++ b/src/gasbench/dataset/configs/synthetic_audio.yaml @@ -796,3 +796,67 @@ datasets: generator_family: mixed-generation content_category: speech notes: Synthetic subset of the official RuASD anti-spoofing dataset + + # ── Released v23 holdouts ───────────────────────────────────────────────── + + - name: indic-tts-urdu + path: 34data/indic-tts-urdu + modality: audio + media_type: synthetic + source_format: zip + media_per_archive: 500 + archives_per_dataset: 2 + generator_family: mixed-generation + content_category: speech + + - name: ai4bharat-speecharenabench-gu + path: ai4bharat/SpeechArenaBench + modality: audio + media_type: synthetic + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_revision: fdd26e85ff051036e3eb49d7053d2b8f15c14ec0 + hf_subfolders: [gu] + data_columns: [audio_a] + generator_family: mixed-generation + content_category: speech + + - name: ai4bharat-speecharenabench-ml + path: ai4bharat/SpeechArenaBench + modality: audio + media_type: synthetic + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_revision: fdd26e85ff051036e3eb49d7053d2b8f15c14ec0 + hf_subfolders: [ml] + data_columns: [audio_a] + generator_family: mixed-generation + content_category: speech + + - name: ai4bharat-speecharenabench-or + path: ai4bharat/SpeechArenaBench + modality: audio + media_type: synthetic + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_revision: fdd26e85ff051036e3eb49d7053d2b8f15c14ec0 + hf_subfolders: [or] + data_columns: [audio_a] + generator_family: mixed-generation + content_category: speech + + - name: ai4bharat-speecharenabench-ur + path: ai4bharat/SpeechArenaBench + modality: audio + media_type: synthetic + source_format: parquet + media_per_archive: 500 + archives_per_dataset: 2 + hf_revision: fdd26e85ff051036e3eb49d7053d2b8f15c14ec0 + hf_subfolders: [ur] + data_columns: [audio_a] + generator_family: mixed-generation + content_category: speech diff --git a/src/gasbench/dataset/configs/synthetic_images.yaml b/src/gasbench/dataset/configs/synthetic_images.yaml index a031471..76da567 100644 --- a/src/gasbench/dataset/configs/synthetic_images.yaml +++ b/src/gasbench/dataset/configs/synthetic_images.yaml @@ -942,3 +942,160 @@ datasets: notes: PICA-100K is purely synthetic (PICABench; video-derived src and edited tgt). Not split — src is not camera-real. generator_family: mixed-generation content_category: diverse + + # ── Released v23 holdouts ───────────────────────────────────────────────── + + - name: synthetic-human + path: 34data/synthetic-human + modality: image + media_type: synthetic + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: mixed-generation + content_category: faces + + - name: ai-generated-ecommerce-damaged-product + path: 34data/ai-generated-ecommerce-damaged-product + modality: image + media_type: synthetic + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: mixed-generation + content_category: diverse + + - name: ai-generated-ecommerce-fake-logistics + path: 34data/ai-generated-ecommerce-fake-logistics + modality: image + media_type: synthetic + source_format: parquet + media_per_archive: 1000 + archives_per_dataset: 1 + generator_family: mixed-generation + content_category: diverse + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: swappir-gpen-celeba-hq + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: synthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['GPEN_CelebA_HQ.zip'] + generator_family: gpen + content_category: faces + + - name: swappir-gpen-fairface + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: synthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['GPEN_fairface.zip'] + generator_family: gpen + content_category: faces + + - name: swappir-celeba-hq-roop + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: semisynthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['CelebA_HQ_roop.zip'] + exclude_paths: ['GPEN_'] + generator_family: faceswap + content_category: faces + + - name: swappir-celeba-hq-simswap + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: semisynthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['CelebA_HQ_simswap.zip'] + exclude_paths: ['GPEN_'] + generator_family: faceswap + content_category: faces + + - name: swappir-fairface-roop + path: Sumsub/Swappir + hf_revision: 02bf824fdd87577979105718d066d30aae780dca + modality: image + media_type: semisynthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['fairface_roop.zip'] + exclude_paths: ['GPEN_'] + generator_family: faceswap + content_category: faces + + - name: scaledf-ffpp-deepfakes + path: scaledf/ScaleDF + hf_revision: fdc0f92f1f133d5c8742578917aa280b32d9d8ce + modality: image + media_type: semisynthetic + source_format: tar + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['ScaleDF/train/FFpp_Deepfakes.tar'] + generator_family: faceswap + content_category: faces + + - name: scaledf-ffpp-face2face + path: scaledf/ScaleDF + hf_revision: fdc0f92f1f133d5c8742578917aa280b32d9d8ce + modality: image + media_type: semisynthetic + source_format: tar + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['ScaleDF/train/FFpp_Face2Face.tar'] + generator_family: face2face + content_category: faces + + - name: scaledf-e4e + path: scaledf/ScaleDF + hf_revision: fdc0f92f1f133d5c8742578917aa280b32d9d8ce + modality: image + media_type: semisynthetic + source_format: tar + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['ScaleDF/train/E4E_faces.tar'] + generator_family: e4e + content_category: faces + + - name: scaledf-diffusionclip + path: scaledf/ScaleDF + hf_revision: fdc0f92f1f133d5c8742578917aa280b32d9d8ce + modality: image + media_type: semisynthetic + source_format: tar + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['ScaleDF/train/DiffusionCLIP_faces.tar'] + generator_family: diffusionclip + content_category: faces + + - name: dgm4-styleclip + path: rshaojimmy/DGM4 + hf_revision: 9efd3f1ca14cb1dac1692cdce0beea6844b968a6 + modality: image + media_type: semisynthetic + source_format: zip + media_per_archive: 1000 + archives_per_dataset: 1 + include_paths: ['manipulation/StyleCLIP.zip'] + generator_family: styleclip + content_category: faces diff --git a/src/gasbench/dataset/configs/synthetic_videos.yaml b/src/gasbench/dataset/configs/synthetic_videos.yaml index 7ed8cb6..4758c89 100644 --- a/src/gasbench/dataset/configs/synthetic_videos.yaml +++ b/src/gasbench/dataset/configs/synthetic_videos.yaml @@ -1266,3 +1266,89 @@ datasets: generator_family: mixed-generation content_category: diverse notes: Released from v22 bmcore holdouts + + # ── Released v23 holdouts ───────────────────────────────────────────────── + + - name: videogen-rewardbench-easyanimatev4 + path: 34data/videogen-rewardbench-easyanimatev4 + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: easyanimate + content_category: diverse + + - name: videogen-rewardbench-opensora1-2 + path: 34data/videogen-rewardbench-opensora1-2 + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: opensora + content_category: diverse + + - name: videogen-rewardbench-qingying + path: 34data/videogen-rewardbench-qingying + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: qingying + content_category: diverse + + - name: videogen-rewardbench-tongyi + path: 34data/videogen-rewardbench-tongyi + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: tongyi + content_category: diverse + + - name: videogen-rewardbench-vidu + path: 34data/videogen-rewardbench-vidu + modality: video + media_type: synthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: vidu + content_category: diverse + + - name: edtalk + path: 34data/edtalk + modality: video + media_type: semisynthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: edtalk + content_category: faces + + - name: float-talking-head + path: 34data/float-talking-head + modality: video + media_type: semisynthetic + source_format: mp4 + media_per_archive: 500 + archives_per_dataset: 1 + generator_family: float + content_category: faces + + # ── Released v23 holdouts — deferred to upstream repos (no 34data mirror) ── + + - name: tip-i2v-i2vgenxl + path: WenhaoWang/TIP-I2V + hf_revision: 958f51df0ba725f53e2387bc19011a001d5ce7a4 + modality: video + media_type: synthetic + source_format: tar + media_per_archive: 500 + archives_per_dataset: 1 + include_paths: ['subset_videos_tar/i2vgenxl_videos_subset_1.tar'] + generator_family: i2vgenxl + content_category: diverse diff --git a/tests/unit/test_multiclass_taxonomy.py b/tests/unit/test_multiclass_taxonomy.py index d1045fe..1ce0669 100644 --- a/tests/unit/test_multiclass_taxonomy.py +++ b/tests/unit/test_multiclass_taxonomy.py @@ -147,7 +147,6 @@ def test_full_frame_neural_edits_are_synthetic(self): "fakeparts-stylechange", "senorita-controllable-videos", "senorita-style-transfer", - "semisynthetic-video", ): assert names[name].media_type == "synthetic", name